[Kernel]Support W4A8 Grouped GEMM on Hopper (#29691)
Signed-off-by: czhu-cohere <conway.zhu@cohere.com>
This commit is contained in:
@@ -63,8 +63,10 @@ if HAS_TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
CutlassBatchedExpertsFp8,
|
||||
CutlassExpertsFp8,
|
||||
CutlassExpertsW4A8Fp8,
|
||||
cutlass_moe_fp4,
|
||||
cutlass_moe_fp8,
|
||||
cutlass_moe_w4a8_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import DeepGemmExperts
|
||||
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
|
||||
@@ -88,8 +90,10 @@ if HAS_TRITON:
|
||||
"grouped_topk",
|
||||
"cutlass_moe_fp8",
|
||||
"cutlass_moe_fp4",
|
||||
"cutlass_moe_w4a8_fp8",
|
||||
"CutlassExpertsFp8",
|
||||
"CutlassBatchedExpertsFp8",
|
||||
"CutlassExpertsW4A8Fp8",
|
||||
"TritonExperts",
|
||||
"BatchedTritonExperts",
|
||||
"DeepGemmExperts",
|
||||
|
||||
@@ -143,6 +143,7 @@ class FusedMoEQuantDesc:
|
||||
scale: Union[torch.Tensor, "PrecisionConfig", None] = None
|
||||
|
||||
# Quantization alphas or gscales, used for nvfp4 types.
|
||||
# W4A8 FP8: used for per-channel scales
|
||||
# TODO(bnell): put some of these in subclasses
|
||||
alpha_or_gscale: torch.Tensor | None = None
|
||||
|
||||
@@ -442,7 +443,9 @@ class FusedMoEQuantConfig:
|
||||
- a1_scale: Optional scale to be used for a1.
|
||||
- a2_scale: Optional scale to be used for a2.
|
||||
- g1_alphas: Optional global quantization scales for w1 (for nvfp4).
|
||||
per-channel scales for w1 (for W4A8 FP8).
|
||||
- g2_alphas: Optional global quantization scales for w2 (for nvfp4).
|
||||
per-channel scales for w2 (for W4A8 FP8).
|
||||
- a1_gscale: Optional global quantization scales for a1 (for nvfp4).
|
||||
- a2_gscale: Optional global quantization scales for a2 (for nvfp4).
|
||||
- w1_bias: Optional biases for w1 (GPT OSS Triton).
|
||||
@@ -461,6 +464,7 @@ class FusedMoEQuantConfig:
|
||||
"mxfp4",
|
||||
"mxfp6_e3m2",
|
||||
"mxfp6_e2m3",
|
||||
"int4",
|
||||
}
|
||||
|
||||
if weight_dtype is None:
|
||||
@@ -671,6 +675,31 @@ def int8_w8a16_moe_quant_config(
|
||||
)
|
||||
|
||||
|
||||
def int4_w4afp8_moe_quant_config(
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
g1_alphas: torch.Tensor,
|
||||
g2_alphas: torch.Tensor,
|
||||
per_act_token_quant: bool = False,
|
||||
per_out_ch_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for fp8 activations and int4 weights.
|
||||
"""
|
||||
return FusedMoEQuantConfig.make(
|
||||
torch.float8_e4m3fn, # quant dtype for activations
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
per_out_ch_quant=per_out_ch_quant,
|
||||
block_shape=block_shape,
|
||||
weight_dtype="int4", # weight dtype for weights
|
||||
)
|
||||
|
||||
|
||||
def biased_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
|
||||
@@ -1052,3 +1052,404 @@ def run_cutlass_block_scaled_fused_experts(
|
||||
return (
|
||||
c2[c_map].view(m, topk, k) * topk_weights.view(m, topk, 1).to(out_dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
# W4A8
|
||||
def run_cutlass_moe_w4a8_fp8(
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation_callable: Callable,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
w1_chan_scale: torch.Tensor,
|
||||
w2_chan_scale: torch.Tensor,
|
||||
a_strides1: torch.Tensor,
|
||||
a_strides2: torch.Tensor,
|
||||
b_strides1: torch.Tensor,
|
||||
b_strides2: torch.Tensor,
|
||||
c_strides1: torch.Tensor,
|
||||
c_strides2: torch.Tensor,
|
||||
s_strides1: torch.Tensor,
|
||||
s_strides2: torch.Tensor,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_num_tokens: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
per_act_token: bool,
|
||||
per_out_ch: bool,
|
||||
use_batched_format: bool,
|
||||
topk_weights: torch.Tensor | None,
|
||||
group_size: int,
|
||||
):
|
||||
a1q = hidden_states
|
||||
M = a1q.size(0)
|
||||
local_E = w1.size(0)
|
||||
device = a1q.device
|
||||
_, K, N_packed = w2.shape
|
||||
N = N_packed * 8 # logical N, pack 8 int4 into 1 int32
|
||||
|
||||
assert per_act_token, "W4A8 must use per-token scales"
|
||||
assert per_out_ch, "W4A8 must use per-channel scales"
|
||||
assert w1_scale is not None
|
||||
assert w2_scale is not None
|
||||
assert w1_scale.dtype == torch.float8_e4m3fn
|
||||
assert w2_scale.dtype == torch.float8_e4m3fn
|
||||
assert w1.dtype == torch.int32
|
||||
assert w2.dtype == torch.int32
|
||||
assert w1_chan_scale.dtype == torch.float32
|
||||
assert w2_chan_scale.dtype == torch.float32
|
||||
assert w1.size(0) == w2.size(0), "Weights expert number mismatch"
|
||||
assert a1q_scale is not None
|
||||
assert a2_scale is None
|
||||
assert out_dtype in [torch.bfloat16], f"Invalid output dtype: {out_dtype}"
|
||||
if expert_map is not None:
|
||||
assert expert_num_tokens is None
|
||||
assert not use_batched_format, "batched format not supported yet"
|
||||
assert group_size == 128, f"Only group size 128 supported but got {group_size=}"
|
||||
|
||||
assert global_num_experts != -1
|
||||
assert w1.size(2) * 8 == K, (
|
||||
f"w1 hidden size mismatch: got {w1.size(2) * 8}, expected {K=}"
|
||||
)
|
||||
|
||||
# Translate info from expert_map to topk_ids
|
||||
if expert_map is not None:
|
||||
local_topk_ids = torch.where(
|
||||
expert_map[topk_ids] != -1, expert_map[topk_ids], -1
|
||||
)
|
||||
else:
|
||||
local_topk_ids = topk_ids
|
||||
|
||||
topk = local_topk_ids.size(1)
|
||||
a1q_perm = _resize_cache(workspace2.view(dtype=torch.float8_e4m3fn), (M * topk, K))
|
||||
mm1_out = _resize_cache(workspace13, (M * topk, N * 2))
|
||||
act_out = _resize_cache(workspace2, (M * topk, N))
|
||||
# original workspace are based on input hidden_states dtype (bf16)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M * topk, N)
|
||||
)
|
||||
mm2_out = _resize_cache(workspace2, (M * topk, K))
|
||||
|
||||
problem_sizes1 = torch.empty(
|
||||
(global_num_experts, 3), dtype=torch.int32, device=device
|
||||
)
|
||||
problem_sizes2 = torch.empty(
|
||||
(global_num_experts, 3), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
num_expert = global_num_experts if expert_map is None else expert_map.size(0)
|
||||
# permuted a1q reuses workspace2
|
||||
a1q, a1q_scale, expert_offsets, inv_perm, _ = moe_permute(
|
||||
a1q,
|
||||
a1q_scale,
|
||||
topk_ids,
|
||||
num_expert,
|
||||
local_E,
|
||||
expert_map,
|
||||
permuted_hidden_states=a1q_perm,
|
||||
)
|
||||
expert_offsets = expert_offsets[:-1]
|
||||
|
||||
# For RS gemm SwapAB is always enabled (swap logical M, N in the problem shape)
|
||||
ops.get_cutlass_moe_mm_problem_sizes(
|
||||
local_topk_ids,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
global_num_experts,
|
||||
N,
|
||||
K,
|
||||
force_swap_ab=True,
|
||||
)
|
||||
|
||||
ops.cutlass_w4a8_moe_mm(
|
||||
mm1_out,
|
||||
a1q,
|
||||
w1,
|
||||
a1q_scale,
|
||||
w1_chan_scale,
|
||||
w1_scale,
|
||||
group_size,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
a_strides1,
|
||||
b_strides1,
|
||||
c_strides1,
|
||||
s_strides1,
|
||||
)
|
||||
|
||||
activation_callable(act_out, mm1_out)
|
||||
|
||||
a2q, a2q_scale = ops.scaled_fp8_quant(
|
||||
act_out, a2_scale, use_per_token_if_dynamic=per_act_token, output=quant_out
|
||||
)
|
||||
|
||||
if expert_map is not None:
|
||||
mm2_out.fill_(0)
|
||||
|
||||
ops.cutlass_w4a8_moe_mm(
|
||||
mm2_out,
|
||||
a2q,
|
||||
w2,
|
||||
a2q_scale,
|
||||
w2_chan_scale,
|
||||
w2_scale,
|
||||
group_size,
|
||||
expert_offsets,
|
||||
problem_sizes2,
|
||||
a_strides2,
|
||||
b_strides2,
|
||||
c_strides2,
|
||||
s_strides2,
|
||||
)
|
||||
|
||||
# for non-chunking mode the output is resized from workspace13
|
||||
# so we need to make sure mm2_out uses workspace2.
|
||||
moe_unpermute(
|
||||
out=output,
|
||||
permuted_hidden_states=mm2_out,
|
||||
topk_weights=topk_weights,
|
||||
inv_permuted_idx=inv_perm,
|
||||
)
|
||||
|
||||
|
||||
class CutlassExpertsW4A8Fp8(mk.FusedMoEPermuteExpertsUnpermute):
|
||||
def __init__(
|
||||
self,
|
||||
out_dtype: torch.dtype | None,
|
||||
a_strides1: torch.Tensor,
|
||||
a_strides2: torch.Tensor,
|
||||
b_strides1: torch.Tensor,
|
||||
b_strides2: torch.Tensor,
|
||||
c_strides1: torch.Tensor,
|
||||
c_strides2: torch.Tensor,
|
||||
s_strides1: torch.Tensor,
|
||||
s_strides2: torch.Tensor,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
group_size: int,
|
||||
):
|
||||
super().__init__(quant_config)
|
||||
self.out_dtype = out_dtype
|
||||
self.a_strides1 = a_strides1
|
||||
self.a_strides2 = a_strides2
|
||||
self.b_strides1 = b_strides1
|
||||
self.b_strides2 = b_strides2
|
||||
self.c_strides1 = c_strides1
|
||||
self.c_strides2 = c_strides2
|
||||
self.s_strides1 = s_strides1
|
||||
self.s_strides2 = s_strides2
|
||||
self.group_size = group_size
|
||||
|
||||
@property
|
||||
def activation_formats(
|
||||
self,
|
||||
) -> tuple[mk.FusedMoEActivationFormat, mk.FusedMoEActivationFormat]:
|
||||
return (
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
def supports_chunking(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return True
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
# topk weights and reduction are fused in moe_unpermute cuda kernel
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype:
|
||||
return self.out_dtype if self.out_dtype is not None else act_dtype
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
workspace1 = (M * topk, max(N, K))
|
||||
workspace2 = (M * topk, max(N // 2, K))
|
||||
output = (M, K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: str,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor | None,
|
||||
workspace2: torch.Tensor | None,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert self.w1_zp is None, "w1_zp is not supported in CUTLASS MoE"
|
||||
assert self.w2_zp is None, "w2_zp is not supported in CUTLASS MoE"
|
||||
|
||||
expert_num_tokens = None
|
||||
activation_callable = lambda o, i: self.activation(activation, o, i)
|
||||
|
||||
use_batched_format = (
|
||||
self.activation_formats[0] == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
)
|
||||
assert not use_batched_format, "batched format not supported"
|
||||
|
||||
in_dtype = hidden_states.dtype
|
||||
|
||||
run_cutlass_moe_w4a8_fp8(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_ids,
|
||||
activation_callable,
|
||||
global_num_experts,
|
||||
expert_map,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
a1q_scale,
|
||||
a2_scale,
|
||||
self.g1_alphas, # per-channel scales
|
||||
self.g2_alphas, # per-channel scales
|
||||
self.a_strides1,
|
||||
self.a_strides2,
|
||||
self.b_strides1,
|
||||
self.b_strides2,
|
||||
self.c_strides1,
|
||||
self.c_strides2,
|
||||
self.s_strides1,
|
||||
self.s_strides2,
|
||||
workspace13,
|
||||
workspace2,
|
||||
expert_num_tokens,
|
||||
self.out_dtype if self.out_dtype is not None else in_dtype,
|
||||
self.per_act_token_quant,
|
||||
self.per_out_ch_quant,
|
||||
use_batched_format,
|
||||
topk_weights,
|
||||
self.group_size,
|
||||
)
|
||||
|
||||
|
||||
def cutlass_moe_w4a8_fp8(
|
||||
a: torch.Tensor,
|
||||
w1_q: torch.Tensor,
|
||||
w2_q: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
a_strides1: torch.Tensor,
|
||||
a_strides2: torch.Tensor,
|
||||
b_strides1: torch.Tensor,
|
||||
b_strides2: torch.Tensor,
|
||||
c_strides1: torch.Tensor,
|
||||
c_strides2: torch.Tensor,
|
||||
s_strides1: torch.Tensor,
|
||||
s_strides2: torch.Tensor,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
activation: str = "silu",
|
||||
expert_map: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
global_num_experts: int = -1,
|
||||
group_size: int = 128,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a w4a8-quantized Mixture of Experts (MoE) layer
|
||||
using two sets of quantized weights, w1_q and w2_q, and top-k gating
|
||||
mechanism. The matrix multiplications are implemented with CUTLASS
|
||||
mixed-dtype grouped gemm.
|
||||
|
||||
Parameters:
|
||||
- a (torch.Tensor): The input tensor to the MoE layer.
|
||||
Shape: [M, K]
|
||||
- w1_q (torch.Tensor): The first set of fp8-quantized expert weights.
|
||||
Shape: [num_experts, 2*N, K // packed_factor]
|
||||
- w2_q (torch.Tensor): The second set of fp8-quantized expert weights.
|
||||
Shape: [num_experts, K, N // packed_factor]
|
||||
- topk_weights (torch.Tensor): The weights of each token->expert mapping.
|
||||
- topk_ids (torch.Tensor): The token->expert mappings.
|
||||
- a_strides1 (torch.Tensor): The input strides for the first gemm.
|
||||
Shape: [num_experts]
|
||||
- a_strides2 (torch.Tensor): The input strides for the second gemm.
|
||||
Shape: [num_experts]
|
||||
- b_strides1 (torch.Tensor): The packed layout for the first gemm weights.
|
||||
Shape: [num_experts, 3]
|
||||
dtype: torch.int32
|
||||
- b_strides2 (torch.Tensor): The packed layout for the second gemm weights.
|
||||
Shape: [num_experts, 3]
|
||||
dtype: torch.int32
|
||||
- c_strides1 (torch.Tensor): The output strides for the first gemm.
|
||||
Shape: [num_experts]
|
||||
- c_strides2 (torch.Tensor): The output strides for the second gemm.
|
||||
Shape: [num_experts]
|
||||
- s_strides1 (torch.Tensor): strides for the group-wise scales for the first gemm.
|
||||
Shape: [num_experts, 2]
|
||||
dtype: torch.int64
|
||||
- s_strides2 (torch.Tensor): strides for the group-wise scales for the second gemm.
|
||||
Shape: [num_experts, 2]
|
||||
dtype: torch.int64
|
||||
- per_act_token (Optional[bool]): Whether the scale is per-token or
|
||||
per-tensor.
|
||||
- activation (str): The activation function to use.
|
||||
- expert_map (Optional[torch.Tensor]): In the case of Expert parallel,
|
||||
every Rank is responsible for a subset of experts. expert_map is a
|
||||
mapping from global expert-id to local expert-id. When expert_map[i]
|
||||
is -1, it means that this Rank is not responsible for global
|
||||
expert-id i.
|
||||
- apply_router_weight_on_input (bool): When true, the topk weights are
|
||||
applied directly on the inputs. This is only applicable when topk is 1.
|
||||
- global_num_experts (int): The total number of experts.
|
||||
- group_size (int): The number of weights per scale factor
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The bf16 output tensor after applying the MoE layer.
|
||||
"""
|
||||
assert quant_config is not None
|
||||
|
||||
num_experts = global_num_experts if global_num_experts != -1 else w1_q.size(0)
|
||||
|
||||
fn = mk.FusedMoEModularKernel(
|
||||
MoEPrepareAndFinalizeNoEP(),
|
||||
CutlassExpertsW4A8Fp8(
|
||||
out_dtype=a.dtype,
|
||||
a_strides1=a_strides1,
|
||||
a_strides2=a_strides2,
|
||||
b_strides1=b_strides1,
|
||||
b_strides2=b_strides2,
|
||||
c_strides1=c_strides1,
|
||||
c_strides2=c_strides2,
|
||||
s_strides1=s_strides1,
|
||||
s_strides2=s_strides2,
|
||||
quant_config=quant_config,
|
||||
group_size=group_size,
|
||||
),
|
||||
)
|
||||
|
||||
return fn(
|
||||
a,
|
||||
w1_q,
|
||||
w2_q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=expert_map,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
@@ -367,7 +367,7 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
class FusedMoEPermuteExpertsUnpermute(ABC):
|
||||
"""
|
||||
An abstract base class for the [Permute-Experts-Unpermute] step described
|
||||
above.
|
||||
above.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -256,7 +256,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
if format is not None
|
||||
else is_activation_quantization_format(quant_format)
|
||||
)
|
||||
# TODO(czhu): w4a8fp8 is in packed-quantized format
|
||||
# w4a8fp8 is in packed-quantized format
|
||||
# but needs input activation quantization
|
||||
input_activations = quant_config.get("input_activations")
|
||||
if act_quant_format or input_activations:
|
||||
|
||||
+339
-1
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
int4_w4a16_moe_quant_config,
|
||||
int4_w4afp8_moe_quant_config,
|
||||
int8_w8a8_moe_quant_config,
|
||||
int8_w8a16_moe_quant_config,
|
||||
nvfp4_moe_quant_config,
|
||||
@@ -79,7 +80,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_moe_fp8_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import swizzle_blockscale
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
convert_bf16_scales_to_fp8,
|
||||
convert_packed_uint4b8_to_signed_int4_inplace,
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
all_close_1d,
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
@@ -204,6 +209,11 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
return CompressedTensorsW8A8Int8MoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif quant_config._is_fp8_w4a8_sm90(weight_quant, input_quant):
|
||||
logger.info_once("Using CompressedTensorsW4A8Fp8MoEMethod")
|
||||
return CompressedTensorsW4A8Fp8MoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif quant_config._is_dynamic_token_w4a8_int(weight_quant, input_quant):
|
||||
return CompressedTensorsW4A8Int8MoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
@@ -2428,3 +2438,331 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
apply_router_weight_on_input,
|
||||
int(_act_kind(activation)),
|
||||
)
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant: QuantizationArgs,
|
||||
input_quant: QuantizationArgs,
|
||||
moe: FusedMoEConfig,
|
||||
layer_name: str | None = None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_quant
|
||||
self.input_quant = input_quant
|
||||
|
||||
self.group_size = self.weight_quant.group_size
|
||||
self.num_bits = self.weight_quant.num_bits
|
||||
self.packed_factor = 32 // self.num_bits
|
||||
|
||||
assert self.weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for W4A8 MoE"
|
||||
)
|
||||
assert self.weight_quant.actorder != "group"
|
||||
assert self.group_size == 128, "Only group size 128 supported for W4A8 MoE"
|
||||
|
||||
self.disable_expert_map = False
|
||||
self.layer_name = layer_name
|
||||
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
|
||||
self.quant_fp8 = QuantFP8(static=False, group_shape=GroupShape.PER_TOKEN)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
# requirement for CUTLASS reorder_tensor
|
||||
assert hidden_size % 256 == 0, f"{hidden_size=} must be divisible by 256"
|
||||
assert intermediate_size_per_partition % 256 == 0, (
|
||||
f"{intermediate_size_per_partition=} must be divisible by 256"
|
||||
)
|
||||
# storage type, pack 8xint4 into int32
|
||||
params_dtype = torch.int32
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight_packed = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // self.packed_factor,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight_packed)
|
||||
set_weight_attrs(w13_weight_packed, extra_weight_attrs)
|
||||
|
||||
w2_weight_packed = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight_packed)
|
||||
set_weight_attrs(w2_weight_packed, extra_weight_attrs)
|
||||
|
||||
# SCALES
|
||||
# weight_scale refers to the group-wise scales
|
||||
# they are initially loaded as bf16, we will convert to fp8
|
||||
# after loading
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.group_size,
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
# Add PER-GROUP quantization for FusedMoE.weight_loader.
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
# weight shapes
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
layer.register_parameter("w2_weight_shape", w2_weight_shape)
|
||||
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
|
||||
w13_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
# don't use input scales
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
device = layer.w13_weight_packed.device
|
||||
|
||||
# STRIDES
|
||||
# A, C
|
||||
self.a_strides1_c_strides2 = torch.full(
|
||||
(layer.local_num_experts,),
|
||||
layer.hidden_size,
|
||||
device=device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
self.a_strides2 = torch.full(
|
||||
(layer.local_num_experts,),
|
||||
layer.intermediate_size_per_partition,
|
||||
device=device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
self.c_strides1 = torch.full(
|
||||
(layer.local_num_experts,),
|
||||
2 * layer.intermediate_size_per_partition,
|
||||
device=device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
# S (group-wise scales)
|
||||
# sizeof(StrideS) = 16 bytes, so we need to use 2xint64 to encode it
|
||||
self.s_strides1 = torch.zeros(
|
||||
(layer.local_num_experts, 2), device=device, dtype=torch.int64
|
||||
)
|
||||
self.s_strides1[:, 0] = 2 * layer.intermediate_size_per_partition
|
||||
|
||||
self.s_strides2 = torch.zeros(
|
||||
(layer.local_num_experts, 2), device=device, dtype=torch.int64
|
||||
)
|
||||
self.s_strides2[:, 0] = layer.hidden_size
|
||||
|
||||
# encode and reorder weight tensors, and get the layout to pass to
|
||||
# the grouped gemm kernel. `b_strides1/2` specifies the entire layout
|
||||
convert_packed_uint4b8_to_signed_int4_inplace(layer.w13_weight_packed)
|
||||
w13_weight_shuffled, self.b_strides1 = (
|
||||
ops.cutlass_encode_and_reorder_int4b_grouped(layer.w13_weight_packed)
|
||||
)
|
||||
replace_parameter(layer, "w13_weight_packed", w13_weight_shuffled)
|
||||
convert_packed_uint4b8_to_signed_int4_inplace(layer.w2_weight_packed)
|
||||
w2_weight_shuffled, self.b_strides2 = (
|
||||
ops.cutlass_encode_and_reorder_int4b_grouped(layer.w2_weight_packed)
|
||||
)
|
||||
replace_parameter(layer, "w2_weight_packed", w2_weight_shuffled)
|
||||
|
||||
# convert bf16 scales to (fp8_scales, channel_scales)
|
||||
w13_weight_scale, w13_weight_chan_scale = convert_bf16_scales_to_fp8(
|
||||
self.quant_fp8, layer.w13_weight_scale
|
||||
)
|
||||
w2_weight_scale, w2_weight_chan_scale = convert_bf16_scales_to_fp8(
|
||||
self.quant_fp8, layer.w2_weight_scale
|
||||
)
|
||||
|
||||
# register channel scales
|
||||
layer.register_parameter(
|
||||
"w13_weight_chan_scale",
|
||||
torch.nn.Parameter(w13_weight_chan_scale, requires_grad=False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"w2_weight_chan_scale",
|
||||
torch.nn.Parameter(w2_weight_chan_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
# The scales are stored as (E, N, K // 128) but the kernel expects
|
||||
# (E, K // 128, N) in row-major format, so we need to permute the last 2 dims
|
||||
# and make it contiguous
|
||||
w13_weight_scale_packed = ops.cutlass_pack_scale_fp8(
|
||||
w13_weight_scale.permute(0, 2, 1).contiguous()
|
||||
)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_weight_scale_packed)
|
||||
w2_weight_scale_packed = ops.cutlass_pack_scale_fp8(
|
||||
w2_weight_scale.permute(0, 2, 1).contiguous()
|
||||
)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_weight_scale_packed)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalize | None:
|
||||
return super().maybe_make_prepare_finalize(routing_tables)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
# Store quantization scales; both per-group and per-channel
|
||||
# Note we haven't specified the group size here because
|
||||
# the quant config logic assumes group-wise scaling
|
||||
# and channel-wise scaling are exclusive.
|
||||
return int4_w4afp8_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale, # group scale
|
||||
w2_scale=layer.w2_weight_scale, # group scale
|
||||
g1_alphas=layer.w13_weight_chan_scale,
|
||||
g2_alphas=layer.w2_weight_chan_scale,
|
||||
per_act_token_quant=True, # always use dynamc per-token
|
||||
per_out_ch_quant=True, # always use per-channel
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
|
||||
layer: torch.nn.Module,
|
||||
) -> mk.FusedMoEPermuteExpertsUnpermute:
|
||||
assert self.moe_quant_config is not None
|
||||
assert (
|
||||
prepare_finalize.activation_format == FusedMoEActivationFormat.Standard
|
||||
), "BatchedExperts not supported"
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import CutlassExpertsW4A8Fp8
|
||||
|
||||
experts: FusedMoEPermuteExpertsUnpermute
|
||||
|
||||
logger.debug("CutlassExpertsW4A8Fp8(%s)", self.__class__.__name__)
|
||||
experts = CutlassExpertsW4A8Fp8(
|
||||
out_dtype=self.moe.in_dtype,
|
||||
a_strides1=self.a_strides1_c_strides2,
|
||||
a_strides2=self.a_strides2,
|
||||
b_strides1=self.b_strides1,
|
||||
b_strides2=self.b_strides2,
|
||||
c_strides1=self.c_strides1,
|
||||
c_strides2=self.a_strides1_c_strides2,
|
||||
s_strides1=self.s_strides1,
|
||||
s_strides2=self.s_strides2,
|
||||
quant_config=self.moe_quant_config,
|
||||
group_size=self.group_size,
|
||||
)
|
||||
|
||||
num_dispatchers = prepare_finalize.num_dispatchers()
|
||||
self.disable_expert_map = (
|
||||
num_dispatchers > 1 or not experts.supports_expert_map()
|
||||
)
|
||||
|
||||
return experts
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
renormalize: bool,
|
||||
use_grouped_topk: bool = False,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
enable_eplb: bool = False,
|
||||
expert_load_view: torch.Tensor | None = None,
|
||||
logical_to_physical_map: torch.Tensor | None = None,
|
||||
logical_replica_count: torch.Tensor | None = None,
|
||||
):
|
||||
if enable_eplb:
|
||||
raise NotImplementedError(
|
||||
"EPLB not supported for `CompressedTensorsW4A8Fp8MoEMethod` yet."
|
||||
)
|
||||
assert self.moe_quant_config is not None
|
||||
topk_weights, topk_ids, _ = layer.select_experts(
|
||||
hidden_states=x,
|
||||
router_logits=router_logits,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
cutlass_moe_w4a8_fp8,
|
||||
)
|
||||
|
||||
return cutlass_moe_w4a8_fp8(
|
||||
x,
|
||||
layer.w13_weight_packed,
|
||||
layer.w2_weight_packed,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_config=self.moe_quant_config,
|
||||
activation=activation,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=None if self.disable_expert_map else expert_map,
|
||||
a_strides1=self.a_strides1_c_strides2,
|
||||
a_strides2=self.a_strides2,
|
||||
b_strides1=self.b_strides1,
|
||||
b_strides2=self.b_strides2,
|
||||
c_strides1=self.c_strides1,
|
||||
c_strides2=self.a_strides1_c_strides2,
|
||||
s_strides1=self.s_strides1,
|
||||
s_strides2=self.s_strides2,
|
||||
group_size=self.group_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return False
|
||||
|
||||
+4
-11
@@ -128,14 +128,15 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
),
|
||||
)
|
||||
|
||||
# TODO(czhu): allocate the packed fp8 scales memory here?
|
||||
# the scales will be expanded by 8x via `cutlass_pack_scale_fp8`
|
||||
# After loading, we will transform bf16 -> fp8 ->
|
||||
# expand by 8x via `cutlass_pack_scale_fp8`
|
||||
# and construct per-channel fp32 scales.
|
||||
weight_scale_args = {
|
||||
"weight_loader": weight_loader,
|
||||
"data": torch.empty(
|
||||
output_size_per_partition,
|
||||
scales_and_zp_size,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
}
|
||||
|
||||
@@ -152,17 +153,9 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
data=torch.empty(2, dtype=torch.int64), weight_loader=weight_loader
|
||||
)
|
||||
|
||||
# per-channel scales
|
||||
weight_chan_scale = ChannelQuantScaleParameter(
|
||||
data=torch.empty((output_size_per_partition, 1), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("weight_packed", weight)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
layer.register_parameter("weight_shape", weight_shape)
|
||||
layer.register_parameter("weight_chan_scale", weight_chan_scale)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
|
||||
@@ -6,7 +6,11 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
convert_bf16_scales_to_fp8,
|
||||
convert_packed_uint4b8_to_signed_int4_inplace,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
@@ -48,7 +52,6 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
"CUTLASS W4A8, only supported int4",
|
||||
)
|
||||
|
||||
# TODO(czhu): support -1 (column-wise)
|
||||
if c.group_size != 128:
|
||||
return False, "Only group_size 128 is supported"
|
||||
|
||||
@@ -71,9 +74,9 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
|
||||
# `weight_scale` is: {input_dim = 0, output_dim = 1}
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
# TODO(czhu): optimize speed/mem usage
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
convert_packed_uint4b8_to_signed_int4_inplace(x.data)
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
x.data = ops.cutlass_encode_and_reorder_int4b(x.data.t().contiguous().t())
|
||||
return x
|
||||
@@ -85,10 +88,18 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
x.data = ops.cutlass_pack_scale_fp8(x.data)
|
||||
return x
|
||||
|
||||
w_s = getattr(layer, self.w_s_name)
|
||||
fp8_scales, chan_scales = convert_bf16_scales_to_fp8(self.quant_fp8, w_s.data)
|
||||
w_s.data = fp8_scales
|
||||
|
||||
# register per-channel scales
|
||||
layer.register_parameter(
|
||||
"weight_chan_scale", torch.nn.Parameter(chan_scales, requires_grad=False)
|
||||
)
|
||||
|
||||
# Encode/reorder weights and pack scales
|
||||
self._transform_param(layer, self.w_q_name, transform_w_q)
|
||||
self._transform_param(layer, self.w_s_name, transform_w_s)
|
||||
self._transform_param(layer, "weight_chan_scale", lambda x: x)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file is used for /tests and /benchmarks"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import ClassVar, NamedTuple
|
||||
@@ -691,3 +691,51 @@ def cutlass_fp4_supported() -> bool:
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
capability = -1 if capability_tuple is None else capability_tuple.to_int()
|
||||
return cutlass_scaled_mm_supports_fp4(capability)
|
||||
|
||||
|
||||
def convert_bf16_scales_to_fp8(
|
||||
quant_fp8: Callable, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
|
||||
expected by W4A8 GEMM kernels.
|
||||
"""
|
||||
assert scales.is_contiguous(), (
|
||||
f"scale tensor must be contiguous, got {scales.stride()=}"
|
||||
)
|
||||
assert scales.is_cuda, "scales must be on gpu"
|
||||
|
||||
orig_shape = scales.shape
|
||||
k_groups = orig_shape[-1]
|
||||
flat_scales = scales.view(-1, k_groups)
|
||||
|
||||
fp8_scales, chan_scales = quant_fp8(flat_scales)
|
||||
fp8_scales = (fp8_scales.float() / 8.0).to(torch.float8_e4m3fn)
|
||||
chan_scales *= 8.0
|
||||
|
||||
# restore original shape
|
||||
fp8_scales = fp8_scales.view(orig_shape)
|
||||
chan_scales = chan_scales.view(orig_shape[:-1], -1)
|
||||
|
||||
return fp8_scales, chan_scales
|
||||
|
||||
|
||||
def convert_packed_uint4b8_to_signed_int4_inplace(t: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Convert int4b8 (packed to int32) to signed int4
|
||||
"""
|
||||
assert t.is_cuda, "tensor must be on gpu"
|
||||
assert t.dtype == torch.int32, f"expected int32 packed weights but got {t.dtype}"
|
||||
|
||||
# loop through the 8 4-bit nibbles in each int32 entry
|
||||
for i in range(8):
|
||||
shift = 4 * i
|
||||
# extract the i-th 4-bit nibble
|
||||
nib = (t >> shift) & 0xF
|
||||
# clear the original nibble by masking out
|
||||
t &= ~(0xF << shift)
|
||||
# convert int4b8 [0..15] to signed int4 [-8..7] by subtracting 8
|
||||
# and update in-place
|
||||
t |= ((nib - 8) & 0xF) << shift
|
||||
|
||||
return t
|
||||
|
||||
Reference in New Issue
Block a user