[Kernel] Integrate CUTLASS MoE kernel with PPLX (#18762)
Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
6e0cd10f72
commit
84166fee97
+27
-2
@@ -899,11 +899,36 @@ def shuffle_rows(input_tensor: torch.Tensor, dst2src_map: torch.Tensor):
|
||||
return output_tensor
|
||||
|
||||
|
||||
def get_cutlass_pplx_moe_mm_data(expert_offsets: torch.Tensor,
|
||||
problem_sizes1: torch.Tensor,
|
||||
problem_sizes2: torch.Tensor,
|
||||
expert_num_tokens: torch.Tensor,
|
||||
num_local_experts: int, padded_m: int, n: int,
|
||||
k: int):
|
||||
"""
|
||||
Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
used in CUTLASS-based fused MoE.
|
||||
|
||||
The function takes in expert_num_tokens (token count per expert) and
|
||||
non_zero_expert_idxs (consecutive indices of experts with non-zero token
|
||||
counts) and uses them to compute:
|
||||
- expert_offsets: Indices that mark at which token index each expert begins
|
||||
its computation.
|
||||
- problem_sizes1, problem_sizes2: MxNxK sizes of each expert's
|
||||
multiplication in two grouped MMs used in
|
||||
the fused MoE operation.
|
||||
"""
|
||||
return torch.ops._C.get_cutlass_pplx_moe_mm_data(
|
||||
expert_offsets, problem_sizes1, problem_sizes2, expert_num_tokens,
|
||||
num_local_experts, padded_m, n, k)
|
||||
|
||||
|
||||
def cutlass_moe_mm(out_tensors: torch.Tensor, a_tensors: torch.Tensor,
|
||||
b_tensors: torch.Tensor, a_scales: torch.Tensor,
|
||||
b_scales: torch.Tensor, expert_offsets: torch.Tensor,
|
||||
problem_sizes: torch.Tensor, a_strides: torch.Tensor,
|
||||
b_strides: torch.Tensor, c_strides: torch.Tensor):
|
||||
b_strides: torch.Tensor, c_strides: torch.Tensor,
|
||||
per_act_token: bool, per_out_ch: bool):
|
||||
"""
|
||||
A single grouped matrix multiplication used in CUTLASS-based fused MoE.
|
||||
The function executes fp8-quantized OUT = AB matrix multiplication.
|
||||
@@ -918,7 +943,7 @@ def cutlass_moe_mm(out_tensors: torch.Tensor, a_tensors: torch.Tensor,
|
||||
return torch.ops._C.cutlass_moe_mm(out_tensors, a_tensors, b_tensors,
|
||||
a_scales, b_scales, expert_offsets,
|
||||
problem_sizes, a_strides, b_strides,
|
||||
c_strides)
|
||||
c_strides, per_act_token, per_out_ch)
|
||||
|
||||
|
||||
def cutlass_fp4_moe_mm(a_tensors: torch.Tensor, b_tensors: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user