[Perf][MoE] Skip padded work in block-FP8 DeepGEMM experts (#59128)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b6d8e8afd9
commit
5ab4e4445b
@@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1(
|
||||
BLOCK_E: tl.constexpr,
|
||||
BLOCK_EXPERT_NUM: tl.constexpr,
|
||||
ALIGN_M: tl.constexpr,
|
||||
USE_PSUM_LAYOUT: tl.constexpr,
|
||||
):
|
||||
cur_expert = tl.program_id(0)
|
||||
|
||||
@@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1(
|
||||
tl.store(expert_start_loc + cur_expert, cur_expert_start)
|
||||
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
|
||||
|
||||
m_indices_start_ptr = m_indices + cur_expert_start
|
||||
off_expert = tl.arange(0, BLOCK_E)
|
||||
if USE_PSUM_LAYOUT:
|
||||
# Logical ends include preceding experts' alignment gaps.
|
||||
tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num)
|
||||
else:
|
||||
m_indices_start_ptr = m_indices + cur_expert_start
|
||||
off_expert = tl.arange(0, BLOCK_E)
|
||||
|
||||
# any rows in the per-expert aligned region that do not correspond to
|
||||
# real tokens are left untouched here and should remain initialized to
|
||||
# -1 so DeepGEMM can skip them
|
||||
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
||||
offs = start_m + off_expert
|
||||
mask = offs < cur_expert_token_num
|
||||
tl.store(
|
||||
m_indices_start_ptr + offs,
|
||||
cur_expert,
|
||||
mask=mask,
|
||||
)
|
||||
# Unwritten padding retains the -1 initialization.
|
||||
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
||||
offs = start_m + off_expert
|
||||
mask = offs < cur_expert_token_num
|
||||
tl.store(
|
||||
m_indices_start_ptr + offs,
|
||||
cur_expert,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -285,6 +288,7 @@ def ep_scatter(
|
||||
align_m: int = 128,
|
||||
block_size: int = 128,
|
||||
pack_ue8m0: bool = False,
|
||||
use_psum_layout: bool = False,
|
||||
):
|
||||
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
|
||||
BLOCK_E = 128
|
||||
@@ -295,7 +299,10 @@ def ep_scatter(
|
||||
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
|
||||
grid = num_experts
|
||||
|
||||
assert m_indices.shape[0] % align_m == 0
|
||||
if use_psum_layout:
|
||||
assert m_indices.shape == (num_experts,)
|
||||
else:
|
||||
assert m_indices.shape[0] % align_m == 0
|
||||
assert expert_start_loc.shape[0] == num_experts
|
||||
|
||||
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
||||
@@ -311,6 +318,7 @@ def ep_scatter(
|
||||
BLOCK_E=BLOCK_E,
|
||||
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
|
||||
ALIGN_M=align_m,
|
||||
USE_PSUM_LAYOUT=use_psum_layout,
|
||||
)
|
||||
|
||||
grid = min(recv_topk.shape[0], 1024 * 8)
|
||||
@@ -465,6 +473,7 @@ def deepgemm_moe_permute(
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
aq_out: torch.Tensor | None = None,
|
||||
block_size: int | None = None,
|
||||
use_psum_layout: bool = False,
|
||||
):
|
||||
assert aq.ndim == 2
|
||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||
@@ -509,17 +518,15 @@ def deepgemm_moe_permute(
|
||||
else:
|
||||
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
|
||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
||||
# completely invalid / padded blocks that should be skipped. We always
|
||||
# initialize expert_ids to -1 so any row that is not explicitly written
|
||||
# by the scatter kernel will be treated as invalid and skipped by
|
||||
# DeepGEMM's scheduler.
|
||||
expert_ids = torch.full(
|
||||
(M_sum,),
|
||||
fill_value=-1,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if use_psum_layout:
|
||||
grouped_layout = torch.empty(
|
||||
(local_num_experts,), device=device, dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
# Negative expert IDs mark padding for the contiguous scheduler.
|
||||
grouped_layout = torch.full(
|
||||
(M_sum,), fill_value=-1, device=device, dtype=torch.int32
|
||||
)
|
||||
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
|
||||
|
||||
expert_num_tokens = None
|
||||
@@ -541,14 +548,15 @@ def deepgemm_moe_permute(
|
||||
expert_map=expert_map,
|
||||
output_tensor=aq_out,
|
||||
output_tensor_scale=aq_scale_out,
|
||||
m_indices=expert_ids,
|
||||
m_indices=grouped_layout,
|
||||
output_index=inv_perm,
|
||||
align_m=align_used,
|
||||
block_size=block_k,
|
||||
pack_ue8m0=pack_ue8m0,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
|
||||
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
|
||||
return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used
|
||||
|
||||
|
||||
def deepgemm_unpermute_and_reduce(
|
||||
|
||||
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self.block_shape is not None
|
||||
block_k = self.block_shape[1]
|
||||
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
group_size=block_k,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
|
||||
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
|
||||
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
# Both GEMMs and quantization must agree on the live expert ranges.
|
||||
use_psum_layout = (
|
||||
not self.mxfp8
|
||||
and (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and DeepGemmQuantScaleFMT.from_oracle()
|
||||
== DeepGemmQuantScaleFMT.UE8M0
|
||||
)
|
||||
)
|
||||
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
|
||||
)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
|
||||
# FP8-block keeps the default (128) alignment.
|
||||
block_size=self.block_shape[1] if self.mxfp8 else None,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
|
||||
# (1, 32); the FP8 block path keeps the default (128) recipe.
|
||||
gemm_kwargs = (
|
||||
gemm_kwargs: dict = (
|
||||
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
|
||||
if self.mxfp8
|
||||
else {}
|
||||
)
|
||||
|
||||
if use_psum_layout:
|
||||
gemm_kwargs["use_psum_layout"] = True
|
||||
|
||||
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
||||
# otherwise the scheduler can pick the wrong expert id from m_indices
|
||||
# under cudagraph replay.
|
||||
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1, self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2, self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user