[Perf][MoE] Skip padded work in SM100 FP4 DeepGEMM experts (#59044)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Robert Shaw
parent
cc2e4fecf2
commit
68ebea789a
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
|
||||
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
counts: list[int],
|
||||
group_size: int,
|
||||
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
x *= 16
|
||||
clean_x = x.clone()
|
||||
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
|
||||
output_dtype = (
|
||||
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
|
||||
)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
|
||||
|
||||
def set_counts(values):
|
||||
positions = []
|
||||
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
output.view(torch.uint8).fill_(91)
|
||||
graph.replay()
|
||||
reference_q, reference_s = quantize(x)
|
||||
assert actual_q.dtype == reference_q.dtype
|
||||
torch.testing.assert_close(
|
||||
actual_q.view(torch.uint8)[live],
|
||||
reference_q.view(torch.uint8)[live],
|
||||
|
||||
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_k = self._ACT_BLOCK_K
|
||||
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
|
||||
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
use_psum_layout = current_platform.is_device_capability_family(100)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user