[MoE] Unify DeepGEMM MegaMoE experts behind one feature-matrix layer; add NVFP4 (#61538)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-10-15 19:02:01 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent 4d9a8d6a41
commit 4d58fa254a
10 changed files with 2753 additions and 1836 deletions
@@ -123,19 +123,12 @@ def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
dg = SimpleNamespace(
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
get_block_m_for_mega_moe=lambda *_: 16,
)
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
monkeypatch.setattr(
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
)
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
module = SimpleNamespace(
gate=SimpleNamespace(input_size=4096, output_size=128),
experts=SimpleNamespace(
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
),
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
use_sequence_parallel=False,
)
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]