[MoE] Unify DeepGEMM MegaMoE experts behind one feature-matrix layer; add NVFP4 (#61538)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
4d9a8d6a41
commit
4d58fa254a
@@ -123,19 +123,12 @@ def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
|
||||
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
|
||||
dg = SimpleNamespace(
|
||||
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
|
||||
get_block_m_for_mega_moe=lambda *_: 16,
|
||||
)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
|
||||
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
|
||||
)
|
||||
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
|
||||
module = SimpleNamespace(
|
||||
gate=SimpleNamespace(input_size=4096, output_size=128),
|
||||
experts=SimpleNamespace(
|
||||
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
|
||||
),
|
||||
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
|
||||
use_sequence_parallel=False,
|
||||
)
|
||||
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
|
||||
|
||||
Reference in New Issue
Block a user