[DSv4.1] Integrate Mega-Gate from DeepGEMM (#56266)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Thien Tran
2026-09-17 12:06:36 -07:00
committed by GitHub
co-authored by OpenAI Codex zjy0516
parent a9a7e45f31
commit 41f9104fa6
9 changed files with 369 additions and 35 deletions
+3 -2
View File
@@ -132,6 +132,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
draft = SimpleNamespace(
use_mega_moe=False,
use_sequence_parallel=False,
hc_mult=hc_mult,
layers=[make_layer(mix) for mix in mixes],
@@ -452,7 +453,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
device=DEVICE, dtype=torch.bfloat16
)
decoder.attn = lambda positions, x, _: x * 0.5
decoder.ffn = lambda x, input_ids: x * 0.25
decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25
with torch.device(DEVICE):
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02
@@ -536,7 +537,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
device=DEVICE, dtype=torch.bfloat16
)
decoder.attn = lambda positions, x, _: x * 0.5
decoder.ffn = lambda x, input_ids: x * 0.25
decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25
with torch.device(DEVICE):
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02