[Kimi-K3][Kernel] Optimize the low-M fused latent MoE tail (#54168)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Yongye Zhu
2026-08-28 10:45:36 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 6f91e3d953
commit d9dabfa351
6 changed files with 651 additions and 192 deletions
+107 -2
View File
@@ -153,6 +153,57 @@ def _make_deferred_routed_output(
)
def _make_bf16_top16_deferred_output(
num_tokens: int,
device: torch.device,
*,
drop_last_route: bool = False,
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
top_k = 16
num_routes = num_tokens * top_k
num_permuted_rows = num_routes + 7
expanded_output = torch.randn(
num_routes,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
expert_weights = torch.rand(
num_tokens,
top_k,
device=device,
dtype=torch.bfloat16,
)
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
gemm2_permuted = torch.empty(
num_permuted_rows,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
gemm2_permuted[expanded_idx] = expanded_output
expanded_output = expanded_output.view(num_tokens, top_k, LATENT_SIZE)
expanded_idx = expanded_idx.view(num_tokens, top_k)
if drop_last_route:
expanded_output[:, -1].zero_()
expanded_idx[:, -1] = -1
finalized = (
(expanded_output.float() * expert_weights[:, :, None].float())
.sum(1)
.to(torch.bfloat16)
)
return (
UnfinalizedMoEOutput(
gemm2_permuted=gemm2_permuted,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32),
),
finalized,
)
@ray.remote(num_gpus=1, max_calls=1)
def _test_latent_moe_tail_worker(
monkeypatch: pytest.MonkeyPatch,
@@ -197,7 +248,8 @@ def _test_latent_moe_tail_worker(
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
# M=65 crosses two 32-CTA token waves and reuses the first DSM slot.
for iteration, num_tokens in enumerate((1, 5, 8, 16, 33, 65, 5)):
torch.manual_seed(100 * iteration + rank + 1)
routed_output = torch.randn(
num_tokens,
@@ -310,9 +362,17 @@ def _test_deferred_finalize_parity_worker(
rms_eps=EPS,
experts_per_token=TOP_K,
)
bf16_deferred_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
experts_per_token=16,
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 16)):
for iteration, num_tokens in enumerate((1, 5, 16, 33, 65)):
torch.manual_seed(2000 + 100 * iteration + rank)
deferred_output, finalized_output = _make_deferred_routed_output(
num_tokens,
@@ -340,6 +400,51 @@ def _test_deferred_finalize_parity_worker(
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
for iteration, num_tokens in enumerate((1, 2, 4, 5, 16)):
torch.manual_seed(3000 + 100 * iteration + rank)
deferred_output, finalized_output = _make_bf16_top16_deferred_output(
num_tokens,
device,
drop_last_route=num_tokens == 5,
)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
expected = finalized_op(
finalized_output,
shared_output,
rms_weight,
up_weight,
)
actual = bf16_deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
if num_tokens == 4:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_actual = bf16_deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
graph.replay()
torch.testing.assert_close(
graph_actual,
expected,
atol=8e-2,
rtol=3e-2,
)
def _run_deferred_finalize_parity_test(
monkeypatch: pytest.MonkeyPatch,