[K3 Perf] Fuse MXFP4 top-k finalization into latent-tail, ~5% E2E latency reduction (#53152)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wentao Ye
2026-08-21 13:29:59 -04:00
committed by GitHub
co-authored by Yongye Zhu OpenAI Codex
parent fe76112ff2
commit 7a2fdbaac4
17 changed files with 720 additions and 139 deletions
@@ -13,6 +13,7 @@ from tests.utils import (
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
from vllm.platforms import current_platform
@@ -20,6 +21,57 @@ from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
EPS = 0.1
TOP_K = 8
def _make_deferred_routed_output(
num_tokens: int,
device: torch.device,
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
num_routes = num_tokens * TOP_K
num_permuted_rows = num_routes + 7
expanded_output = torch.randn(
num_routes,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
expert_weights = torch.rand(
num_tokens,
TOP_K,
device=device,
dtype=torch.bfloat16,
)
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
gemm2_permuted = torch.empty(
num_permuted_rows,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
gemm2_permuted[expanded_idx] = expanded_output
finalized = torch.zeros(
num_tokens,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
expanded_output = expanded_output.view(num_tokens, TOP_K, LATENT_SIZE)
for slot in range(TOP_K):
finalized.add_(expanded_output[:, slot] * expert_weights[:, slot, None])
return (
UnfinalizedMoEOutput(
gemm2_permuted=gemm2_permuted,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32).view(
num_tokens, TOP_K
),
),
finalized,
)
@ray.remote(num_gpus=1, max_calls=1)
@@ -131,6 +183,99 @@ def _run_latent_moe_tail_test(
)
@ray.remote(num_gpus=1, max_calls=1)
def _test_deferred_finalize_parity_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
)
torch.manual_seed(1000 + rank)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
up_weight = (
torch.randn(
HIDDEN_SIZE,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
/ LATENT_SIZE**0.5
)
finalized_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
)
deferred_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
experts_per_token=TOP_K,
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 16)):
torch.manual_seed(2000 + 100 * iteration + rank)
deferred_output, finalized_output = _make_deferred_routed_output(
num_tokens,
device,
)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
expected = finalized_op(
finalized_output,
shared_output,
rms_weight,
up_weight,
)
actual = deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def _run_deferred_finalize_parity_test(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
) -> None:
if not current_platform.is_device_capability_family(100):
pytest.skip("K3 latent-MoE tail fusion requires SM100")
multi_process_parallel(
monkeypatch,
tp_size,
1,
_test_deferred_finalize_parity_worker,
)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_tp8_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
@@ -143,3 +288,10 @@ def test_latent_moe_tail_tp16_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 16)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_deferred_finalize_matches_finalized(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_deferred_finalize_parity_test(monkeypatch, 8)