[GLM5] Implement op fusion for GLM5/DSV3.2 (#46876)

This commit is contained in:
Woosuk Kwon
2026-06-28 08:17:39 -07:00
committed by GitHub
parent 5c91039c41
commit 89876b0c54
9 changed files with 1531 additions and 92 deletions
@@ -1287,6 +1287,12 @@ class FusedMoEConfig:
has_bias: bool = False
is_lora_enabled: bool = False
# When True, the MoE skips its final cross-rank all-reduce (and the separate
# shared-expert reduce), returning the partial per-rank sum. The caller is
# then responsible for the reduction (e.g. fusing it into the next RMSNorm).
# Only honored on the non-reduced (late-AR) TP path. Default False.
skip_final_all_reduce: bool = False
# SwiGLU clamp limit. When set, backends that do not implement the clamp
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
# cannot silently select one and drop the clamp.
@@ -423,6 +423,7 @@ class MoERunner(MoERunnerInterface):
if (
shared_output is not None
and not self.moe_config.is_sequence_parallel
and not self.moe_config.skip_final_all_reduce
and self._fused_output_is_reduced
):
shared_output = tensor_model_parallel_all_reduce(shared_output)
@@ -445,6 +446,7 @@ class MoERunner(MoERunnerInterface):
# - The MK already reduced the fused output itself.
if (
not self.moe_config.is_sequence_parallel
and not self.moe_config.skip_final_all_reduce
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
and not self._fused_output_is_reduced
):
@@ -227,6 +227,7 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
@@ -303,7 +304,13 @@ def sparse_attn_indexer(
scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
# The buffer must be pre-filled with -1 (the "no token" sentinel) before the
# top-k kernels scatter valid indices into it. On the fused deepseek_v32
# nvidia path, _fused_norm_rope_kernel already cleared the same
# [:num_tokens, :topk] region earlier in this forward, so skip the redundant
# fill.
if not skip_topk_buffer_clear:
topk_indices_buffer[: hidden_states.shape[0]] = -1
if has_prefill:
prefill_metadata = attn_metadata_narrowed.prefill
assert prefill_metadata is not None
@@ -546,6 +553,7 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
return topk_indices_buffer