[GLM5] Implement op fusion for GLM5/DSV3.2 (#46876)
This commit is contained in:
@@ -1287,6 +1287,12 @@ class FusedMoEConfig:
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
# When True, the MoE skips its final cross-rank all-reduce (and the separate
|
||||
# shared-expert reduce), returning the partial per-rank sum. The caller is
|
||||
# then responsible for the reduction (e.g. fusing it into the next RMSNorm).
|
||||
# Only honored on the non-reduced (late-AR) TP path. Default False.
|
||||
skip_final_all_reduce: bool = False
|
||||
|
||||
# SwiGLU clamp limit. When set, backends that do not implement the clamp
|
||||
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
|
||||
# cannot silently select one and drop the clamp.
|
||||
|
||||
@@ -423,6 +423,7 @@ class MoERunner(MoERunnerInterface):
|
||||
if (
|
||||
shared_output is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and self._fused_output_is_reduced
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
@@ -445,6 +446,7 @@ class MoERunner(MoERunnerInterface):
|
||||
# - The MK already reduced the fused output itself.
|
||||
if (
|
||||
not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not self._fused_output_is_reduced
|
||||
):
|
||||
|
||||
@@ -227,6 +227,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
@@ -303,7 +304,13 @@ def sparse_attn_indexer(
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
# The buffer must be pre-filled with -1 (the "no token" sentinel) before the
|
||||
# top-k kernels scatter valid indices into it. On the fused deepseek_v32
|
||||
# nvidia path, _fused_norm_rope_kernel already cleared the same
|
||||
# [:num_tokens, :topk] region earlier in this forward, so skip the redundant
|
||||
# fill.
|
||||
if not skip_topk_buffer_clear:
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
if has_prefill:
|
||||
prefill_metadata = attn_metadata_narrowed.prefill
|
||||
assert prefill_metadata is not None
|
||||
@@ -546,6 +553,7 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user