[GLM5] Implement op fusion for GLM5/DSV3.2 (#46876)

This commit is contained in:
Woosuk Kwon
2026-06-28 08:17:39 -07:00
committed by GitHub
parent 5c91039c41
commit 89876b0c54
9 changed files with 1531 additions and 92 deletions
@@ -227,6 +227,7 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
@@ -303,7 +304,13 @@ def sparse_attn_indexer(
scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
# The buffer must be pre-filled with -1 (the "no token" sentinel) before the
# top-k kernels scatter valid indices into it. On the fused deepseek_v32
# nvidia path, _fused_norm_rope_kernel already cleared the same
# [:num_tokens, :topk] region earlier in this forward, so skip the redundant
# fill.
if not skip_topk_buffer_clear:
topk_indices_buffer[: hidden_states.shape[0]] = -1
if has_prefill:
prefill_metadata = attn_metadata_narrowed.prefill
assert prefill_metadata is not None
@@ -546,6 +553,7 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
return topk_indices_buffer