From 9b9fc4039c25a6e4fe0ae97361b62edd74b8b47e Mon Sep 17 00:00:00 2001 From: liangel-02 Date: Tue, 28 Jul 2026 07:26:55 -0600 Subject: [PATCH] add epilogue hook to flex attention (#45841) Signed-off-by: Angel Li Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Matthew Bonanni Co-authored-by: Michael Goin --- vllm/v1/attention/backends/flex_attention.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index e7fe585219..83144751ae 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -12,6 +12,7 @@ import torch import torch._dynamo.decorators import torch.nn.functional as F from torch.nn.attention.flex_attention import ( + AuxRequest, BlockMask, _mask_mod_signature, _score_mod_signature, @@ -1228,6 +1229,9 @@ class FlexAttentionImpl(AttentionImpl): if block_n is not None: self.block_n = block_n + # Optional post-attention epilogue transform + self.out_transform = kwargs.get("out_transform") + @staticmethod def view_as_4d(tensor: torch.Tensor) -> torch.Tensor: """View a 3d tensor as 4D.""" @@ -1392,8 +1396,13 @@ class FlexAttentionImpl(AttentionImpl): self.scale, enable_gqa=enable_gqa, kernel_options=kernel_options, + return_aux=AuxRequest(lse=True) if self.out_transform is not None else None, ) + if self.out_transform is not None: + out, aux = out + out = self.out_transform(out, aux.lse) + # Flex doesn't have an out variant today, rely on epilogue fusion out = out.permute(0, 2, 1, 3).squeeze(0) output[:num_actual_tokens, :, :].copy_(out)