add epilogue hook to flex attention (#45841)
Signed-off-by: Angel Li <liangel@meta.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Matthew Bonanni
Michael Goin
parent
98e91a9600
commit
9b9fc4039c
@@ -12,6 +12,7 @@ import torch
|
||||
import torch._dynamo.decorators
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.attention.flex_attention import (
|
||||
AuxRequest,
|
||||
BlockMask,
|
||||
_mask_mod_signature,
|
||||
_score_mod_signature,
|
||||
@@ -1228,6 +1229,9 @@ class FlexAttentionImpl(AttentionImpl):
|
||||
if block_n is not None:
|
||||
self.block_n = block_n
|
||||
|
||||
# Optional post-attention epilogue transform
|
||||
self.out_transform = kwargs.get("out_transform")
|
||||
|
||||
@staticmethod
|
||||
def view_as_4d(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""View a 3d tensor as 4D."""
|
||||
@@ -1392,8 +1396,13 @@ class FlexAttentionImpl(AttentionImpl):
|
||||
self.scale,
|
||||
enable_gqa=enable_gqa,
|
||||
kernel_options=kernel_options,
|
||||
return_aux=AuxRequest(lse=True) if self.out_transform is not None else None,
|
||||
)
|
||||
|
||||
if self.out_transform is not None:
|
||||
out, aux = out
|
||||
out = self.out_transform(out, aux.lse)
|
||||
|
||||
# Flex doesn't have an out variant today, rely on epilogue fusion
|
||||
out = out.permute(0, 2, 1, 3).squeeze(0)
|
||||
output[:num_actual_tokens, :, :].copy_(out)
|
||||
|
||||
Reference in New Issue
Block a user