[Perf] Fuse DeepEncoder relative bias in Triton attention (#55629)

Signed-off-by: levius <2114377220@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Sun
2026-09-08 13:22:04 +00:00
committed by GitHub
co-authored by Codex Isotr0py
parent 5ebce2391e
commit 472d8c9125
+191 -7
View File
@@ -22,11 +22,177 @@ from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.platforms import current_platform
from vllm.triton_utils import LOG2E, tl, triton
from .clip import CLIPEncoder, CLIPVisionEmbeddings
from .utils import AutoWeightsLoader
@triton.jit
def _deepencoder_rel_pos_attention_kernel(
q_ptr,
k_ptr,
v_ptr,
rel_h_ptr,
rel_w_ptr,
out_ptr,
scale,
stride_qb,
stride_qh,
stride_qm,
stride_qd,
stride_kb,
stride_kh,
stride_kn,
stride_kd,
stride_vb,
stride_vh,
stride_vn,
stride_vd,
stride_rhb,
stride_rhh,
stride_rhm,
stride_rhk,
stride_rwb,
stride_rwh,
stride_rwm,
stride_rwk,
stride_ob,
stride_oh,
stride_om,
stride_od,
num_tokens: tl.constexpr,
num_heads: tl.constexpr,
key_width: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_D: tl.constexpr,
):
query_block = tl.program_id(0)
batch_head = tl.program_id(1)
batch = batch_head // num_heads
head = batch_head % num_heads
offs_m = query_block * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, BLOCK_N)
offs_d = tl.arange(0, BLOCK_D)
mask_m = offs_m < num_tokens
q_offsets = (
batch * stride_qb
+ head * stride_qh
+ offs_m[:, None] * stride_qm
+ offs_d[None, :] * stride_qd
)
q = tl.load(q_ptr + q_offsets, mask=mask_m[:, None], other=0.0)
row_max = tl.full([BLOCK_M], -float("inf"), tl.float32)
row_sum = tl.zeros([BLOCK_M], tl.float32)
accumulator = tl.zeros([BLOCK_M, BLOCK_D], tl.float32)
qk_scale = scale * LOG2E
for start_n in range(0, num_tokens, BLOCK_N):
key_indices = start_n + offs_n
mask_n = key_indices < num_tokens
k_offsets = (
batch * stride_kb
+ head * stride_kh
+ key_indices[None, :] * stride_kn
+ offs_d[:, None] * stride_kd
)
k = tl.load(k_ptr + k_offsets, mask=mask_n[None, :], other=0.0)
scores = tl.dot(q, k) * qk_scale
key_h = key_indices // key_width
key_w = key_indices % key_width
rel_h_offsets = (
batch * stride_rhb
+ head * stride_rhh
+ offs_m[:, None] * stride_rhm
+ key_h[None, :] * stride_rhk
)
rel_w_offsets = (
batch * stride_rwb
+ head * stride_rwh
+ offs_m[:, None] * stride_rwm
+ key_w[None, :] * stride_rwk
)
score_mask = mask_m[:, None] & mask_n[None, :]
rel_h = tl.load(rel_h_ptr + rel_h_offsets, mask=score_mask, other=0.0)
rel_w = tl.load(rel_w_ptr + rel_w_offsets, mask=score_mask, other=0.0)
scores += (rel_h + rel_w) * LOG2E
scores = tl.where(score_mask, scores, -float("inf"))
next_max = tl.maximum(row_max, tl.max(scores, axis=1))
probabilities = tl.math.exp2(scores - next_max[:, None])
alpha = tl.math.exp2(row_max - next_max)
next_sum = row_sum * alpha + tl.sum(probabilities, axis=1)
accumulator *= alpha[:, None]
v_offsets = (
batch * stride_vb
+ head * stride_vh
+ key_indices[:, None] * stride_vn
+ offs_d[None, :] * stride_vd
)
value = tl.load(v_ptr + v_offsets, mask=mask_n[:, None], other=0.0)
accumulator = tl.dot(probabilities.to(value.dtype), value, accumulator)
row_max = next_max
row_sum = next_sum
output = accumulator / row_sum[:, None]
out_offsets = (
batch * stride_ob
+ head * stride_oh
+ offs_m[:, None] * stride_om
+ offs_d[None, :] * stride_od
)
tl.store(out_ptr + out_offsets, output, mask=mask_m[:, None])
def deepencoder_rel_pos_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
rel_h: torch.Tensor,
rel_w: torch.Tensor,
width: int,
scale: float,
) -> torch.Tensor:
"""Run DeepEncoder global attention with fused relative-position bias."""
batch, heads, num_tokens, head_dim = q.shape
output = torch.empty_like(q)
block_m = 64
block_n = 64
grid = (triton.cdiv(num_tokens, block_m), batch * heads)
_deepencoder_rel_pos_attention_kernel[grid](
q,
k,
v,
rel_h,
rel_w,
output,
scale,
*q.stride(),
*k.stride(),
*v.stride(),
*rel_h.stride(),
*rel_w.stride(),
*output.stride(),
num_tokens=num_tokens,
num_heads=heads,
key_width=width,
BLOCK_M=block_m,
BLOCK_N=block_n,
BLOCK_D=head_dim,
num_warps=4,
num_stages=2,
)
return output
class MLPBlock(nn.Module):
def __init__(
self,
@@ -234,6 +400,7 @@ class Block(nn.Module):
num_heads=num_heads,
qkv_bias=qkv_bias,
use_rel_pos=use_rel_pos,
use_triton_attention=window_size == 0,
rel_pos_zero_init=rel_pos_zero_init,
input_size=input_size if window_size == 0 else (window_size, window_size),
)
@@ -277,6 +444,7 @@ class RelPosAttention(PluggableLayer):
num_heads: int = 8,
qkv_bias: bool = True,
use_rel_pos: bool = False,
use_triton_attention: bool = False,
rel_pos_zero_init: bool = True,
input_size: tuple[int, int] | None = None,
) -> None:
@@ -285,6 +453,8 @@ class RelPosAttention(PluggableLayer):
dim (int): Number of input channels.
num_heads (int): Number of attention heads.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
use_triton_attention (bool): If True, fuse relative position bias in
a Triton attention kernel on CUDA-alike platforms.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
input_size (tuple(int, int) or None): Input resolution for calculating the relative
positional parameter size.
@@ -298,6 +468,7 @@ class RelPosAttention(PluggableLayer):
self.proj = nn.Linear(dim, dim)
self.use_rel_pos = use_rel_pos
self.use_triton_attention = use_triton_attention
if self.use_rel_pos:
assert input_size is not None, (
"Input size must be provided if using relative positional encoding."
@@ -333,14 +504,27 @@ class RelPosAttention(PluggableLayer):
rel_w = rel_w.view(
B, self.num_heads, rel_w.size(1), rel_w.size(2), rel_w.size(3)
)
attn_bias = (rel_h + rel_w).view(
B, self.num_heads, rel_h.size(2), rel_h.size(3) * rel_w.size(4)
)
x = torch.nn.functional.scaled_dot_product_attention(
q, k, v, attn_mask=attn_bias
)
use_triton = self.use_triton_attention and current_platform.is_cuda_alike()
if use_triton:
x = deepencoder_rel_pos_attention(
q,
k,
v,
rel_h.squeeze(-1),
rel_w.squeeze(-2),
W,
self.scale,
)
else:
attn_bias = (rel_h + rel_w).view(
B,
self.num_heads,
rel_h.size(2),
rel_h.size(3) * rel_w.size(4),
)
x = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias)
else:
x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
x = F.scaled_dot_product_attention(q, k, v)
x = (
x.view(B, self.num_heads, H, W, -1)