[Model] Add Hopper FA4 relative attention for Inkling (#48858)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -13,6 +13,8 @@ guide::
|
||||
with causal (and optionally sliding-window) masking handled by the backend.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -21,6 +23,7 @@ from vllm.models.inkling.nvidia.attention import (
|
||||
compute_log_scaling_tau,
|
||||
)
|
||||
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
|
||||
_use_sheared_bias,
|
||||
inkling_fa4_num_splits,
|
||||
inkling_fa4_rel_attention,
|
||||
)
|
||||
@@ -78,6 +81,23 @@ def test_num_splits_hopper_is_unsplit(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "expected"),
|
||||
[(9, False), (10, True), (11, True), (12, False)],
|
||||
)
|
||||
def test_sheared_bias_architecture_selection(monkeypatch, major, expected):
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(major=major, minor=0),
|
||||
)
|
||||
_use_sheared_bias.cache_clear()
|
||||
try:
|
||||
assert _use_sheared_bias() is expected
|
||||
finally:
|
||||
_use_sheared_bias.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_platform(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
@@ -251,6 +271,14 @@ def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
preallocated_out = torch.empty_like(q)
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=window_left is not None,
|
||||
batch_size=num_seqs,
|
||||
max_query_len=max(q_lens),
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
max_kv_len=max(kv_lens),
|
||||
)
|
||||
out = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
@@ -264,6 +292,7 @@ def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
num_splits=num_splits,
|
||||
out=preallocated_out,
|
||||
)
|
||||
assert out.data_ptr() == preallocated_out.data_ptr()
|
||||
@@ -285,6 +314,24 @@ def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0
|
||||
torch.testing.assert_close(out.float(), ref.float(), atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_score_mod_relative_attention(monkeypatch):
|
||||
module = importlib.import_module("vllm.models.inkling.nvidia.ops.fa4_rel_attention")
|
||||
monkeypatch.setattr(module, "_use_sheared_bias", lambda: False)
|
||||
_run_case(
|
||||
[(64, 64), (1, 80)],
|
||||
num_heads=4,
|
||||
num_kv_heads=4,
|
||||
rel_extent=128,
|
||||
window_left=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
@@ -12,6 +16,47 @@ def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
|
||||
return 1 << max(0, max_seqlen_q - 1).bit_length()
|
||||
|
||||
|
||||
@cache
|
||||
def _use_sheared_bias() -> bool:
|
||||
capability = current_platform.get_device_capability()
|
||||
return capability is not None and capability.major in (10, 11)
|
||||
|
||||
|
||||
@cache
|
||||
def _get_score_mod(rel_extent: int) -> Callable:
|
||||
"""Return the score modification that adds Inkling relative bias."""
|
||||
import cutlass.cute as cute
|
||||
from cutlass.cute import Float32
|
||||
|
||||
from vllm.vllm_flash_attn.cute.seqlen_info import SeqlenInfoQK
|
||||
|
||||
@cute.jit
|
||||
def score_mod_rel_bias(
|
||||
scores: cute.TensorSSA,
|
||||
b_idx: cute.TensorSSA,
|
||||
h_idx: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info: SeqlenInfoQK,
|
||||
aux_tensors: list[cute.Tensor],
|
||||
) -> cute.TensorSSA:
|
||||
rel_logits = aux_tensors[0]
|
||||
|
||||
seqlen_local_offset = seqlen_info.seqlen_k - seqlen_info.seqlen_q
|
||||
rel_dist = (q_idx + seqlen_local_offset) - kv_idx
|
||||
global_q_idx = seqlen_info.offset_q + q_idx
|
||||
|
||||
rel_dist_0 = rel_dist[0]
|
||||
rel_idx = rel_dist_0 if rel_dist_0 >= 0 else 0
|
||||
rel_idx = rel_idx if rel_idx < rel_extent else (rel_extent - 1)
|
||||
|
||||
rel_bias = rel_logits[global_q_idx[0], h_idx[0], rel_idx]
|
||||
rel_bias = Float32(rel_bias) if rel_dist_0 == rel_idx else Float32(0.0)
|
||||
return scores + rel_bias
|
||||
|
||||
return score_mod_rel_bias
|
||||
|
||||
|
||||
def inkling_fa4_num_splits(
|
||||
*,
|
||||
is_local: bool,
|
||||
@@ -21,7 +66,7 @@ def inkling_fa4_num_splits(
|
||||
num_kv_heads: int,
|
||||
max_kv_len: int,
|
||||
) -> int:
|
||||
"""Return the split-KV cap for FA4 with sheared relative bias."""
|
||||
"""Return the split-KV cap for Inkling relative attention."""
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is not None and capability.major == 9:
|
||||
return 1
|
||||
@@ -78,13 +123,25 @@ def inkling_fa4_rel_attention(
|
||||
per-request KV lengths (``seqused_k``). ``rel_logits`` is
|
||||
``(num_tokens, num_heads, rel_extent)``.
|
||||
|
||||
The bias uses tml-fa4's sheared relative-bias layout.
|
||||
Hopper uses standard FA4's score-mod gather. Blackwell uses tml-fa4's
|
||||
sheared relative-bias layout.
|
||||
"""
|
||||
from vllm.third_party.tml_fa4 import flash_attn_varlen_func
|
||||
|
||||
# cute uses (None, None) to mean "no window".
|
||||
cute_window = (None, None) if window_size == (-1, -1) else window_size
|
||||
|
||||
rel_logits = rel_logits.contiguous()
|
||||
if _use_sheared_bias():
|
||||
from vllm.third_party.tml_fa4 import flash_attn_varlen_func
|
||||
|
||||
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
|
||||
else:
|
||||
from vllm.vllm_flash_attn.cute import flash_attn_varlen_func
|
||||
|
||||
bias_kwargs = {
|
||||
"score_mod": _get_score_mod(rel_extent),
|
||||
"aux_tensors": [rel_logits],
|
||||
}
|
||||
|
||||
ret = flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=key_cache,
|
||||
@@ -99,7 +156,7 @@ def inkling_fa4_rel_attention(
|
||||
num_splits=num_splits,
|
||||
return_lse=False,
|
||||
out=out,
|
||||
rel_bias=rel_logits.contiguous(),
|
||||
**bias_kwargs,
|
||||
)
|
||||
if isinstance(ret, tuple):
|
||||
return ret[0]
|
||||
|
||||
Reference in New Issue
Block a user