[Kernel][Model] Optimize FA4 mm_prefix range lookup (#50294)

Signed-off-by: Juqi Li <2223621784@qq.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Li Juqi
2026-08-05 07:33:49 -07:00
committed by GitHub
co-authored by Matthew Bonanni Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 62c5e21621
commit 66b3c0e61f
5 changed files with 819 additions and 45 deletions
+636
View File
@@ -0,0 +1,636 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""mm_prefix query-range metadata and the FA4 mask_mod that consumes it.
The metadata tests run anywhere. The FA4 tests need SM100 and run the real
kernel against a dense float32 reference, which is the only way to check what
cannot be reasoned about from the Python side:
* ``q_ranges[token_idx, 0]`` indexing an aux tensor with a runtime ``Int32``.
* ``cu_seqlens_q[b] + q_local`` matching FA4's own packing of a varlen batch.
The mask_mod is the *entire* mask (``(causal ∧ window) ∨ mm_prefix``), so
callers must pass ``causal=False`` and no FA-layer window. FA #155 stopped
auto-clearing those when ``mask_mod`` is set; leaving ``causal=True`` would
short out the mask_mod on SM90 and clip bidirectional ranges on SM100.
"""
import numpy as np
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import fill_mm_prefix_query_ranges
def _fa4_available() -> bool:
if not current_platform.is_cuda():
return False
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
return is_fa_version_supported(4)
requires_fa4 = pytest.mark.skipif(
not _fa4_available(), reason="FA4 mm_prefix mask_mod requires FA4"
)
# Imported conditionally because these pull in CuTe / FA4 build artifacts that
# are absent wherever requires_fa4 skips.
if _fa4_available():
from types import MethodType
from tests.v1.attention.test_attention_backends import (
MockAttentionLayer,
create_and_prepopulate_kv_cache,
)
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm.config import set_current_vllm_config
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionImpl,
FlashAttentionMetadataBuilder,
_make_mm_prefix_mask_mod,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, get_kv_quant_mode
from vllm.vllm_flash_attn.flash_attn_interface import flash_attn_varlen_func
DEVICE = torch.device("cuda:0")
DTYPE = torch.bfloat16
HEAD_SIZE = 128
NUM_HEADS = 8
NUM_KV_HEADS = 8
SLIDING_WINDOW_LEFT = 129
# --------------------------------------------------------------------------- #
# Metadata construction
# --------------------------------------------------------------------------- #
# Stands in for the builder's persistent (max_num_batched_tokens, 2) buffer.
STAGING_CAPACITY = 64
def _query_ranges(mm_ranges, query_lens, seq_lens):
"""Fill a staging buffer and return the written rows, or None if empty."""
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
)
# Poison the buffer so a stale row surviving the fill is visible.
out = np.full((STAGING_CAPACITY, 2), 12345, dtype=np.int32)
num_tokens = fill_mm_prefix_query_ranges(
out,
mm_ranges,
query_start_loc,
torch.tensor(seq_lens, dtype=torch.int32),
)
if num_tokens == 0:
return None
return torch.from_numpy(out[:num_tokens])
def test_matches_range_scan_semantics_with_context_offset():
"""Pin the equivalence the O(1) lookup rests on.
The kernel checks ``r_start <= kv_idx <= r_end`` for the range containing
the query token, which is only equal to the old scan's
``any(q in r and kv in r)`` because ranges never overlap. Request 1 carries
a context offset so the local-to-absolute query mapping is exercised too.
"""
mm_ranges = {0: [(1, 3), (5, 7)], 1: [(2, 4), (9, 12)]}
query_lens = [8, 6]
seq_lens = [8, 13]
query_ranges = _query_ranges(mm_ranges, query_lens, seq_lens)
assert query_ranges is not None
token_start = 0
for req_idx, query_len in enumerate(query_lens):
context_len = seq_lens[req_idx] - query_len
for q_local in range(query_len):
q_abs = context_len + q_local
r_start, r_end = query_ranges[token_start + q_local].tolist()
for kv_idx in range(seq_lens[req_idx]):
old_scan_keep = any(
start < end and start <= q_abs <= end and start <= kv_idx <= end
for start, end in mm_ranges[req_idx]
)
new_keep = r_start <= kv_idx <= r_end
assert new_keep == old_scan_keep, (req_idx, q_abs, kv_idx)
token_start += query_len
def test_ranges_beyond_scheduled_chunk_are_clipped():
"""Chunked prefill must not error or produce a wrong mask.
``disable_chunked_mm_input`` keeps a single mm item intact but still splits
a prompt across steps, so a request's ranges routinely sit entirely past the
tokens scheduled so far. Sizing by query token makes the out-of-chunk part
a no-op rather than an out-of-bounds condition.
"""
# Step 1 of "TTTT IIIIII": only the 4 text tokens are scheduled, so the
# range is entirely ahead of the chunk. The old bounds check raised here.
assert _query_ranges({0: [(4, 9)]}, query_lens=[4], seq_lens=[4]) is None
# A range starting mid-chunk and running past its end keeps its absolute
# bounds, so the bidirectional block still reaches the range's later keys
# once they are cached. Query tokens here are absolute positions 2..5.
query_ranges = _query_ranges({0: [(4, 9)]}, query_lens=[4], seq_lens=[6])
assert query_ranges is not None
expected = torch.tensor([[-1, -1], [-1, -1], [4, 9], [4, 9]], dtype=torch.int32)
torch.testing.assert_close(query_ranges, expected)
def test_buffer_reuse_does_not_leak_previous_rows():
"""The staging buffer persists across steps, so every reported row must be
rewritten. Guards against dropping the ``-1`` fill as an optimization: a
step whose ranges shrink would otherwise expose the prior step's bounds and
silently widen the bidirectional mask.
"""
out = np.zeros((STAGING_CAPACITY, 2), dtype=np.int32)
query_start_loc = torch.tensor([0, 8], dtype=torch.int32)
seq_lens = torch.tensor([8], dtype=torch.int32)
num_tokens = fill_mm_prefix_query_ranges(
out, {0: [(0, 7)]}, query_start_loc, seq_lens
)
assert num_tokens == 8
np.testing.assert_array_equal(out[:8], np.tile([0, 7], (8, 1)))
# Same rows scheduled, but now only tokens 2..3 sit inside a range.
num_tokens = fill_mm_prefix_query_ranges(
out, {0: [(2, 3)]}, query_start_loc, seq_lens
)
assert num_tokens == 8
expected = np.full((8, 2), -1, dtype=np.int32)
expected[2:4] = (2, 3)
np.testing.assert_array_equal(out[:8], expected)
def test_returns_none_when_no_range_covers_a_query_token():
"""No aux tensor means forward() skips the mask_mod entirely.
Returning None rather than an all-``-1`` tensor is what keeps text-only and
decode-only batches off the fill path.
"""
# Text-only batch on an mm_prefix model: every request present, no ranges.
assert _query_ranges({0: [], 1: []}, query_lens=[2, 2], seq_lens=[2, 2]) is None
# Degenerate single-token ranges are skipped, matching the Triton path's
# `start < end` validity check.
assert _query_ranges({0: [(1, 1)]}, query_lens=[4], seq_lens=[4]) is None
# Decode rows: the query token is generated, so it is in no range.
assert _query_ranges({0: [(1, 3)]}, query_lens=[1], seq_lens=[9]) is None
# --------------------------------------------------------------------------- #
# FA4 kernel
# --------------------------------------------------------------------------- #
def _cu_seqlens(lens: list[int]) -> torch.Tensor:
out = torch.zeros(len(lens) + 1, dtype=torch.int32)
out[1:] = torch.tensor(lens, dtype=torch.int32).cumsum(0)
return out
def _dense_reference(
q,
k_per_req,
v_per_req,
query_lens,
seq_lens,
mm_ranges,
sliding_window_left,
scale,
mm_clamp_sw=0,
):
"""Dense float32 ``(causal AND window) OR mm_prefix`` per request.
``mm_clamp_sw`` confines the bidirectional block to the window, reproducing
Gemma4's ``mm_prefix_clamp_sliding_window``. Only the past bound applies, so
future keys inside a range still pass.
"""
out = torch.empty_like(q, dtype=torch.float32)
repeats = q.shape[1] // k_per_req[0].shape[1]
q_off = 0
for req_idx, (q_len, k_len) in enumerate(zip(query_lens, seq_lens)):
ctx = k_len - q_len
q_pos = torch.arange(q_len, device=q.device) + ctx
k_pos = torch.arange(k_len, device=q.device)
delta = q_pos[:, None] - k_pos[None, :]
keep = delta >= 0
if sliding_window_left is not None:
keep &= delta < sliding_window_left
for start, end in mm_ranges.get(req_idx, []):
if start >= end:
continue
q_in = (q_pos >= start) & (q_pos <= end)
k_in = (k_pos >= start) & (k_pos <= end)
mm = q_in[:, None] & k_in[None, :]
if mm_clamp_sw > 0:
mm &= delta < mm_clamp_sw
keep |= mm
q_i = q[q_off : q_off + q_len].float().transpose(0, 1)
k_i = k_per_req[req_idx].float().transpose(0, 1)
v_i = v_per_req[req_idx].float().transpose(0, 1)
if repeats > 1:
k_i = k_i.repeat_interleave(repeats, dim=0)
v_i = v_i.repeat_interleave(repeats, dim=0)
scores = (q_i @ k_i.transpose(-1, -2)) * scale
scores = scores.masked_fill(~keep[None], float("-inf"))
out[q_off : q_off + q_len] = (scores.softmax(-1) @ v_i).transpose(0, 1)
q_off += q_len
return out
def _split(packed: torch.Tensor, lens: list[int]) -> list[torch.Tensor]:
return list(packed.split(lens))
def _reference(q, k, v, query_lens, seq_lens, mm_ranges, sw_left, scale, **kwargs):
return _dense_reference(
q,
_split(k, seq_lens),
_split(v, seq_lens),
query_lens,
seq_lens,
mm_ranges,
sw_left,
scale,
**kwargs,
)
def _run_kernel(
q, k, v, query_lens, seq_lens, mm_ranges, sliding_window_left, scale, mm_clamp_sw=0
):
cu_q = _cu_seqlens(query_lens)
cu_k = _cu_seqlens(seq_lens)
staging = torch.full((int(cu_q[-1]), 2), 12345, dtype=torch.int32).numpy()
num_rows = fill_mm_prefix_query_ranges(
staging, mm_ranges, cu_q, torch.tensor(seq_lens, dtype=torch.int32)
)
assert num_rows > 0, "test case must have at least one in-range query token"
q_ranges = torch.from_numpy(staging[:num_rows]).to(DEVICE)
cu_q_gpu = cu_q.to(DEVICE)
out = torch.empty_like(q)
flash_attn_varlen_func(
q=q,
k=k,
v=v,
out=out,
cu_seqlens_q=cu_q_gpu,
cu_seqlens_k=cu_k.to(DEVICE),
max_seqlen_q=max(query_lens),
max_seqlen_k=max(seq_lens),
softmax_scale=scale,
causal=False,
fa_version=4,
mask_mod=_make_mm_prefix_mask_mod(
sliding_window=mm_clamp_sw, sliding_window_left=sliding_window_left
),
aux_tensors=[q_ranges, cu_q_gpu],
)
return out
def _randn_qkv(query_lens, seq_lens):
torch.manual_seed(0)
q = torch.randn(sum(query_lens), NUM_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
k = torch.randn(sum(seq_lens), NUM_KV_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
v = torch.randn(sum(seq_lens), NUM_KV_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
return q, k, v, HEAD_SIZE**-0.5
# (name, query_lens, seq_lens, mm_ranges). One shape per index computation the
# mask_mod performs; shapes that differ only in request count are omitted.
CASES = [
# cu_seqlens_q[b] + q_local: unequal query lens across the batch, the
# packing the pooling workload never produced.
(
"varlen_batch",
[128, 64, 200],
[128, 64, 200],
{0: [(8, 71)], 1: [(0, 31), (40, 55)], 2: [(64, 191)]},
),
# q_abs = q_idx + (seqlen_k - seqlen_q): ranges are absolute prompt
# positions, so part of each range sits in the context.
(
"context_offset",
[64],
[320],
{0: [(32, 95), (200, 287)]},
),
# Decode rows: the query token is generated, so it is in no range and must
# get the (-1, -1) sentinel. One prefill row keeps the batch mixed.
(
"mixed_prefill_decode",
[160, 1, 1, 1],
[160, 512, 300, 97],
{0: [(32, 127)], 1: [(4, 259)], 2: [(8, 71)], 3: [(1, 64)]},
),
]
# A 320-token range, wider than the window. Every CASES range is narrower than
# the window and therefore clamp-insensitive, so the window modes are exercised
# on this shape instead: images larger than the window are exactly why
# mm_prefix_clamp_sliding_window exists.
WIDE_RANGE_CASE = ([384], [384], {0: [(32, 351)]})
# (sliding_window_left, mm_clamp_sw). sliding_window_left selects between the
# two mask_mod variants; mm_clamp_sw gates the Gemma4 clamp inside the sliding
# one. Gemma4 sets mm_clamp_sw == sw_val, and the unclamped variant ignores
# mm_clamp_sw, so a clamp without a window is not a reachable configuration.
WINDOW_MODES = [
(None, 0),
(SLIDING_WINDOW_LEFT, 0),
(SLIDING_WINDOW_LEFT, SLIDING_WINDOW_LEFT),
]
WINDOW_IDS = ["full_causal", "window", "window_clamped"]
@requires_fa4
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
def test_mm_prefix_mask_mod_matches_dense_reference(case):
_, query_lens, seq_lens, mm_ranges = case
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
args = (q, k, v, query_lens, seq_lens, mm_ranges, SLIDING_WINDOW_LEFT, scale)
torch.testing.assert_close(
_run_kernel(*args).float(), _reference(*args), atol=2e-2, rtol=2e-2
)
@requires_fa4
@pytest.mark.parametrize("window_mode", WINDOW_MODES, ids=WINDOW_IDS)
def test_window_modes_on_a_range_wider_than_the_window(window_mode):
"""Cover the three reachable window configurations.
The clamped mode additionally asserts the clamp changes the mask at all,
without which a range narrower than the window would leave the clamp branch
unexercised while the test still passed.
"""
sliding_window_left, mm_clamp_sw = window_mode
query_lens, seq_lens, mm_ranges = WIDE_RANGE_CASE
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
args = (q, k, v, query_lens, seq_lens, mm_ranges, sliding_window_left, scale)
expected = _reference(*args, mm_clamp_sw=mm_clamp_sw)
if mm_clamp_sw:
assert not torch.allclose(
expected, _reference(*args, mm_clamp_sw=0), atol=2e-2, rtol=2e-2
), "range is not wide enough to exercise the clamp branch"
actual = _run_kernel(*args, mm_clamp_sw=mm_clamp_sw)
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
@requires_fa4
def test_ranges_outside_chunk_degrade_to_causal():
"""A range fully behind the scheduled chunk must not widen the mask.
Under chunked prefill a request's ranges routinely sit entirely in the
already-computed context. Indexing by query token makes those a no-op; the
result must be identical to running with no ranges at all.
"""
query_lens, seq_lens = [64], [512]
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
# Range (16, 79) is entirely below the chunk start (448), plus one live
# range so the fill still reports rows.
args = (q, k, v, query_lens, seq_lens)
actual = _run_kernel(*args, {0: [(16, 79), (448, 511)]}, None, scale)
expected = _reference(*args, {0: [(448, 511)]}, None, scale)
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
# --------------------------------------------------------------------------- #
# FA4 through the metadata builder and the paged KV cache
# --------------------------------------------------------------------------- #
# Stock google/gemma-4-E2B-it ships text_config.use_bidirectional_attention as
# null, so is_mm_prefix_lm is False out of the box. Tests that exercise the
# builder must flip it via `_enable_mm_prefix`. Only config.json / tokenizer
# are read; weights are never loaded.
MODEL = "google/gemma-4-E2B-it"
BLOCK_SIZE = 16
SLIDING_WINDOW = 256
# Request 0 is a prefill chunk whose range starts mid-context; requests 1-3 are
# decodes whose generated token sits outside every range.
PAGED_SEQ_LENS = [352, 513, 200, 97]
PAGED_QUERY_LENS = [96, 1, 1, 1]
PAGED_MM_RANGES = {
0: [(224, 287), (300, 351)],
1: [(4, 259)],
2: [(0, 63), (80, 143)],
3: [(1, 64)],
}
def _enable_mm_prefix(vllm_config):
"""Flip stock Gemma4's null use_bidirectional_attention to "vision".
``ModelConfig.__init__`` derives ``model_arch_config`` once from
``hf_text_config``, and ``is_mm_prefix_lm`` is a ``cached_property`` that
may already hold False from that first derivation, so re-derive and drop
the cache.
"""
model_config = vllm_config.model_config
model_config.hf_text_config.use_bidirectional_attention = "vision"
model_config.model_arch_config = model_config.get_model_arch_config()
model_config.__dict__.pop("is_mm_prefix_lm", None)
return vllm_config
@requires_fa4
def test_decode_only_batch_reports_no_ranges():
"""Decode-only steps must carry no mm_prefix metadata at all.
This is what makes FULL CUDA graph capture consistent for this feature.
Capture runs through ``_dummy_run`` with no multimodal requests, so no
mask_mod is attached to the captured graph; a decode-only replay must
therefore also want no mask_mod, or the captured graph would silently drop
it. Indexing by query token gives that for free, because a generated token
is always past every prompt range.
The range-id scheme keyed off the request having ranges at all, so it
attached a mask_mod to decode steps and did not have this property.
"""
vllm_config = _enable_mm_prefix(
create_vllm_config(
model_name=MODEL,
max_model_len=1024,
block_size=BLOCK_SIZE,
num_gpu_blocks=256,
)
)
decode_batch = BatchSpec(
seq_lens=[513, 200, 97], query_lens=[1, 1, 1], name="decode_only"
)
common = create_common_attn_metadata(decode_batch, BLOCK_SIZE, DEVICE)
common.mm_req_doc_ranges = {0: [(4, 259)], 1: [(0, 63)], 2: [(1, 64)]}
builder = FlashAttentionMetadataBuilder(
create_standard_kv_cache_spec(vllm_config),
["model.layers.0.self_attn.attn"],
vllm_config,
DEVICE,
)
md = builder.build(common_prefix_len=0, common_attn_metadata=common)
assert md.mm_prefix_query_range_tensor is None
@requires_fa4
@pytest.mark.parametrize(
"head_size",
[
# 128: FA4's default SM100 path (not the dedicated hd256 kernel), so
# paged KV + seqused_k works on B200 and H100. Non-paged mm_prefix
# cases in this file already use HEAD_SIZE=128.
pytest.param(128, id="hd128"),
# 512: Gemma4 global head dim; FA4 resolves on H100, skips on
# Blackwell (TMEM).
pytest.param(512, id="global_512"),
],
)
def test_mm_prefix_kv_cache_path(head_size: int):
"""FA4 + mask_mod + block_table matches a dense reference.
Use head_size=128 on B200/H100 (FA4 + paged/`seqused_k` supported) and
512 on H100 only (Blackwell FA4 rejects 512 via TMEM). Avoid 256: SM100's
dedicated hd256 2CTA kernel asserts away ``seqused_k``, which paged KV
requires.
Tensors, kv_cache_spec, and FlashAttentionImpl must all use the same
``head_size`` — ``run_attention_backend`` / ``create_standard_kv_cache_spec``
read ``get_head_size()`` (=512 for Gemma4), so this test builds them
explicitly. The sliding window is set on the impl; mm_prefix encodes it in
mask_mod and clears FA's built-in window at forward time.
"""
torch.manual_seed(0)
vllm_config = _enable_mm_prefix(
create_vllm_config(
model_name=MODEL,
max_model_len=max(PAGED_SEQ_LENS),
block_size=BLOCK_SIZE,
num_gpu_blocks=2048,
)
)
assert vllm_config.model_config.is_mm_prefix_lm
mc = vllm_config.model_config
# Keep every get_head_size() reader (builder helpers, etc.) on the same dim.
mc.get_head_size = MethodType(lambda self: head_size, mc)
# Skip under the same config Gemma4 forces (flash_attn_version=4), so the
# check matches FlashAttentionImpl's get_flash_attn_version call.
with set_current_vllm_config(vllm_config):
if get_flash_attn_version(head_size=head_size) != 4:
pytest.skip(f"FA4 does not support head_size={head_size} on this device")
num_heads = mc.get_num_attention_heads(vllm_config.parallel_config)
num_kv_heads = mc.get_num_kv_heads(vllm_config.parallel_config)
scale = head_size**-0.5
batch = BatchSpec(
seq_lens=PAGED_SEQ_LENS, query_lens=PAGED_QUERY_LENS, name="mm_prefix_mixed"
)
kv_cache_spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=mc.dtype,
sliding_window=SLIDING_WINDOW,
kv_quant_mode=get_kv_quant_mode(vllm_config.cache_config.cache_dtype),
)
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
common.mm_req_doc_ranges = PAGED_MM_RANGES
qs, new_ks, new_vs, k_fulls, v_fulls, k_ctxs, v_ctxs = [], [], [], [], [], [], []
for q_len, s_len in zip(PAGED_QUERY_LENS, PAGED_SEQ_LENS):
ctx = s_len - q_len
shape = (num_kv_heads, head_size)
qs.append(torch.randn(q_len, num_heads, head_size, dtype=DTYPE, device=DEVICE))
k_full = torch.randn(s_len, *shape, dtype=DTYPE, device=DEVICE)
v_full = torch.randn(s_len, *shape, dtype=DTYPE, device=DEVICE)
k_fulls.append(k_full)
v_fulls.append(v_full)
k_ctxs.append(k_full[:ctx])
v_ctxs.append(v_full[:ctx])
new_ks.append(k_full[ctx:])
new_vs.append(v_full[ctx:])
query = torch.cat(qs)
key = torch.cat(new_ks)
value = torch.cat(new_vs)
kv_cache = create_and_prepopulate_kv_cache(
k_contexts=k_ctxs,
v_contexts=v_ctxs,
block_size=BLOCK_SIZE,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=DTYPE,
device=DEVICE,
num_blocks=2048,
common_attn_metadata=common,
randomize_blocks=True,
)
# Packed KV layout is [..., 2 * head_size]; impl.split(head_size) needs match.
assert kv_cache.shape[-1] == 2 * head_size
layer_names = ["model.layers.0.self_attn.attn"]
with set_current_vllm_config(vllm_config):
builder = FlashAttentionMetadataBuilder(
kv_cache_spec, layer_names, vllm_config, DEVICE
)
attn_metadata = builder.build(common_prefix_len=0, common_attn_metadata=common)
# head_size=128 keeps FA4 under local attention on Blackwell (the
# FA4→FA2 demotion is only for local+256). Window is still encoded in
# mask_mod at forward; FA's built-in window is cleared there.
impl = FlashAttentionImpl(
num_heads=num_heads,
head_size=head_size,
scale=scale,
num_kv_heads=num_kv_heads,
alibi_slopes=None,
sliding_window=SLIDING_WINDOW,
kv_cache_dtype="auto",
)
assert impl.vllm_flash_attn_version == 4, (
f"expected FA4, got FA{impl.vllm_flash_attn_version} for "
f"head_size={head_size}"
)
assert impl.head_size == head_size == kv_cache_spec.head_size
mock_layer = MockAttentionLayer(DEVICE)
output = torch.empty_like(query)
impl.do_kv_cache_update(
mock_layer, key, value, kv_cache, attn_metadata.slot_mapping
)
actual = impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
ref_args = (query, k_fulls, v_fulls, PAGED_QUERY_LENS, PAGED_SEQ_LENS)
expected = _dense_reference(*ref_args, PAGED_MM_RANGES, SLIDING_WINDOW, scale)
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
# Without this the assertion above would also pass if the mask_mod were
# silently dropped, since the batch would then be plain causal + window.
causal_only = _dense_reference(*ref_args, {}, SLIDING_WINDOW, scale)
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
"test batch does not actually exercise the mm_prefix branch"
)
+2
View File
@@ -2078,6 +2078,8 @@ class Gemma4ForConditionalGeneration(
metadata.mm_prefix_range = None
if hasattr(metadata, "mm_prefix_range_tensor"):
metadata.mm_prefix_range_tensor = None
if hasattr(metadata, "mm_prefix_query_range_tensor"):
metadata.mm_prefix_query_range_tensor = None
if isinstance(attn_metadata, list):
for ub_metadata in attn_metadata:
+1 -1
View File
@@ -471,7 +471,7 @@ class CommonAttentionMetadata:
"""PrefixLM bidirectional ranges for multimodal tokens. Maps
request index to list of (start, end) token position ranges
where bidirectional attention should apply. None for text-only
batches or non-PrefixLM models."""
batches or non-PrefixLM models. A request's ranges must not overlap."""
rswa_prefix_lens: torch.Tensor | None = None
"""(batch_size,) per-request prefix length (prompt/image token count) for
+109 -44
View File
@@ -3,6 +3,7 @@
"""Attention layer with FlashAttention."""
import copy
import functools
from dataclasses import dataclass
from typing import ClassVar
@@ -12,6 +13,7 @@ import torch
from vllm.model_executor.layers.attention import Attention
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
PIN_MEMORY,
canonicalize_singleton_dim_strides,
is_quantized_kv_cache,
)
@@ -28,7 +30,10 @@ from vllm.v1.attention.backends.fa_utils import (
is_fa_version_supported,
is_flash_attn_varlen_func_available,
)
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.attention.backends.utils import (
fill_mm_prefix_query_ranges,
get_dcp_local_seq_lens,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
@@ -284,9 +289,10 @@ class FlashAttentionMetadata:
sliding_window: tuple[int, int] | None = None
# PrefixLM bidirectional ranges for multimodal tokens.
# Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range.
mm_prefix_range_tensor: torch.Tensor | None = None
# PrefixLM bidirectional range containing each scheduled query token.
# Shape: (num_actual_tokens, 2) int32, absolute [start, end] bounds;
# (-1, -1) for query tokens outside every multimodal range.
mm_prefix_query_range_tensor: torch.Tensor | None = None
# Reference Sliding Window Attention (R-SWA) fields.
# rswa_prefix_lens: per-request prompt lengths [num_reqs], int32, CUDA.
@@ -455,6 +461,21 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
[self.rswa_window], dtype=torch.int32, device=self.device
)
# mm_prefix: persistent staging + device buffers owned by this builder,
# sized by scheduled query tokens so build() never allocates.
self.mm_prefix_query_ranges_cpu: torch.Tensor | None = None
self.mm_prefix_query_ranges_np: np.ndarray | None = None
self.mm_prefix_query_ranges_gpu: torch.Tensor | None = None
if self.model_config.is_mm_prefix_lm:
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.mm_prefix_query_ranges_cpu = torch.empty(
(max_num_tokens, 2), dtype=torch.int32, pin_memory=PIN_MEMORY
)
self.mm_prefix_query_ranges_np = self.mm_prefix_query_ranges_cpu.numpy()
self.mm_prefix_query_ranges_gpu = torch.empty(
(max_num_tokens, 2), dtype=torch.int32, device=self.device
)
def build(
self,
common_prefix_len: int,
@@ -696,16 +717,31 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
)
# Compute mm_prefix range tensor if the batch contains
# multimodal tokens with bidirectional ranges.
# multimodal tokens with bidirectional ranges. Built for every FA
# group; Gemma4 nulls the field for its non-sliding layers.
mm_ranges = common_attn_metadata.mm_req_doc_ranges
if mm_ranges is not None:
from vllm.v1.attention.backends.utils import (
compute_mm_prefix_range_tensor,
if mm_ranges is not None and self.mm_prefix_query_ranges_np is not None:
# The upper bound is exact for prefill rows, which is where
# mm_prefix ranges live; decode rows only ever get an optimistic
# (larger) context, moving them further past every range.
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None, (
"mm_prefix requires seq_lens_cpu_upper_bound"
)
attn_metadata.mm_prefix_range_tensor = compute_mm_prefix_range_tensor(
mm_ranges, num_reqs, seq_lens.device
num_mm_tokens = fill_mm_prefix_query_ranges(
self.mm_prefix_query_ranges_np,
mm_ranges,
common_attn_metadata.query_start_loc_cpu,
common_attn_metadata.seq_lens_cpu_upper_bound,
)
if num_mm_tokens > 0:
assert self.mm_prefix_query_ranges_cpu is not None
assert self.mm_prefix_query_ranges_gpu is not None
mm_query_ranges = self.mm_prefix_query_ranges_gpu[:num_mm_tokens]
mm_query_ranges.copy_(
self.mm_prefix_query_ranges_cpu[:num_mm_tokens],
non_blocking=True,
)
attn_metadata.mm_prefix_query_range_tensor = mm_query_ranges
# R-SWA: copy prefix lengths into persistent buffers (outside the
# compiled region) so forward() never allocates during CUDA graph
@@ -971,23 +1007,25 @@ class FlashAttentionImpl(AttentionImpl):
causal = attn_metadata.causal
is_dynamic_causal = isinstance(causal, torch.Tensor)
mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor
mm_prefix_query_ranges = attn_metadata.mm_prefix_query_range_tensor
mm_mask_mod = None
mm_aux = None
if (
mm_prefix_ranges is not None
mm_prefix_query_ranges is not None
and not is_dynamic_causal
and causal is True
and self.vllm_flash_attn_version == 4
):
max_ranges = mm_prefix_ranges.shape[1]
# Sliding window value in Triton convention
# (1 + window_size[0]). Global-attention layers
# store (-1, -1) → sw stays None / 0.
# Use the layer impl's window, not attn_metadata's. The
# metadata field comes from kv_cache_spec (model-wide, e.g.
# Gemma4's 512), while the impl holds the per-layer window
# the mask_mod must encode (e.g. a test override).
# Triton convention: 1 + window_size[0]. Global layers store
# (-1, -1) → sw stays None.
layer_window = self.sliding_window
sw_val = (
1 + sliding_window_size[0]
if sliding_window_size is not None
and sliding_window_size[0] >= 0
1 + layer_window[0]
if layer_window is not None and layer_window[0] >= 0
else None
)
# Gemma4: also clamp the bidirectional block to the
@@ -1000,11 +1038,16 @@ class FlashAttentionImpl(AttentionImpl):
):
mm_clamp_sw = sw_val
mm_mask_mod = _make_mm_prefix_mask_mod(
max_ranges,
sliding_window=mm_clamp_sw,
sliding_window_left=sw_val,
)
mm_aux = [mm_prefix_ranges]
mm_aux = [mm_prefix_query_ranges, attn_metadata.query_start_loc]
# mm_prefix is (causal ∧ window) ∨ bidirectional-range —
# not ⊆ causal. FA #155 stopped auto-clearing causal/local
# when mask_mod is set, so the caller must disable them or
# the built-in causal path shorts out / clips the mask_mod.
causal = False
sliding_window_size = None
# R-SWA: use CuTE-DSL mask_mod on FA4 for exact token-level
# mask without block-size approximation. The mask_mod encodes
@@ -1378,14 +1421,19 @@ class FlashAttentionImpl(AttentionImpl):
return output
@functools.cache
def _make_mm_prefix_mask_mod(
max_ranges: int,
sliding_window: int = 0,
sliding_window_left: int | None = None,
):
"""Build a CuTE-DSL mask_mod implementing
``(causal AND sliding_window) OR mm_prefix``.
Cached so identical ``(sliding_window, sliding_window_left)`` reuse the
same function object. FA4's ``hash_callable`` mixes ``repr()`` of closure
cells into the compile key; the nested ``_load_q_range`` would otherwise
get a new address each call and force a full JIT recompile every forward.
The FA4 kernel passes *local* ``q_idx`` (0-based within the current
prefill chunk) while ``kv_idx`` is absolute (0-based over the full
KV cache). We recover the absolute Q position via
@@ -1394,6 +1442,15 @@ def _make_mm_prefix_mask_mod(
use consistent absolute positions. This matches the Triton
reference path (``compute_kv_seq_mask``).
``aux_tensors[0]`` holds the absolute ``[start, end]`` bounds of the
mm_prefix range containing each scheduled query token (``(-1, -1)`` when
none), and ``aux_tensors[1]`` is ``cu_seqlens_q``, used to turn the local
``q_idx`` into a packed row index. Because mm_prefix ranges never overlap,
``r_start <= kv_idx <= r_end`` is exactly "query and key share a range", so
no key-side lookup is needed and ``kv_idx`` is never used as an index.
The ``(-1, -1)`` sentinel falls out for free: ``kv_idx <= -1`` is false for
every valid key.
``sliding_window_left`` enforces the sliding window on the causal
term (None = full causal, no window). ``sliding_window`` clamps the
bidirectional block to the window (0 = unclamped; >0 = Gemma4 local
@@ -1405,8 +1462,29 @@ def _make_mm_prefix_mask_mod(
from vllm.vllm_flash_attn.cute.utils import ( # type: ignore[import-untyped]
scalar_to_ssa,
ssa_to_scalar,
)
@cute.jit
def _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx):
"""Load the mm_prefix range bounds for this query row.
Both loads depend only on the query index, so they hoist out of the
unrolled per-element mask loop. ``ssa_to_scalar`` reads lane 0, so one
call must not span query rows (hence the ``__vec_size__`` pin below).
Clamping keeps ``token_idx`` in bounds for partial tiles and padded
(``seqlen_q == 0``) rows; neither produces output.
"""
q_ranges = aux_tensors[0]
cu_seqlens_q = aux_tensors[1]
b = batch_idx[0]
q_local = cutlass.min(ssa_to_scalar(q_idx), seqlen_info.seqlen_q - Int32(1))
token_idx = cutlass.max(cu_seqlens_q[b] + q_local, Int32(0))
return (
scalar_to_ssa(q_ranges[token_idx, 0], Int32),
scalar_to_ssa(q_ranges[token_idx, 1], Int32),
)
if sliding_window_left is not None:
@cute.jit
@@ -1422,18 +1500,11 @@ def _make_mm_prefix_mask_mod(
q_abs = q_idx + ctx_off
sw = scalar_to_ssa(Int32(sliding_window_left), Int32)
keep = (kv_idx <= q_abs) & ((q_abs - kv_idx) < sw)
ranges = aux_tensors[0]
b = batch_idx[0]
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
valid = r_start < r_end
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
mm = q_in & k_in
if sliding_window > 0:
mm = mm & ((q_abs - kv_idx) < sw)
keep = keep | mm
r_start, r_end = _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx)
mm = (kv_idx >= r_start) & (kv_idx <= r_end)
if sliding_window > 0:
mm = mm & ((q_abs - kv_idx) < sw)
keep = keep | mm
return keep
else:
@@ -1450,18 +1521,12 @@ def _make_mm_prefix_mask_mod(
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
q_abs = q_idx + ctx_off
keep = kv_idx <= q_abs
ranges = aux_tensors[0]
b = batch_idx[0]
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
valid = r_start < r_end
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
keep = keep | (q_in & k_in)
r_start, r_end = _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx)
keep = keep | ((kv_idx >= r_start) & (kv_idx <= r_end))
return keep
mm_prefix_mask_mod.use_fast_sampling = True
mm_prefix_mask_mod.__vec_size__ = 1 # _load_q_range takes lane 0 of q_idx
return mm_prefix_mask_mod
+71
View File
@@ -75,6 +75,77 @@ def compute_mm_prefix_range_tensor(
return padded.view(num_seqs, max_ranges, 2)
def fill_mm_prefix_query_ranges(
out: np.ndarray,
mm_prefix_range: dict[int, list[tuple[int, int]]] | None,
query_start_loc_cpu: torch.Tensor,
seq_lens_cpu: torch.Tensor,
) -> int:
"""Map each scheduled query token to the mm_prefix range containing it.
Writes into ``out``, a caller-owned ``(max_num_batched_tokens, 2)`` int32
staging buffer, and returns the number of rows written (0 if no range
covers any scheduled query token, in which case ``out`` is untouched and
the caller should skip the mask_mod entirely). Row ``i`` holds the absolute
``[start, end]`` bounds of the bidirectional range that query token ``i``
belongs to, or ``(-1, -1)`` when it is outside every range.
mm_prefix ranges never overlap, so "query and key share a range" is
equivalent to "the key lies inside the query's own range". The kernel
therefore needs no key-side lookup, and this metadata is sized by scheduled
query tokens rather than by context length -- bounded by
``max_num_batched_tokens`` instead of ``num_seqs * max_seq_len``.
Ranges are absolute prompt positions and may extend past the tokens
scheduled so far under chunked prefill; the portion outside the current
chunk is simply not recorded. Degenerate ranges (``start >= end``) are
skipped to match the Triton path's ``start < end`` validity check.
``seq_lens_cpu`` only needs to be exact for prefill rows, since mm_prefix
ranges cover prompt tokens: an over-estimate on a decode row shifts that
row's query position further past every range, which still matches nothing.
"""
if mm_prefix_range is None:
return 0
query_start_loc = query_start_loc_cpu.numpy()
num_actual_tokens = int(query_start_loc[-1])
if num_actual_tokens <= 0:
return 0
assert num_actual_tokens <= out.shape[0], (
f"mm_prefix staging buffer holds {out.shape[0]} tokens, got {num_actual_tokens}"
)
# Resolve every span before touching `out`, so batches whose ranges all
# fall outside the scheduled tokens skip the fill entirely.
spans: list[tuple[int, int, int, int]] = []
for req_idx, req_ranges in mm_prefix_range.items():
if not req_ranges:
continue
token_start = int(query_start_loc[req_idx])
query_len = int(query_start_loc[req_idx + 1]) - token_start
if query_len <= 0:
continue
# Absolute position of this request's first scheduled query token.
context_len = int(seq_lens_cpu[req_idx]) - query_len
for start, end in req_ranges:
if start >= end:
continue
first = max(start - context_len, 0)
last = min(end - context_len, query_len - 1)
if first > last:
continue
spans.append((token_start + first, token_start + last + 1, start, end))
if not spans:
return 0
out[:num_actual_tokens] = -1
for row_start, row_end, start, end in spans:
out[row_start:row_end] = (start, end)
return num_actual_tokens
def is_valid_kv_cache_layout(value: str) -> bool:
return value in get_args(KVCacheLayoutType)