[Kernel][Model] Optimize FA4 mm_prefix range lookup (#50294)
Signed-off-by: Juqi Li <2223621784@qq.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62c5e21621
commit
66b3c0e61f
@@ -0,0 +1,636 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""mm_prefix query-range metadata and the FA4 mask_mod that consumes it.
|
||||
|
||||
The metadata tests run anywhere. The FA4 tests need SM100 and run the real
|
||||
kernel against a dense float32 reference, which is the only way to check what
|
||||
cannot be reasoned about from the Python side:
|
||||
|
||||
* ``q_ranges[token_idx, 0]`` indexing an aux tensor with a runtime ``Int32``.
|
||||
* ``cu_seqlens_q[b] + q_local`` matching FA4's own packing of a varlen batch.
|
||||
|
||||
The mask_mod is the *entire* mask (``(causal ∧ window) ∨ mm_prefix``), so
|
||||
callers must pass ``causal=False`` and no FA-layer window. FA #155 stopped
|
||||
auto-clearing those when ``mask_mod`` is set; leaving ``causal=True`` would
|
||||
short out the mask_mod on SM90 and clip bidirectional ranges on SM100.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import fill_mm_prefix_query_ranges
|
||||
|
||||
|
||||
def _fa4_available() -> bool:
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
|
||||
|
||||
return is_fa_version_supported(4)
|
||||
|
||||
|
||||
requires_fa4 = pytest.mark.skipif(
|
||||
not _fa4_available(), reason="FA4 mm_prefix mask_mod requires FA4"
|
||||
)
|
||||
|
||||
# Imported conditionally because these pull in CuTe / FA4 build artifacts that
|
||||
# are absent wherever requires_fa4 skips.
|
||||
if _fa4_available():
|
||||
from types import MethodType
|
||||
|
||||
from tests.v1.attention.test_attention_backends import (
|
||||
MockAttentionLayer,
|
||||
create_and_prepopulate_kv_cache,
|
||||
)
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionImpl,
|
||||
FlashAttentionMetadataBuilder,
|
||||
_make_mm_prefix_mask_mod,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, get_kv_quant_mode
|
||||
from vllm.vllm_flash_attn.flash_attn_interface import flash_attn_varlen_func
|
||||
|
||||
DEVICE = torch.device("cuda:0")
|
||||
DTYPE = torch.bfloat16
|
||||
HEAD_SIZE = 128
|
||||
NUM_HEADS = 8
|
||||
NUM_KV_HEADS = 8
|
||||
SLIDING_WINDOW_LEFT = 129
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Metadata construction
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# Stands in for the builder's persistent (max_num_batched_tokens, 2) buffer.
|
||||
STAGING_CAPACITY = 64
|
||||
|
||||
|
||||
def _query_ranges(mm_ranges, query_lens, seq_lens):
|
||||
"""Fill a staging buffer and return the written rows, or None if empty."""
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
)
|
||||
# Poison the buffer so a stale row surviving the fill is visible.
|
||||
out = np.full((STAGING_CAPACITY, 2), 12345, dtype=np.int32)
|
||||
num_tokens = fill_mm_prefix_query_ranges(
|
||||
out,
|
||||
mm_ranges,
|
||||
query_start_loc,
|
||||
torch.tensor(seq_lens, dtype=torch.int32),
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return None
|
||||
return torch.from_numpy(out[:num_tokens])
|
||||
|
||||
|
||||
def test_matches_range_scan_semantics_with_context_offset():
|
||||
"""Pin the equivalence the O(1) lookup rests on.
|
||||
|
||||
The kernel checks ``r_start <= kv_idx <= r_end`` for the range containing
|
||||
the query token, which is only equal to the old scan's
|
||||
``any(q in r and kv in r)`` because ranges never overlap. Request 1 carries
|
||||
a context offset so the local-to-absolute query mapping is exercised too.
|
||||
"""
|
||||
mm_ranges = {0: [(1, 3), (5, 7)], 1: [(2, 4), (9, 12)]}
|
||||
query_lens = [8, 6]
|
||||
seq_lens = [8, 13]
|
||||
|
||||
query_ranges = _query_ranges(mm_ranges, query_lens, seq_lens)
|
||||
assert query_ranges is not None
|
||||
|
||||
token_start = 0
|
||||
for req_idx, query_len in enumerate(query_lens):
|
||||
context_len = seq_lens[req_idx] - query_len
|
||||
for q_local in range(query_len):
|
||||
q_abs = context_len + q_local
|
||||
r_start, r_end = query_ranges[token_start + q_local].tolist()
|
||||
for kv_idx in range(seq_lens[req_idx]):
|
||||
old_scan_keep = any(
|
||||
start < end and start <= q_abs <= end and start <= kv_idx <= end
|
||||
for start, end in mm_ranges[req_idx]
|
||||
)
|
||||
new_keep = r_start <= kv_idx <= r_end
|
||||
assert new_keep == old_scan_keep, (req_idx, q_abs, kv_idx)
|
||||
token_start += query_len
|
||||
|
||||
|
||||
def test_ranges_beyond_scheduled_chunk_are_clipped():
|
||||
"""Chunked prefill must not error or produce a wrong mask.
|
||||
|
||||
``disable_chunked_mm_input`` keeps a single mm item intact but still splits
|
||||
a prompt across steps, so a request's ranges routinely sit entirely past the
|
||||
tokens scheduled so far. Sizing by query token makes the out-of-chunk part
|
||||
a no-op rather than an out-of-bounds condition.
|
||||
"""
|
||||
# Step 1 of "TTTT IIIIII": only the 4 text tokens are scheduled, so the
|
||||
# range is entirely ahead of the chunk. The old bounds check raised here.
|
||||
assert _query_ranges({0: [(4, 9)]}, query_lens=[4], seq_lens=[4]) is None
|
||||
|
||||
# A range starting mid-chunk and running past its end keeps its absolute
|
||||
# bounds, so the bidirectional block still reaches the range's later keys
|
||||
# once they are cached. Query tokens here are absolute positions 2..5.
|
||||
query_ranges = _query_ranges({0: [(4, 9)]}, query_lens=[4], seq_lens=[6])
|
||||
assert query_ranges is not None
|
||||
expected = torch.tensor([[-1, -1], [-1, -1], [4, 9], [4, 9]], dtype=torch.int32)
|
||||
torch.testing.assert_close(query_ranges, expected)
|
||||
|
||||
|
||||
def test_buffer_reuse_does_not_leak_previous_rows():
|
||||
"""The staging buffer persists across steps, so every reported row must be
|
||||
rewritten. Guards against dropping the ``-1`` fill as an optimization: a
|
||||
step whose ranges shrink would otherwise expose the prior step's bounds and
|
||||
silently widen the bidirectional mask.
|
||||
"""
|
||||
out = np.zeros((STAGING_CAPACITY, 2), dtype=np.int32)
|
||||
query_start_loc = torch.tensor([0, 8], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([8], dtype=torch.int32)
|
||||
|
||||
num_tokens = fill_mm_prefix_query_ranges(
|
||||
out, {0: [(0, 7)]}, query_start_loc, seq_lens
|
||||
)
|
||||
assert num_tokens == 8
|
||||
np.testing.assert_array_equal(out[:8], np.tile([0, 7], (8, 1)))
|
||||
|
||||
# Same rows scheduled, but now only tokens 2..3 sit inside a range.
|
||||
num_tokens = fill_mm_prefix_query_ranges(
|
||||
out, {0: [(2, 3)]}, query_start_loc, seq_lens
|
||||
)
|
||||
assert num_tokens == 8
|
||||
expected = np.full((8, 2), -1, dtype=np.int32)
|
||||
expected[2:4] = (2, 3)
|
||||
np.testing.assert_array_equal(out[:8], expected)
|
||||
|
||||
|
||||
def test_returns_none_when_no_range_covers_a_query_token():
|
||||
"""No aux tensor means forward() skips the mask_mod entirely.
|
||||
|
||||
Returning None rather than an all-``-1`` tensor is what keeps text-only and
|
||||
decode-only batches off the fill path.
|
||||
"""
|
||||
# Text-only batch on an mm_prefix model: every request present, no ranges.
|
||||
assert _query_ranges({0: [], 1: []}, query_lens=[2, 2], seq_lens=[2, 2]) is None
|
||||
# Degenerate single-token ranges are skipped, matching the Triton path's
|
||||
# `start < end` validity check.
|
||||
assert _query_ranges({0: [(1, 1)]}, query_lens=[4], seq_lens=[4]) is None
|
||||
# Decode rows: the query token is generated, so it is in no range.
|
||||
assert _query_ranges({0: [(1, 3)]}, query_lens=[1], seq_lens=[9]) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# FA4 kernel
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _cu_seqlens(lens: list[int]) -> torch.Tensor:
|
||||
out = torch.zeros(len(lens) + 1, dtype=torch.int32)
|
||||
out[1:] = torch.tensor(lens, dtype=torch.int32).cumsum(0)
|
||||
return out
|
||||
|
||||
|
||||
def _dense_reference(
|
||||
q,
|
||||
k_per_req,
|
||||
v_per_req,
|
||||
query_lens,
|
||||
seq_lens,
|
||||
mm_ranges,
|
||||
sliding_window_left,
|
||||
scale,
|
||||
mm_clamp_sw=0,
|
||||
):
|
||||
"""Dense float32 ``(causal AND window) OR mm_prefix`` per request.
|
||||
|
||||
``mm_clamp_sw`` confines the bidirectional block to the window, reproducing
|
||||
Gemma4's ``mm_prefix_clamp_sliding_window``. Only the past bound applies, so
|
||||
future keys inside a range still pass.
|
||||
"""
|
||||
out = torch.empty_like(q, dtype=torch.float32)
|
||||
repeats = q.shape[1] // k_per_req[0].shape[1]
|
||||
q_off = 0
|
||||
for req_idx, (q_len, k_len) in enumerate(zip(query_lens, seq_lens)):
|
||||
ctx = k_len - q_len
|
||||
q_pos = torch.arange(q_len, device=q.device) + ctx
|
||||
k_pos = torch.arange(k_len, device=q.device)
|
||||
delta = q_pos[:, None] - k_pos[None, :]
|
||||
|
||||
keep = delta >= 0
|
||||
if sliding_window_left is not None:
|
||||
keep &= delta < sliding_window_left
|
||||
|
||||
for start, end in mm_ranges.get(req_idx, []):
|
||||
if start >= end:
|
||||
continue
|
||||
q_in = (q_pos >= start) & (q_pos <= end)
|
||||
k_in = (k_pos >= start) & (k_pos <= end)
|
||||
mm = q_in[:, None] & k_in[None, :]
|
||||
if mm_clamp_sw > 0:
|
||||
mm &= delta < mm_clamp_sw
|
||||
keep |= mm
|
||||
|
||||
q_i = q[q_off : q_off + q_len].float().transpose(0, 1)
|
||||
k_i = k_per_req[req_idx].float().transpose(0, 1)
|
||||
v_i = v_per_req[req_idx].float().transpose(0, 1)
|
||||
if repeats > 1:
|
||||
k_i = k_i.repeat_interleave(repeats, dim=0)
|
||||
v_i = v_i.repeat_interleave(repeats, dim=0)
|
||||
|
||||
scores = (q_i @ k_i.transpose(-1, -2)) * scale
|
||||
scores = scores.masked_fill(~keep[None], float("-inf"))
|
||||
out[q_off : q_off + q_len] = (scores.softmax(-1) @ v_i).transpose(0, 1)
|
||||
q_off += q_len
|
||||
return out
|
||||
|
||||
|
||||
def _split(packed: torch.Tensor, lens: list[int]) -> list[torch.Tensor]:
|
||||
return list(packed.split(lens))
|
||||
|
||||
|
||||
def _reference(q, k, v, query_lens, seq_lens, mm_ranges, sw_left, scale, **kwargs):
|
||||
return _dense_reference(
|
||||
q,
|
||||
_split(k, seq_lens),
|
||||
_split(v, seq_lens),
|
||||
query_lens,
|
||||
seq_lens,
|
||||
mm_ranges,
|
||||
sw_left,
|
||||
scale,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _run_kernel(
|
||||
q, k, v, query_lens, seq_lens, mm_ranges, sliding_window_left, scale, mm_clamp_sw=0
|
||||
):
|
||||
cu_q = _cu_seqlens(query_lens)
|
||||
cu_k = _cu_seqlens(seq_lens)
|
||||
|
||||
staging = torch.full((int(cu_q[-1]), 2), 12345, dtype=torch.int32).numpy()
|
||||
num_rows = fill_mm_prefix_query_ranges(
|
||||
staging, mm_ranges, cu_q, torch.tensor(seq_lens, dtype=torch.int32)
|
||||
)
|
||||
assert num_rows > 0, "test case must have at least one in-range query token"
|
||||
q_ranges = torch.from_numpy(staging[:num_rows]).to(DEVICE)
|
||||
cu_q_gpu = cu_q.to(DEVICE)
|
||||
|
||||
out = torch.empty_like(q)
|
||||
flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
out=out,
|
||||
cu_seqlens_q=cu_q_gpu,
|
||||
cu_seqlens_k=cu_k.to(DEVICE),
|
||||
max_seqlen_q=max(query_lens),
|
||||
max_seqlen_k=max(seq_lens),
|
||||
softmax_scale=scale,
|
||||
causal=False,
|
||||
fa_version=4,
|
||||
mask_mod=_make_mm_prefix_mask_mod(
|
||||
sliding_window=mm_clamp_sw, sliding_window_left=sliding_window_left
|
||||
),
|
||||
aux_tensors=[q_ranges, cu_q_gpu],
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _randn_qkv(query_lens, seq_lens):
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(sum(query_lens), NUM_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
|
||||
k = torch.randn(sum(seq_lens), NUM_KV_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
|
||||
v = torch.randn(sum(seq_lens), NUM_KV_HEADS, HEAD_SIZE, dtype=DTYPE, device=DEVICE)
|
||||
return q, k, v, HEAD_SIZE**-0.5
|
||||
|
||||
|
||||
# (name, query_lens, seq_lens, mm_ranges). One shape per index computation the
|
||||
# mask_mod performs; shapes that differ only in request count are omitted.
|
||||
CASES = [
|
||||
# cu_seqlens_q[b] + q_local: unequal query lens across the batch, the
|
||||
# packing the pooling workload never produced.
|
||||
(
|
||||
"varlen_batch",
|
||||
[128, 64, 200],
|
||||
[128, 64, 200],
|
||||
{0: [(8, 71)], 1: [(0, 31), (40, 55)], 2: [(64, 191)]},
|
||||
),
|
||||
# q_abs = q_idx + (seqlen_k - seqlen_q): ranges are absolute prompt
|
||||
# positions, so part of each range sits in the context.
|
||||
(
|
||||
"context_offset",
|
||||
[64],
|
||||
[320],
|
||||
{0: [(32, 95), (200, 287)]},
|
||||
),
|
||||
# Decode rows: the query token is generated, so it is in no range and must
|
||||
# get the (-1, -1) sentinel. One prefill row keeps the batch mixed.
|
||||
(
|
||||
"mixed_prefill_decode",
|
||||
[160, 1, 1, 1],
|
||||
[160, 512, 300, 97],
|
||||
{0: [(32, 127)], 1: [(4, 259)], 2: [(8, 71)], 3: [(1, 64)]},
|
||||
),
|
||||
]
|
||||
|
||||
# A 320-token range, wider than the window. Every CASES range is narrower than
|
||||
# the window and therefore clamp-insensitive, so the window modes are exercised
|
||||
# on this shape instead: images larger than the window are exactly why
|
||||
# mm_prefix_clamp_sliding_window exists.
|
||||
WIDE_RANGE_CASE = ([384], [384], {0: [(32, 351)]})
|
||||
|
||||
# (sliding_window_left, mm_clamp_sw). sliding_window_left selects between the
|
||||
# two mask_mod variants; mm_clamp_sw gates the Gemma4 clamp inside the sliding
|
||||
# one. Gemma4 sets mm_clamp_sw == sw_val, and the unclamped variant ignores
|
||||
# mm_clamp_sw, so a clamp without a window is not a reachable configuration.
|
||||
WINDOW_MODES = [
|
||||
(None, 0),
|
||||
(SLIDING_WINDOW_LEFT, 0),
|
||||
(SLIDING_WINDOW_LEFT, SLIDING_WINDOW_LEFT),
|
||||
]
|
||||
WINDOW_IDS = ["full_causal", "window", "window_clamped"]
|
||||
|
||||
|
||||
@requires_fa4
|
||||
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
|
||||
def test_mm_prefix_mask_mod_matches_dense_reference(case):
|
||||
_, query_lens, seq_lens, mm_ranges = case
|
||||
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
|
||||
args = (q, k, v, query_lens, seq_lens, mm_ranges, SLIDING_WINDOW_LEFT, scale)
|
||||
torch.testing.assert_close(
|
||||
_run_kernel(*args).float(), _reference(*args), atol=2e-2, rtol=2e-2
|
||||
)
|
||||
|
||||
|
||||
@requires_fa4
|
||||
@pytest.mark.parametrize("window_mode", WINDOW_MODES, ids=WINDOW_IDS)
|
||||
def test_window_modes_on_a_range_wider_than_the_window(window_mode):
|
||||
"""Cover the three reachable window configurations.
|
||||
|
||||
The clamped mode additionally asserts the clamp changes the mask at all,
|
||||
without which a range narrower than the window would leave the clamp branch
|
||||
unexercised while the test still passed.
|
||||
"""
|
||||
sliding_window_left, mm_clamp_sw = window_mode
|
||||
query_lens, seq_lens, mm_ranges = WIDE_RANGE_CASE
|
||||
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
|
||||
args = (q, k, v, query_lens, seq_lens, mm_ranges, sliding_window_left, scale)
|
||||
|
||||
expected = _reference(*args, mm_clamp_sw=mm_clamp_sw)
|
||||
if mm_clamp_sw:
|
||||
assert not torch.allclose(
|
||||
expected, _reference(*args, mm_clamp_sw=0), atol=2e-2, rtol=2e-2
|
||||
), "range is not wide enough to exercise the clamp branch"
|
||||
|
||||
actual = _run_kernel(*args, mm_clamp_sw=mm_clamp_sw)
|
||||
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_fa4
|
||||
def test_ranges_outside_chunk_degrade_to_causal():
|
||||
"""A range fully behind the scheduled chunk must not widen the mask.
|
||||
|
||||
Under chunked prefill a request's ranges routinely sit entirely in the
|
||||
already-computed context. Indexing by query token makes those a no-op; the
|
||||
result must be identical to running with no ranges at all.
|
||||
"""
|
||||
query_lens, seq_lens = [64], [512]
|
||||
q, k, v, scale = _randn_qkv(query_lens, seq_lens)
|
||||
|
||||
# Range (16, 79) is entirely below the chunk start (448), plus one live
|
||||
# range so the fill still reports rows.
|
||||
args = (q, k, v, query_lens, seq_lens)
|
||||
actual = _run_kernel(*args, {0: [(16, 79), (448, 511)]}, None, scale)
|
||||
expected = _reference(*args, {0: [(448, 511)]}, None, scale)
|
||||
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# FA4 through the metadata builder and the paged KV cache
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# Stock google/gemma-4-E2B-it ships text_config.use_bidirectional_attention as
|
||||
# null, so is_mm_prefix_lm is False out of the box. Tests that exercise the
|
||||
# builder must flip it via `_enable_mm_prefix`. Only config.json / tokenizer
|
||||
# are read; weights are never loaded.
|
||||
MODEL = "google/gemma-4-E2B-it"
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
SLIDING_WINDOW = 256
|
||||
|
||||
# Request 0 is a prefill chunk whose range starts mid-context; requests 1-3 are
|
||||
# decodes whose generated token sits outside every range.
|
||||
PAGED_SEQ_LENS = [352, 513, 200, 97]
|
||||
PAGED_QUERY_LENS = [96, 1, 1, 1]
|
||||
PAGED_MM_RANGES = {
|
||||
0: [(224, 287), (300, 351)],
|
||||
1: [(4, 259)],
|
||||
2: [(0, 63), (80, 143)],
|
||||
3: [(1, 64)],
|
||||
}
|
||||
|
||||
|
||||
def _enable_mm_prefix(vllm_config):
|
||||
"""Flip stock Gemma4's null use_bidirectional_attention to "vision".
|
||||
|
||||
``ModelConfig.__init__`` derives ``model_arch_config`` once from
|
||||
``hf_text_config``, and ``is_mm_prefix_lm`` is a ``cached_property`` that
|
||||
may already hold False from that first derivation, so re-derive and drop
|
||||
the cache.
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config.use_bidirectional_attention = "vision"
|
||||
model_config.model_arch_config = model_config.get_model_arch_config()
|
||||
model_config.__dict__.pop("is_mm_prefix_lm", None)
|
||||
return vllm_config
|
||||
|
||||
|
||||
@requires_fa4
|
||||
def test_decode_only_batch_reports_no_ranges():
|
||||
"""Decode-only steps must carry no mm_prefix metadata at all.
|
||||
|
||||
This is what makes FULL CUDA graph capture consistent for this feature.
|
||||
Capture runs through ``_dummy_run`` with no multimodal requests, so no
|
||||
mask_mod is attached to the captured graph; a decode-only replay must
|
||||
therefore also want no mask_mod, or the captured graph would silently drop
|
||||
it. Indexing by query token gives that for free, because a generated token
|
||||
is always past every prompt range.
|
||||
|
||||
The range-id scheme keyed off the request having ranges at all, so it
|
||||
attached a mask_mod to decode steps and did not have this property.
|
||||
"""
|
||||
vllm_config = _enable_mm_prefix(
|
||||
create_vllm_config(
|
||||
model_name=MODEL,
|
||||
max_model_len=1024,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_gpu_blocks=256,
|
||||
)
|
||||
)
|
||||
decode_batch = BatchSpec(
|
||||
seq_lens=[513, 200, 97], query_lens=[1, 1, 1], name="decode_only"
|
||||
)
|
||||
common = create_common_attn_metadata(decode_batch, BLOCK_SIZE, DEVICE)
|
||||
common.mm_req_doc_ranges = {0: [(4, 259)], 1: [(0, 63)], 2: [(1, 64)]}
|
||||
|
||||
builder = FlashAttentionMetadataBuilder(
|
||||
create_standard_kv_cache_spec(vllm_config),
|
||||
["model.layers.0.self_attn.attn"],
|
||||
vllm_config,
|
||||
DEVICE,
|
||||
)
|
||||
md = builder.build(common_prefix_len=0, common_attn_metadata=common)
|
||||
assert md.mm_prefix_query_range_tensor is None
|
||||
|
||||
|
||||
@requires_fa4
|
||||
@pytest.mark.parametrize(
|
||||
"head_size",
|
||||
[
|
||||
# 128: FA4's default SM100 path (not the dedicated hd256 kernel), so
|
||||
# paged KV + seqused_k works on B200 and H100. Non-paged mm_prefix
|
||||
# cases in this file already use HEAD_SIZE=128.
|
||||
pytest.param(128, id="hd128"),
|
||||
# 512: Gemma4 global head dim; FA4 resolves on H100, skips on
|
||||
# Blackwell (TMEM).
|
||||
pytest.param(512, id="global_512"),
|
||||
],
|
||||
)
|
||||
def test_mm_prefix_kv_cache_path(head_size: int):
|
||||
"""FA4 + mask_mod + block_table matches a dense reference.
|
||||
|
||||
Use head_size=128 on B200/H100 (FA4 + paged/`seqused_k` supported) and
|
||||
512 on H100 only (Blackwell FA4 rejects 512 via TMEM). Avoid 256: SM100's
|
||||
dedicated hd256 2CTA kernel asserts away ``seqused_k``, which paged KV
|
||||
requires.
|
||||
|
||||
Tensors, kv_cache_spec, and FlashAttentionImpl must all use the same
|
||||
``head_size`` — ``run_attention_backend`` / ``create_standard_kv_cache_spec``
|
||||
read ``get_head_size()`` (=512 for Gemma4), so this test builds them
|
||||
explicitly. The sliding window is set on the impl; mm_prefix encodes it in
|
||||
mask_mod and clears FA's built-in window at forward time.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
vllm_config = _enable_mm_prefix(
|
||||
create_vllm_config(
|
||||
model_name=MODEL,
|
||||
max_model_len=max(PAGED_SEQ_LENS),
|
||||
block_size=BLOCK_SIZE,
|
||||
num_gpu_blocks=2048,
|
||||
)
|
||||
)
|
||||
assert vllm_config.model_config.is_mm_prefix_lm
|
||||
|
||||
mc = vllm_config.model_config
|
||||
# Keep every get_head_size() reader (builder helpers, etc.) on the same dim.
|
||||
mc.get_head_size = MethodType(lambda self: head_size, mc)
|
||||
|
||||
# Skip under the same config Gemma4 forces (flash_attn_version=4), so the
|
||||
# check matches FlashAttentionImpl's get_flash_attn_version call.
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if get_flash_attn_version(head_size=head_size) != 4:
|
||||
pytest.skip(f"FA4 does not support head_size={head_size} on this device")
|
||||
|
||||
num_heads = mc.get_num_attention_heads(vllm_config.parallel_config)
|
||||
num_kv_heads = mc.get_num_kv_heads(vllm_config.parallel_config)
|
||||
scale = head_size**-0.5
|
||||
|
||||
batch = BatchSpec(
|
||||
seq_lens=PAGED_SEQ_LENS, query_lens=PAGED_QUERY_LENS, name="mm_prefix_mixed"
|
||||
)
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=mc.dtype,
|
||||
sliding_window=SLIDING_WINDOW,
|
||||
kv_quant_mode=get_kv_quant_mode(vllm_config.cache_config.cache_dtype),
|
||||
)
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
|
||||
common.mm_req_doc_ranges = PAGED_MM_RANGES
|
||||
|
||||
qs, new_ks, new_vs, k_fulls, v_fulls, k_ctxs, v_ctxs = [], [], [], [], [], [], []
|
||||
for q_len, s_len in zip(PAGED_QUERY_LENS, PAGED_SEQ_LENS):
|
||||
ctx = s_len - q_len
|
||||
shape = (num_kv_heads, head_size)
|
||||
qs.append(torch.randn(q_len, num_heads, head_size, dtype=DTYPE, device=DEVICE))
|
||||
k_full = torch.randn(s_len, *shape, dtype=DTYPE, device=DEVICE)
|
||||
v_full = torch.randn(s_len, *shape, dtype=DTYPE, device=DEVICE)
|
||||
k_fulls.append(k_full)
|
||||
v_fulls.append(v_full)
|
||||
k_ctxs.append(k_full[:ctx])
|
||||
v_ctxs.append(v_full[:ctx])
|
||||
new_ks.append(k_full[ctx:])
|
||||
new_vs.append(v_full[ctx:])
|
||||
|
||||
query = torch.cat(qs)
|
||||
key = torch.cat(new_ks)
|
||||
value = torch.cat(new_vs)
|
||||
kv_cache = create_and_prepopulate_kv_cache(
|
||||
k_contexts=k_ctxs,
|
||||
v_contexts=v_ctxs,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=DTYPE,
|
||||
device=DEVICE,
|
||||
num_blocks=2048,
|
||||
common_attn_metadata=common,
|
||||
randomize_blocks=True,
|
||||
)
|
||||
|
||||
# Packed KV layout is [..., 2 * head_size]; impl.split(head_size) needs match.
|
||||
assert kv_cache.shape[-1] == 2 * head_size
|
||||
|
||||
layer_names = ["model.layers.0.self_attn.attn"]
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = FlashAttentionMetadataBuilder(
|
||||
kv_cache_spec, layer_names, vllm_config, DEVICE
|
||||
)
|
||||
attn_metadata = builder.build(common_prefix_len=0, common_attn_metadata=common)
|
||||
# head_size=128 keeps FA4 under local attention on Blackwell (the
|
||||
# FA4→FA2 demotion is only for local+256). Window is still encoded in
|
||||
# mask_mod at forward; FA's built-in window is cleared there.
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
scale=scale,
|
||||
num_kv_heads=num_kv_heads,
|
||||
alibi_slopes=None,
|
||||
sliding_window=SLIDING_WINDOW,
|
||||
kv_cache_dtype="auto",
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == 4, (
|
||||
f"expected FA4, got FA{impl.vllm_flash_attn_version} for "
|
||||
f"head_size={head_size}"
|
||||
)
|
||||
assert impl.head_size == head_size == kv_cache_spec.head_size
|
||||
|
||||
mock_layer = MockAttentionLayer(DEVICE)
|
||||
output = torch.empty_like(query)
|
||||
impl.do_kv_cache_update(
|
||||
mock_layer, key, value, kv_cache, attn_metadata.slot_mapping
|
||||
)
|
||||
actual = impl.forward(
|
||||
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
|
||||
)
|
||||
|
||||
ref_args = (query, k_fulls, v_fulls, PAGED_QUERY_LENS, PAGED_SEQ_LENS)
|
||||
expected = _dense_reference(*ref_args, PAGED_MM_RANGES, SLIDING_WINDOW, scale)
|
||||
torch.testing.assert_close(actual.float(), expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
# Without this the assertion above would also pass if the mask_mod were
|
||||
# silently dropped, since the batch would then be plain causal + window.
|
||||
causal_only = _dense_reference(*ref_args, {}, SLIDING_WINDOW, scale)
|
||||
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
|
||||
"test batch does not actually exercise the mm_prefix branch"
|
||||
)
|
||||
@@ -2078,6 +2078,8 @@ class Gemma4ForConditionalGeneration(
|
||||
metadata.mm_prefix_range = None
|
||||
if hasattr(metadata, "mm_prefix_range_tensor"):
|
||||
metadata.mm_prefix_range_tensor = None
|
||||
if hasattr(metadata, "mm_prefix_query_range_tensor"):
|
||||
metadata.mm_prefix_query_range_tensor = None
|
||||
|
||||
if isinstance(attn_metadata, list):
|
||||
for ub_metadata in attn_metadata:
|
||||
|
||||
@@ -471,7 +471,7 @@ class CommonAttentionMetadata:
|
||||
"""PrefixLM bidirectional ranges for multimodal tokens. Maps
|
||||
request index to list of (start, end) token position ranges
|
||||
where bidirectional attention should apply. None for text-only
|
||||
batches or non-PrefixLM models."""
|
||||
batches or non-PrefixLM models. A request's ranges must not overlap."""
|
||||
|
||||
rswa_prefix_lens: torch.Tensor | None = None
|
||||
"""(batch_size,) per-request prefix length (prompt/image token count) for
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""Attention layer with FlashAttention."""
|
||||
|
||||
import copy
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -12,6 +13,7 @@ import torch
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
canonicalize_singleton_dim_strides,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
@@ -28,7 +30,10 @@ from vllm.v1.attention.backends.fa_utils import (
|
||||
is_fa_version_supported,
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
fill_mm_prefix_query_ranges,
|
||||
get_dcp_local_seq_lens,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
@@ -284,9 +289,10 @@ class FlashAttentionMetadata:
|
||||
|
||||
sliding_window: tuple[int, int] | None = None
|
||||
|
||||
# PrefixLM bidirectional ranges for multimodal tokens.
|
||||
# Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range.
|
||||
mm_prefix_range_tensor: torch.Tensor | None = None
|
||||
# PrefixLM bidirectional range containing each scheduled query token.
|
||||
# Shape: (num_actual_tokens, 2) int32, absolute [start, end] bounds;
|
||||
# (-1, -1) for query tokens outside every multimodal range.
|
||||
mm_prefix_query_range_tensor: torch.Tensor | None = None
|
||||
|
||||
# Reference Sliding Window Attention (R-SWA) fields.
|
||||
# rswa_prefix_lens: per-request prompt lengths [num_reqs], int32, CUDA.
|
||||
@@ -455,6 +461,21 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
[self.rswa_window], dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
# mm_prefix: persistent staging + device buffers owned by this builder,
|
||||
# sized by scheduled query tokens so build() never allocates.
|
||||
self.mm_prefix_query_ranges_cpu: torch.Tensor | None = None
|
||||
self.mm_prefix_query_ranges_np: np.ndarray | None = None
|
||||
self.mm_prefix_query_ranges_gpu: torch.Tensor | None = None
|
||||
if self.model_config.is_mm_prefix_lm:
|
||||
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.mm_prefix_query_ranges_cpu = torch.empty(
|
||||
(max_num_tokens, 2), dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
self.mm_prefix_query_ranges_np = self.mm_prefix_query_ranges_cpu.numpy()
|
||||
self.mm_prefix_query_ranges_gpu = torch.empty(
|
||||
(max_num_tokens, 2), dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -696,16 +717,31 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
)
|
||||
|
||||
# Compute mm_prefix range tensor if the batch contains
|
||||
# multimodal tokens with bidirectional ranges.
|
||||
# multimodal tokens with bidirectional ranges. Built for every FA
|
||||
# group; Gemma4 nulls the field for its non-sliding layers.
|
||||
mm_ranges = common_attn_metadata.mm_req_doc_ranges
|
||||
if mm_ranges is not None:
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
compute_mm_prefix_range_tensor,
|
||||
if mm_ranges is not None and self.mm_prefix_query_ranges_np is not None:
|
||||
# The upper bound is exact for prefill rows, which is where
|
||||
# mm_prefix ranges live; decode rows only ever get an optimistic
|
||||
# (larger) context, moving them further past every range.
|
||||
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None, (
|
||||
"mm_prefix requires seq_lens_cpu_upper_bound"
|
||||
)
|
||||
|
||||
attn_metadata.mm_prefix_range_tensor = compute_mm_prefix_range_tensor(
|
||||
mm_ranges, num_reqs, seq_lens.device
|
||||
num_mm_tokens = fill_mm_prefix_query_ranges(
|
||||
self.mm_prefix_query_ranges_np,
|
||||
mm_ranges,
|
||||
common_attn_metadata.query_start_loc_cpu,
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound,
|
||||
)
|
||||
if num_mm_tokens > 0:
|
||||
assert self.mm_prefix_query_ranges_cpu is not None
|
||||
assert self.mm_prefix_query_ranges_gpu is not None
|
||||
mm_query_ranges = self.mm_prefix_query_ranges_gpu[:num_mm_tokens]
|
||||
mm_query_ranges.copy_(
|
||||
self.mm_prefix_query_ranges_cpu[:num_mm_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
attn_metadata.mm_prefix_query_range_tensor = mm_query_ranges
|
||||
|
||||
# R-SWA: copy prefix lengths into persistent buffers (outside the
|
||||
# compiled region) so forward() never allocates during CUDA graph
|
||||
@@ -971,23 +1007,25 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
causal = attn_metadata.causal
|
||||
is_dynamic_causal = isinstance(causal, torch.Tensor)
|
||||
|
||||
mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor
|
||||
mm_prefix_query_ranges = attn_metadata.mm_prefix_query_range_tensor
|
||||
mm_mask_mod = None
|
||||
mm_aux = None
|
||||
if (
|
||||
mm_prefix_ranges is not None
|
||||
mm_prefix_query_ranges is not None
|
||||
and not is_dynamic_causal
|
||||
and causal is True
|
||||
and self.vllm_flash_attn_version == 4
|
||||
):
|
||||
max_ranges = mm_prefix_ranges.shape[1]
|
||||
# Sliding window value in Triton convention
|
||||
# (1 + window_size[0]). Global-attention layers
|
||||
# store (-1, -1) → sw stays None / 0.
|
||||
# Use the layer impl's window, not attn_metadata's. The
|
||||
# metadata field comes from kv_cache_spec (model-wide, e.g.
|
||||
# Gemma4's 512), while the impl holds the per-layer window
|
||||
# the mask_mod must encode (e.g. a test override).
|
||||
# Triton convention: 1 + window_size[0]. Global layers store
|
||||
# (-1, -1) → sw stays None.
|
||||
layer_window = self.sliding_window
|
||||
sw_val = (
|
||||
1 + sliding_window_size[0]
|
||||
if sliding_window_size is not None
|
||||
and sliding_window_size[0] >= 0
|
||||
1 + layer_window[0]
|
||||
if layer_window is not None and layer_window[0] >= 0
|
||||
else None
|
||||
)
|
||||
# Gemma4: also clamp the bidirectional block to the
|
||||
@@ -1000,11 +1038,16 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
):
|
||||
mm_clamp_sw = sw_val
|
||||
mm_mask_mod = _make_mm_prefix_mask_mod(
|
||||
max_ranges,
|
||||
sliding_window=mm_clamp_sw,
|
||||
sliding_window_left=sw_val,
|
||||
)
|
||||
mm_aux = [mm_prefix_ranges]
|
||||
mm_aux = [mm_prefix_query_ranges, attn_metadata.query_start_loc]
|
||||
# mm_prefix is (causal ∧ window) ∨ bidirectional-range —
|
||||
# not ⊆ causal. FA #155 stopped auto-clearing causal/local
|
||||
# when mask_mod is set, so the caller must disable them or
|
||||
# the built-in causal path shorts out / clips the mask_mod.
|
||||
causal = False
|
||||
sliding_window_size = None
|
||||
|
||||
# R-SWA: use CuTE-DSL mask_mod on FA4 for exact token-level
|
||||
# mask without block-size approximation. The mask_mod encodes
|
||||
@@ -1378,14 +1421,19 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
return output
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _make_mm_prefix_mask_mod(
|
||||
max_ranges: int,
|
||||
sliding_window: int = 0,
|
||||
sliding_window_left: int | None = None,
|
||||
):
|
||||
"""Build a CuTE-DSL mask_mod implementing
|
||||
``(causal AND sliding_window) OR mm_prefix``.
|
||||
|
||||
Cached so identical ``(sliding_window, sliding_window_left)`` reuse the
|
||||
same function object. FA4's ``hash_callable`` mixes ``repr()`` of closure
|
||||
cells into the compile key; the nested ``_load_q_range`` would otherwise
|
||||
get a new address each call and force a full JIT recompile every forward.
|
||||
|
||||
The FA4 kernel passes *local* ``q_idx`` (0-based within the current
|
||||
prefill chunk) while ``kv_idx`` is absolute (0-based over the full
|
||||
KV cache). We recover the absolute Q position via
|
||||
@@ -1394,6 +1442,15 @@ def _make_mm_prefix_mask_mod(
|
||||
use consistent absolute positions. This matches the Triton
|
||||
reference path (``compute_kv_seq_mask``).
|
||||
|
||||
``aux_tensors[0]`` holds the absolute ``[start, end]`` bounds of the
|
||||
mm_prefix range containing each scheduled query token (``(-1, -1)`` when
|
||||
none), and ``aux_tensors[1]`` is ``cu_seqlens_q``, used to turn the local
|
||||
``q_idx`` into a packed row index. Because mm_prefix ranges never overlap,
|
||||
``r_start <= kv_idx <= r_end`` is exactly "query and key share a range", so
|
||||
no key-side lookup is needed and ``kv_idx`` is never used as an index.
|
||||
The ``(-1, -1)`` sentinel falls out for free: ``kv_idx <= -1`` is false for
|
||||
every valid key.
|
||||
|
||||
``sliding_window_left`` enforces the sliding window on the causal
|
||||
term (None = full causal, no window). ``sliding_window`` clamps the
|
||||
bidirectional block to the window (0 = unclamped; >0 = Gemma4 local
|
||||
@@ -1405,8 +1462,29 @@ def _make_mm_prefix_mask_mod(
|
||||
|
||||
from vllm.vllm_flash_attn.cute.utils import ( # type: ignore[import-untyped]
|
||||
scalar_to_ssa,
|
||||
ssa_to_scalar,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx):
|
||||
"""Load the mm_prefix range bounds for this query row.
|
||||
|
||||
Both loads depend only on the query index, so they hoist out of the
|
||||
unrolled per-element mask loop. ``ssa_to_scalar`` reads lane 0, so one
|
||||
call must not span query rows (hence the ``__vec_size__`` pin below).
|
||||
Clamping keeps ``token_idx`` in bounds for partial tiles and padded
|
||||
(``seqlen_q == 0``) rows; neither produces output.
|
||||
"""
|
||||
q_ranges = aux_tensors[0]
|
||||
cu_seqlens_q = aux_tensors[1]
|
||||
b = batch_idx[0]
|
||||
q_local = cutlass.min(ssa_to_scalar(q_idx), seqlen_info.seqlen_q - Int32(1))
|
||||
token_idx = cutlass.max(cu_seqlens_q[b] + q_local, Int32(0))
|
||||
return (
|
||||
scalar_to_ssa(q_ranges[token_idx, 0], Int32),
|
||||
scalar_to_ssa(q_ranges[token_idx, 1], Int32),
|
||||
)
|
||||
|
||||
if sliding_window_left is not None:
|
||||
|
||||
@cute.jit
|
||||
@@ -1422,18 +1500,11 @@ def _make_mm_prefix_mask_mod(
|
||||
q_abs = q_idx + ctx_off
|
||||
sw = scalar_to_ssa(Int32(sliding_window_left), Int32)
|
||||
keep = (kv_idx <= q_abs) & ((q_abs - kv_idx) < sw)
|
||||
ranges = aux_tensors[0]
|
||||
b = batch_idx[0]
|
||||
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
|
||||
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
|
||||
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
|
||||
valid = r_start < r_end
|
||||
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
mm = q_in & k_in
|
||||
if sliding_window > 0:
|
||||
mm = mm & ((q_abs - kv_idx) < sw)
|
||||
keep = keep | mm
|
||||
r_start, r_end = _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx)
|
||||
mm = (kv_idx >= r_start) & (kv_idx <= r_end)
|
||||
if sliding_window > 0:
|
||||
mm = mm & ((q_abs - kv_idx) < sw)
|
||||
keep = keep | mm
|
||||
return keep
|
||||
|
||||
else:
|
||||
@@ -1450,18 +1521,12 @@ def _make_mm_prefix_mask_mod(
|
||||
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
|
||||
q_abs = q_idx + ctx_off
|
||||
keep = kv_idx <= q_abs
|
||||
ranges = aux_tensors[0]
|
||||
b = batch_idx[0]
|
||||
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
|
||||
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
|
||||
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
|
||||
valid = r_start < r_end
|
||||
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
keep = keep | (q_in & k_in)
|
||||
r_start, r_end = _load_q_range(q_idx, seqlen_info, aux_tensors, batch_idx)
|
||||
keep = keep | ((kv_idx >= r_start) & (kv_idx <= r_end))
|
||||
return keep
|
||||
|
||||
mm_prefix_mask_mod.use_fast_sampling = True
|
||||
mm_prefix_mask_mod.__vec_size__ = 1 # _load_q_range takes lane 0 of q_idx
|
||||
return mm_prefix_mask_mod
|
||||
|
||||
|
||||
|
||||
@@ -75,6 +75,77 @@ def compute_mm_prefix_range_tensor(
|
||||
return padded.view(num_seqs, max_ranges, 2)
|
||||
|
||||
|
||||
def fill_mm_prefix_query_ranges(
|
||||
out: np.ndarray,
|
||||
mm_prefix_range: dict[int, list[tuple[int, int]]] | None,
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
) -> int:
|
||||
"""Map each scheduled query token to the mm_prefix range containing it.
|
||||
|
||||
Writes into ``out``, a caller-owned ``(max_num_batched_tokens, 2)`` int32
|
||||
staging buffer, and returns the number of rows written (0 if no range
|
||||
covers any scheduled query token, in which case ``out`` is untouched and
|
||||
the caller should skip the mask_mod entirely). Row ``i`` holds the absolute
|
||||
``[start, end]`` bounds of the bidirectional range that query token ``i``
|
||||
belongs to, or ``(-1, -1)`` when it is outside every range.
|
||||
|
||||
mm_prefix ranges never overlap, so "query and key share a range" is
|
||||
equivalent to "the key lies inside the query's own range". The kernel
|
||||
therefore needs no key-side lookup, and this metadata is sized by scheduled
|
||||
query tokens rather than by context length -- bounded by
|
||||
``max_num_batched_tokens`` instead of ``num_seqs * max_seq_len``.
|
||||
|
||||
Ranges are absolute prompt positions and may extend past the tokens
|
||||
scheduled so far under chunked prefill; the portion outside the current
|
||||
chunk is simply not recorded. Degenerate ranges (``start >= end``) are
|
||||
skipped to match the Triton path's ``start < end`` validity check.
|
||||
|
||||
``seq_lens_cpu`` only needs to be exact for prefill rows, since mm_prefix
|
||||
ranges cover prompt tokens: an over-estimate on a decode row shifts that
|
||||
row's query position further past every range, which still matches nothing.
|
||||
"""
|
||||
if mm_prefix_range is None:
|
||||
return 0
|
||||
|
||||
query_start_loc = query_start_loc_cpu.numpy()
|
||||
num_actual_tokens = int(query_start_loc[-1])
|
||||
if num_actual_tokens <= 0:
|
||||
return 0
|
||||
assert num_actual_tokens <= out.shape[0], (
|
||||
f"mm_prefix staging buffer holds {out.shape[0]} tokens, got {num_actual_tokens}"
|
||||
)
|
||||
|
||||
# Resolve every span before touching `out`, so batches whose ranges all
|
||||
# fall outside the scheduled tokens skip the fill entirely.
|
||||
spans: list[tuple[int, int, int, int]] = []
|
||||
for req_idx, req_ranges in mm_prefix_range.items():
|
||||
if not req_ranges:
|
||||
continue
|
||||
token_start = int(query_start_loc[req_idx])
|
||||
query_len = int(query_start_loc[req_idx + 1]) - token_start
|
||||
if query_len <= 0:
|
||||
continue
|
||||
# Absolute position of this request's first scheduled query token.
|
||||
context_len = int(seq_lens_cpu[req_idx]) - query_len
|
||||
for start, end in req_ranges:
|
||||
if start >= end:
|
||||
continue
|
||||
first = max(start - context_len, 0)
|
||||
last = min(end - context_len, query_len - 1)
|
||||
if first > last:
|
||||
continue
|
||||
spans.append((token_start + first, token_start + last + 1, start, end))
|
||||
|
||||
if not spans:
|
||||
return 0
|
||||
|
||||
out[:num_actual_tokens] = -1
|
||||
for row_start, row_end, start, end in spans:
|
||||
out[row_start:row_end] = (start, end)
|
||||
return num_actual_tokens
|
||||
|
||||
|
||||
def is_valid_kv_cache_layout(value: str) -> bool:
|
||||
return value in get_args(KVCacheLayoutType)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user