[Attention][Spec Decode] Multi-query Triton split-K for spec-decode verify, with empty-segment/padding hardening (#56148)

Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com>
Signed-off-by: Venky <23023424+venkywonka@users.noreply.github.com>
Co-authored-by: OpenAI GPT-6 Astra <noreply@openai.com>
Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Venky
2026-10-13 04:37:56 +04:00
committed by GitHub
co-authored by OpenAI GPT-6 Astra Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Claude Opus 5.5
parent 189bbfb9f4
commit 98f7211162
5 changed files with 500 additions and 25 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops import triton_unified_attention as attention
from vllm.v1.attention.ops.triton_attention_helpers import (
apply_softcap,
compute_tile_loop_bounds,
softmax_step,
)
from vllm.v1.attention.ops.triton_unified_attention import (
MAX_3D_QUERY_LEN,
reduce_segments,
unified_attention,
)
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
from vllm.v1.kv_cache_interface import KVQuantMode
pytestmark = pytest.mark.skip_global_cleanup
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
SEQ_THRESHOLD_3D_VALUES = [0, 8]
def _scratch(rows):
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
def _uses_splitk(
monkeypatch,
query_len,
scratch,
*,
causal=True,
threshold=8,
mm_prefix=None,
num_seqs=2,
):
query = torch.empty(
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
)
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
kernel = MagicMock()
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
unified_attention(
query,
cache,
cache,
torch.empty_like(query),
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
query_len,
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
128,
128**-0.5,
causal,
(-1, -1),
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
0.0,
None,
None,
None,
seq_threshold_3D=threshold,
num_par_softmax_segments=16,
softmax_segm_output=scratch[0],
softmax_segm_max=scratch[1],
softmax_segm_expsum=scratch[2],
mm_prefix_range=mm_prefix,
)
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
def _spy_reduce_segments(monkeypatch):
original = attention.reduce_segments
grids = []
def launch(grid):
grids.append(grid)
return original[grid]
spy = MagicMock()
spy.__getitem__.side_effect = launch
monkeypatch.setattr(attention, "reduce_segments", spy)
return grids
@pytest.mark.parametrize(
"query_len,options,expected",
[
(1, {}, True),
(1, {"causal": False}, True),
(5, {}, True),
(6, {"num_seqs": 1}, True),
(8, {}, True),
(9, {"num_seqs": 1}, False),
(5, {"causal": False}, False),
(5, {"causal": "per_sequence"}, False),
(5, {"threshold": 1}, False),
(5, {"batch_invariant": True}, False),
(5, {"mm_prefix": True}, False),
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
(4, {"num_seqs": 4}, True),
(5, {"num_seqs": 4}, False),
(5, {"scratch_rows": 9}, False),
],
)
def test_splitk_admission_preserves_decode_and_bounds_verification(
monkeypatch, query_len, options, expected
):
options = options.copy()
monkeypatch.setattr(
attention, "is_batch_invariant", options.pop("batch_invariant", False)
)
num_seqs = options.setdefault("num_seqs", 2)
if options.get("causal") == "per_sequence":
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
if options.get("mm_prefix"):
options["mm_prefix"] = torch.zeros(
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
)
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
"""Empty segments must not suppress strongly negative, nonempty segments."""
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
if empty_state != "all_empty":
partial[:, :, 0] = 3.0
maxima[:, :, 0] = -1000.0
if empty_state in ("masked", "all_empty"):
maxima[:, :, 1:] = 0.0
masses[:, :, 1:] = 0.0
reduce_segments[(1, 1)](
output,
partial,
maxima,
masses,
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
1,
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
16,
16,
False,
)
torch.testing.assert_close(
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
)
@triton.jit
def _softmax_after_empty_tile(output_ptr):
maximum = tl.full((1,), float("-inf"), tl.float32)
mass = tl.full((1,), 1.0, tl.float32)
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
scores = tl.full((1, 16), -1000.0, tl.float32)
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
"""A masked first tile must not suppress later, strongly negative logits."""
probabilities = torch.empty(16, device=DEVICE_TYPE)
_softmax_after_empty_tile[(1,)](probabilities)
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
@pytest.mark.parametrize(
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
)
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
"""A padded reducer row must return before reading sequence or scratch data."""
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
query_starts = torch.tensor(
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
).cumsum(0, dtype=torch.int32)
reduce_segments[(10, 1)](
output,
partial,
maxima,
masses,
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
len(seq_lens),
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
query_starts,
16,
16,
False,
)
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
torch.testing.assert_close(
output[:active_rows], torch.zeros_like(output[:active_rows])
)
torch.testing.assert_close(
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
)
@pytest.mark.parametrize(
("seq_lens", "num_heads", "q_dtype"),
[
*[
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
],
# Prefill chunk, verification, decode, padding, short query.
pytest.param(
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
),
*[
pytest.param(
[(6, 128), (6, 129), (6, 257)],
heads,
FP8_DTYPE,
id=f"w6-fp8q-kv{heads[1]}",
)
for heads in [(8, 1), (8, 8)]
],
],
)
@pytest.mark.parametrize("sliding_window", [None, 32])
def test_speculative_splitk_matches_reference(
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=128,
sliding_window=sliding_window,
dtype=torch.bfloat16,
block_size=128,
soft_cap=None,
num_blocks=8,
q_dtype=q_dtype,
seq_threshold_3D=32,
)
assert len(reduce_grids) == 1
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn_bf16_query_fp8_kv(
seq_lens=[(6, 128), (6, 129), (6, 4097)],
num_heads=(8, 1),
head_size=128,
block_size=128,
num_blocks=64,
seq_threshold_3D=16,
)
assert len(reduce_grids) == 1
@triton.jit
def _compute_clamped_mm_tile_bounds(
output_ptr,
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
from vllm.v1.attention.backends.triton_attn_diffkv import (
TritonAttentionDiffKVMetadataBuilder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
def _build(
monkeypatch,
builder_cls,
capability,
num_kv_heads,
query_lens,
max_num_seqs=16,
num_speculative_tokens=None,
parallel_drafting=False,
kv_quant_mode=KVQuantMode.NONE,
):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
current_platform, "is_device_capability", lambda cap: cap == capability
)
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: num_kv_heads,
get_head_size=lambda: 128,
rswa_window=None,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
speculative_config=(
None
if num_speculative_tokens is None
else SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=parallel_drafting,
)
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=num_kv_heads,
head_size=128,
dtype=torch.bfloat16,
kv_quant_mode=kv_quant_mode,
)
builder = builder_cls(spec, ["layer.0"], config, "cpu")
batch = BatchSpec(seq_lens=[128] * len(query_lens), query_lens=query_lens)
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
assert metadata.softmax_segm_output.shape[2] == metadata.num_par_softmax_segments
assert metadata.softmax_segm_max.shape[0] >= sum(query_lens)
# DiffKV re-allocates the output; it must stay sized like the base buffers.
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
assert (
metadata.softmax_segm_output.data_ptr()
== builder.softmax_segm_output.data_ptr()
)
return builder, metadata
BUILDERS = [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder]
# Verify batches: the 64-segment scratch holds min(11, max_num_seqs) * (1 + K)
# rows, and its 16-segment view covers every 3D-eligible sequence. 64 segments
# follow the request count, not Q blocks: on SM12.0 they stay faster for verify
# batches up to 11 requests (22 Q blocks at K=3).
@pytest.mark.parametrize("builder_cls", BUILDERS)
@pytest.mark.parametrize(
"max_num_seqs,num_speculative_tokens,num_seqs,rows,segments",
[
(16, 3, 10, 44, 64),
(16, 3, 11, 44, 64),
(16, 3, 12, 44, 16),
(16, 3, 16, 44, 16),
(1, 5, 1, 6, 64),
],
)
def test_split_k_segment_rows_cover_verify_queries(
monkeypatch,
builder_cls,
max_num_seqs,
num_speculative_tokens,
num_seqs,
rows,
segments,
):
builder, metadata = _build(
monkeypatch,
builder_cls,
(12, 0),
1,
[1 + num_speculative_tokens] * num_seqs,
max_num_seqs,
num_speculative_tokens,
)
assert builder.softmax_segm_max.shape[:2] == (rows, 8)
assert metadata.num_par_softmax_segments == segments
# Without SM12.0 64-segment reuse, the scratch holds exactly
# min(seq_threshold_3D, max_num_seqs) * query_len rows for split-K query_lens.
@pytest.mark.parametrize(
"num_speculative_tokens,parallel_drafting,max_num_seqs,rows",
[
(None, False, 8, 8),
(4, False, 8, 40),
(7, False, 8, 64),
(8, False, 8, 64),
(3, True, 8, 56),
(4, True, 8, 64),
(5, False, 1, 6),
],
)
@pytest.mark.parametrize(
"kv_quant_mode", [KVQuantMode.NONE, KVQuantMode.INT4_PER_TOKEN_HEAD]
)
def test_split_k_scratch_rows_follow_speculative_config(
monkeypatch,
num_speculative_tokens,
parallel_drafting,
max_num_seqs,
rows,
kv_quant_mode,
):
# The INT4 launcher never runs multi-query split-K.
if kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
rows = max_num_seqs
builder, _ = _build(
monkeypatch,
TritonAttentionMetadataBuilder,
(9, 0),
1,
[1],
max_num_seqs,
num_speculative_tokens,
parallel_drafting,
kv_quant_mode,
)
assert builder.softmax_segm_output.shape == (rows, 8, 16, 128)
assert builder.softmax_segm_max.shape == (rows, 8, 16)
assert builder.softmax_segm_expsum.shape == (rows, 8, 16)
+27 -5
View File
@@ -39,7 +39,10 @@ from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash,
triton_reshape_and_cache_flash_per_token_head_quant,
)
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
from vllm.v1.attention.ops.triton_unified_attention import (
MAX_3D_QUERY_LEN,
unified_attention,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVQuantMode,
@@ -144,7 +147,10 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
)
self.num_par_softmax_segments = NUM_PAR_SOFTMAX_SEGMENTS
# On SM120, batches whose 16-segment grid under-fills the SMs use 64.
# On SM120, batches whose 16-segment decode grid under-fills the SMs use
# 64. This counts requests, not Q blocks, on purpose: for 1..11 verify
# requests (K=3/5, 8 q / 1 KV head, 2k-128k context) 64 segments were
# faster or within 6% at 2k, and 1.1-3.6x faster from 8k up.
self.max_seqs_64_segments = 0
if current_platform.is_cuda() and current_platform.is_device_capability(
(12, 0)
@@ -152,18 +158,34 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
self.max_seqs_64_segments = (current_platform.num_compute_units() - 1) // (
NUM_PAR_SOFTMAX_SEGMENTS * self.num_heads_kv
)
max_num_tokens_3d = self.seq_threshold_3D
headdim_padded = next_power_of_2(self.headdim)
max_query_len_3d = 1
speculative_config = vllm_config.speculative_config
# The INT4 launcher runs multi-query batches in 2D only.
if (
speculative_config is not None
and kv_cache_spec.kv_quant_mode != KVQuantMode.INT4_PER_TOKEN_HEAD
):
num_speculative_tokens = speculative_config.num_speculative_tokens or 0
query_len = 1 + num_speculative_tokens * (
2 if speculative_config.parallel_drafting else 1
)
max_query_len_3d = min(query_len, MAX_3D_QUERY_LEN)
# Scratch is indexed by query token, including verification tokens.
max_num_seqs_3d = min(
self.seq_threshold_3D, vllm_config.scheduler_config.max_num_seqs
)
max_num_tokens_3d = max_num_seqs_3d * max_query_len_3d
if self.max_seqs_64_segments > 0:
self.num_par_softmax_segments = 64
# build() reuses this scratch at 16 segments with proportionally more rows.
max_num_tokens_3d = max(
min(self.max_seqs_64_segments, max_num_tokens_3d),
min(self.max_seqs_64_segments, max_num_seqs_3d) * max_query_len_3d,
cdiv(
max_num_tokens_3d,
self.num_par_softmax_segments // NUM_PAR_SOFTMAX_SEGMENTS,
),
)
headdim_padded = next_power_of_2(self.headdim)
self.softmax_segm_output = torch.empty(
(
max_num_tokens_3d,
@@ -470,15 +470,15 @@ def softmax_step(S, M, L):
# compute running maximum
# m_j : (BLOCK_M,)
m_j = tl.maximum(M, tl.max(S, axis=1))
# For sliding window there's a chance the max is -inf due to masking of
# the entire row. In this case we need to set m_j 0 to avoid NaN
m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
# Keep an empty row's maximum at -inf for later tiles; use a finite shift
# only for the exponent arithmetic below.
safe_m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
# P : (BLOCK_M, TILE_SIZE)
P = tl.exp(S - m_j[:, None])
P = tl.exp(S - safe_m_j[:, None])
# l_j : (BLOCK_M,)
l_j = tl.sum(P, axis=1)
# alpha : (BLOCK_M, )
alpha = tl.exp(M - m_j)
alpha = tl.exp(M - safe_m_j)
# update constants
L_new = L * alpha + l_j
return m_j, L_new, P, alpha
@@ -33,6 +33,8 @@ from vllm.v1.kv_cache_interface import KVQuantMode
logger = init_logger(__name__)
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
float8_info = torch.finfo(current_platform.fp8_dtype())
# Longest per-request query (e.g. 1 + num_speculative_tokens) run by split-K.
MAX_3D_QUERY_LEN = 8
@triton.jit
@@ -721,12 +723,18 @@ def reduce_segments(
query_token_idx = tl.program_id(0)
query_head_idx = tl.program_id(1)
# Graph padding must be rejected before sequence lookup or scratch access.
if query_token_idx >= tl.load(query_start_len_ptr + num_seqs):
return
seq_idx = find_seq_idx(
query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False
)
# sequence len for this particular sequence
seq_len = tl.load(seq_lens_ptr + seq_idx)
if seq_len <= 0:
return
# number of segments for this particular sequence
num_segments = NUM_SEGMENTS_PER_SEQ
@@ -746,11 +754,16 @@ def reduce_segments(
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)
)
segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf"))
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
# Empty segments must not set the maximum. Unvisited segments retain
# the M=-inf,L=1 initial state.
valid_segment = segm_mask & (segm_expsum > 0) & (segm_max != float("-inf"))
segm_max = tl.where(valid_segment, segm_max, float("-inf"))
overall_max = tl.max(segm_max)
# load and rescale segment exp sums
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
segm_expsum = segm_expsum * tl.exp(segm_max - overall_max)
segment_scale = tl.where(valid_segment, tl.exp(segm_max - overall_max), 0.0)
segm_expsum = segm_expsum * segment_scale
overall_expsum = tl.sum(segm_expsum)
# load, rescale, and add segment attention outputs
@@ -766,7 +779,7 @@ def reduce_segments(
mask=segm_mask[:, None] & dim_mask[None, :],
other=0.0,
)
segm_output *= tl.exp(segm_max - overall_max)[:, None]
segm_output *= segment_scale[:, None]
acc_sum = tl.sum(segm_output, axis=0)
# safely divide by overall_expsum, returning 0.0 if overall_expsum is 0
acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum)
@@ -1047,9 +1060,12 @@ def unified_attention(
)
# Launch the 2D kernel if
# 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or
# 2. The batch includes at least one prefill request, or
# 3. The number of sequences exceeds the configured threshold, or
# 1. No split-K threshold or scratch buffers, or the scratch has fewer
# rows than query tokens, or
# 2. max_seqlen_q > 1 and either it exceeds MAX_3D_QUERY_LEN, masking is
# non-causal, per-sequence causal, or mm-prefix, or the tuned
# large-head path applies, or
# 3. The number of Q blocks exceeds seq_threshold_3D, or
# 4. Batch invariance is enabled
use_3d = not (
seq_threshold_3D is None
@@ -1057,10 +1073,22 @@ def unified_attention(
or softmax_segm_output is None
or softmax_segm_max is None
or softmax_segm_expsum is None
or max_seqlen_q > 1
or num_seqs > seq_threshold_3D
or (
max_seqlen_q > 1
and (
max_seqlen_q > MAX_3D_QUERY_LEN
or not use_causal
or use_per_seq_causal
or use_mm_prefix
or tuned_large_head
)
)
or num_seqs * triton.cdiv(max_seqlen_q, BLOCK_Q) > seq_threshold_3D
or is_batch_invariant
)
if use_3d and softmax_segm_max.shape[0] < q.shape[0]:
logger.debug_once("Split-K scratch rows < query tokens; using 2D.")
use_3d = False
# The kernel signature is the same for 2D and 3D — only the launch
# grid + a handful of constexpr toggles differ. Per-token-head scale