diff --git a/tests/kernels/attention/test_triton_unified_attention.py b/tests/kernels/attention/test_triton_unified_attention.py index d936e37704..b50e97b52b 100644 --- a/tests/kernels/attention/test_triton_unified_attention.py +++ b/tests/kernels/attention/test_triton_unified_attention.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import MagicMock + import pytest import torch @@ -9,11 +11,17 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.math_utils import next_power_of_2 from vllm.utils.torch_utils import set_random_seed +from vllm.v1.attention.ops import triton_unified_attention as attention from vllm.v1.attention.ops.triton_attention_helpers import ( apply_softcap, compute_tile_loop_bounds, + softmax_step, +) +from vllm.v1.attention.ops.triton_unified_attention import ( + MAX_3D_QUERY_LEN, + reduce_segments, + unified_attention, ) -from vllm.v1.attention.ops.triton_unified_attention import unified_attention from vllm.v1.kv_cache_interface import KVQuantMode pytestmark = pytest.mark.skip_global_cleanup @@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048] SEQ_THRESHOLD_3D_VALUES = [0, 8] +def _scratch(rows): + shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)] + return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes] + + +def _uses_splitk( + monkeypatch, + query_len, + scratch, + *, + causal=True, + threshold=8, + mm_prefix=None, + num_seqs=2, +): + query = torch.empty( + (num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu" + ) + cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu") + kernel = MagicMock() + monkeypatch.setattr(attention, "kernel_unified_attention", kernel) + monkeypatch.setattr(attention, "reduce_segments", MagicMock()) + unified_attention( + query, + cache, + cache, + torch.empty_like(query), + torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len, + query_len, + torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"), + 128, + 128**-0.5, + causal, + (-1, -1), + torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"), + 0.0, + None, + None, + None, + seq_threshold_3D=threshold, + num_par_softmax_segments=16, + softmax_segm_output=scratch[0], + softmax_segm_max=scratch[1], + softmax_segm_expsum=scratch[2], + mm_prefix_range=mm_prefix, + ) + return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"] + + +def _spy_reduce_segments(monkeypatch): + original = attention.reduce_segments + grids = [] + + def launch(grid): + grids.append(grid) + return original[grid] + + spy = MagicMock() + spy.__getitem__.side_effect = launch + monkeypatch.setattr(attention, "reduce_segments", spy) + return grids + + +@pytest.mark.parametrize( + "query_len,options,expected", + [ + (1, {}, True), + (1, {"causal": False}, True), + (5, {}, True), + (6, {"num_seqs": 1}, True), + (8, {}, True), + (9, {"num_seqs": 1}, False), + (5, {"causal": False}, False), + (5, {"causal": "per_sequence"}, False), + (5, {"threshold": 1}, False), + (5, {"batch_invariant": True}, False), + (5, {"mm_prefix": True}, False), + # BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8. + (4, {"num_seqs": 4}, True), + (5, {"num_seqs": 4}, False), + (5, {"scratch_rows": 9}, False), + ], +) +def test_splitk_admission_preserves_decode_and_bounds_verification( + monkeypatch, query_len, options, expected +): + options = options.copy() + monkeypatch.setattr( + attention, "is_batch_invariant", options.pop("batch_invariant", False) + ) + num_seqs = options.setdefault("num_seqs", 2) + if options.get("causal") == "per_sequence": + options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu") + if options.get("mm_prefix"): + options["mm_prefix"] = torch.zeros( + (num_seqs, 1, 2), dtype=torch.int32, device="cpu" + ) + scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len)) + assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected + + +@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"]) +def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None: + """Empty segments must not suppress strongly negative, nonempty segments.""" + output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE) + partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE) + maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE) + masses = torch.ones_like(maxima) + if empty_state != "all_empty": + partial[:, :, 0] = 3.0 + maxima[:, :, 0] = -1000.0 + if empty_state in ("masked", "all_empty"): + maxima[:, :, 1:] = 0.0 + masses[:, :, 1:] = 0.0 + reduce_segments[(1, 1)]( + output, + partial, + maxima, + masses, + torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE), + 1, + 1, + 1.0, + output.stride(0), + output.stride(1), + 1, + 16, + 32, + 32, + torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE), + 16, + 16, + False, + ) + torch.testing.assert_close( + output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0) + ) + + +@triton.jit +def _softmax_after_empty_tile(output_ptr): + maximum = tl.full((1,), float("-inf"), tl.float32) + mass = tl.full((1,), 1.0, tl.float32) + empty_scores = tl.full((1, 16), float("-inf"), tl.float32) + maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass) + scores = tl.full((1, 16), -1000.0, tl.float32) + _, mass, probabilities, _ = softmax_step(scores, maximum, mass) + tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass) + + +def test_splitk_softmax_recovers_after_fully_masked_tile() -> None: + """A masked first tile must not suppress later, strongly negative logits.""" + probabilities = torch.empty(16, device=DEVICE_TYPE) + _softmax_after_empty_tile[(1,)](probabilities) + torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16)) + + +@pytest.mark.parametrize( + "query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])] +) +def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None: + """A padded reducer row must return before reading sequence or scratch data.""" + output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE) + partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE) + maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE) + masses = torch.ones_like(maxima) + query_starts = torch.tensor( + [0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE + ).cumsum(0, dtype=torch.int32) + reduce_segments[(10, 1)]( + output, + partial, + maxima, + masses, + torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE), + len(seq_lens), + 1, + 1.0, + output.stride(0), + output.stride(1), + 1, + 16, + 32, + 32, + query_starts, + 16, + 16, + False, + ) + active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0) + torch.testing.assert_close( + output[:active_rows], torch.zeros_like(output[:active_rows]) + ) + torch.testing.assert_close( + output[active_rows:], torch.full_like(output[active_rows:], 123.0) + ) + + +@pytest.mark.parametrize( + ("seq_lens", "num_heads", "q_dtype"), + [ + *[ + pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}") + for w in range(2, MAX_3D_QUERY_LEN + 1, 2) + ], + # Prefill chunk, verification, decode, padding, short query. + pytest.param( + [(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed" + ), + *[ + pytest.param( + [(6, 128), (6, 129), (6, 257)], + heads, + FP8_DTYPE, + id=f"w6-fp8q-kv{heads[1]}", + ) + for heads in [(8, 1), (8, 8)] + ], + ], +) +@pytest.mark.parametrize("sliding_window", [None, 32]) +def test_speculative_splitk_matches_reference( + monkeypatch, seq_lens, num_heads, q_dtype, sliding_window +): + reduce_grids = _spy_reduce_segments(monkeypatch) + test_triton_unified_attn( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=128, + sliding_window=sliding_window, + dtype=torch.bfloat16, + block_size=128, + soft_cap=None, + num_blocks=8, + q_dtype=q_dtype, + seq_threshold_3D=32, + ) + assert len(reduce_grids) == 1 + + +def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch): + reduce_grids = _spy_reduce_segments(monkeypatch) + test_triton_unified_attn_bf16_query_fp8_kv( + seq_lens=[(6, 128), (6, 129), (6, 4097)], + num_heads=(8, 1), + head_size=128, + block_size=128, + num_blocks=64, + seq_threshold_3D=16, + ) + assert len(reduce_grids) == 1 + + @triton.jit def _compute_clamped_mm_tile_bounds( output_ptr, @@ -471,16 +732,19 @@ def test_triton_unified_attn( num_par_softmax_segments = 16 head_size_padded = next_power_of_2(head_size) + scratch_rows = seq_threshold_3D * ( + max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1 + ) softmax_segm_output = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded), + (scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded), dtype=torch.float32, ) softmax_segm_max = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + (scratch_rows, num_query_heads, num_par_softmax_segments), dtype=torch.float32, ) softmax_segm_expsum = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + (scratch_rows, num_query_heads, num_par_softmax_segments), dtype=torch.float32, ) @@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv( num_par_softmax_segments = 16 head_size_padded = next_power_of_2(head_size) + scratch_rows = seq_threshold_3D * ( + max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1 + ) softmax_segm_output = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded), + (scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded), dtype=torch.float32, ) softmax_segm_max = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + (scratch_rows, num_query_heads, num_par_softmax_segments), dtype=torch.float32, ) softmax_segm_expsum = torch.empty( - (seq_threshold_3D, num_query_heads, num_par_softmax_segments), + (scratch_rows, num_query_heads, num_par_softmax_segments), dtype=torch.float32, ) diff --git a/tests/v1/attention/test_triton_attn_segments.py b/tests/v1/attention/test_triton_attn_segments.py new file mode 100644 index 0000000000..ae764a14ac --- /dev/null +++ b/tests/v1/attention/test_triton_attn_segments.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm.config import CUDAGraphMode +from vllm.platforms import current_platform +from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder +from vllm.v1.attention.backends.triton_attn_diffkv import ( + TritonAttentionDiffKVMetadataBuilder, +) +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode + + +def _build( + monkeypatch, + builder_cls, + capability, + num_kv_heads, + query_lens, + max_num_seqs=16, + num_speculative_tokens=None, + parallel_drafting=False, + kv_quant_mode=KVQuantMode.NONE, +): + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr( + current_platform, "is_device_capability", lambda cap: cap == capability + ) + monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188) + config = SimpleNamespace( + model_config=SimpleNamespace( + get_num_attention_heads=lambda _: 8, + get_num_kv_heads=lambda _: num_kv_heads, + get_head_size=lambda: 128, + rswa_window=None, + ), + parallel_config=None, + scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs), + speculative_config=( + None + if num_speculative_tokens is None + else SimpleNamespace( + num_speculative_tokens=num_speculative_tokens, + parallel_drafting=parallel_drafting, + ) + ), + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={} + ), + ) + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=num_kv_heads, + head_size=128, + dtype=torch.bfloat16, + kv_quant_mode=kv_quant_mode, + ) + builder = builder_cls(spec, ["layer.0"], config, "cpu") + batch = BatchSpec(seq_lens=[128] * len(query_lens), query_lens=query_lens) + metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu")) + assert metadata.softmax_segm_output.shape[2] == metadata.num_par_softmax_segments + assert metadata.softmax_segm_max.shape[0] >= sum(query_lens) + # DiffKV re-allocates the output; it must stay sized like the base buffers. + assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0] + assert ( + metadata.softmax_segm_output.data_ptr() + == builder.softmax_segm_output.data_ptr() + ) + return builder, metadata + + +BUILDERS = [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder] + + +# Verify batches: the 64-segment scratch holds min(11, max_num_seqs) * (1 + K) +# rows, and its 16-segment view covers every 3D-eligible sequence. 64 segments +# follow the request count, not Q blocks: on SM12.0 they stay faster for verify +# batches up to 11 requests (22 Q blocks at K=3). +@pytest.mark.parametrize("builder_cls", BUILDERS) +@pytest.mark.parametrize( + "max_num_seqs,num_speculative_tokens,num_seqs,rows,segments", + [ + (16, 3, 10, 44, 64), + (16, 3, 11, 44, 64), + (16, 3, 12, 44, 16), + (16, 3, 16, 44, 16), + (1, 5, 1, 6, 64), + ], +) +def test_split_k_segment_rows_cover_verify_queries( + monkeypatch, + builder_cls, + max_num_seqs, + num_speculative_tokens, + num_seqs, + rows, + segments, +): + builder, metadata = _build( + monkeypatch, + builder_cls, + (12, 0), + 1, + [1 + num_speculative_tokens] * num_seqs, + max_num_seqs, + num_speculative_tokens, + ) + assert builder.softmax_segm_max.shape[:2] == (rows, 8) + assert metadata.num_par_softmax_segments == segments + + +# Without SM12.0 64-segment reuse, the scratch holds exactly +# min(seq_threshold_3D, max_num_seqs) * query_len rows for split-K query_lens. +@pytest.mark.parametrize( + "num_speculative_tokens,parallel_drafting,max_num_seqs,rows", + [ + (None, False, 8, 8), + (4, False, 8, 40), + (7, False, 8, 64), + (8, False, 8, 64), + (3, True, 8, 56), + (4, True, 8, 64), + (5, False, 1, 6), + ], +) +@pytest.mark.parametrize( + "kv_quant_mode", [KVQuantMode.NONE, KVQuantMode.INT4_PER_TOKEN_HEAD] +) +def test_split_k_scratch_rows_follow_speculative_config( + monkeypatch, + num_speculative_tokens, + parallel_drafting, + max_num_seqs, + rows, + kv_quant_mode, +): + # The INT4 launcher never runs multi-query split-K. + if kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD: + rows = max_num_seqs + builder, _ = _build( + monkeypatch, + TritonAttentionMetadataBuilder, + (9, 0), + 1, + [1], + max_num_seqs, + num_speculative_tokens, + parallel_drafting, + kv_quant_mode, + ) + assert builder.softmax_segm_output.shape == (rows, 8, 16, 128) + assert builder.softmax_segm_max.shape == (rows, 8, 16) + assert builder.softmax_segm_expsum.shape == (rows, 8, 16) diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index a95b0c1d0e..8b26bab84f 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -39,7 +39,10 @@ from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( triton_reshape_and_cache_flash, triton_reshape_and_cache_flash_per_token_head_quant, ) -from vllm.v1.attention.ops.triton_unified_attention import unified_attention +from vllm.v1.attention.ops.triton_unified_attention import ( + MAX_3D_QUERY_LEN, + unified_attention, +) from vllm.v1.kv_cache_interface import ( AttentionSpec, KVQuantMode, @@ -144,7 +147,10 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet ) self.num_par_softmax_segments = NUM_PAR_SOFTMAX_SEGMENTS - # On SM120, batches whose 16-segment grid under-fills the SMs use 64. + # On SM120, batches whose 16-segment decode grid under-fills the SMs use + # 64. This counts requests, not Q blocks, on purpose: for 1..11 verify + # requests (K=3/5, 8 q / 1 KV head, 2k-128k context) 64 segments were + # faster or within 6% at 2k, and 1.1-3.6x faster from 8k up. self.max_seqs_64_segments = 0 if current_platform.is_cuda() and current_platform.is_device_capability( (12, 0) @@ -152,18 +158,34 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet self.max_seqs_64_segments = (current_platform.num_compute_units() - 1) // ( NUM_PAR_SOFTMAX_SEGMENTS * self.num_heads_kv ) - max_num_tokens_3d = self.seq_threshold_3D + headdim_padded = next_power_of_2(self.headdim) + max_query_len_3d = 1 + speculative_config = vllm_config.speculative_config + # The INT4 launcher runs multi-query batches in 2D only. + if ( + speculative_config is not None + and kv_cache_spec.kv_quant_mode != KVQuantMode.INT4_PER_TOKEN_HEAD + ): + num_speculative_tokens = speculative_config.num_speculative_tokens or 0 + query_len = 1 + num_speculative_tokens * ( + 2 if speculative_config.parallel_drafting else 1 + ) + max_query_len_3d = min(query_len, MAX_3D_QUERY_LEN) + # Scratch is indexed by query token, including verification tokens. + max_num_seqs_3d = min( + self.seq_threshold_3D, vllm_config.scheduler_config.max_num_seqs + ) + max_num_tokens_3d = max_num_seqs_3d * max_query_len_3d if self.max_seqs_64_segments > 0: self.num_par_softmax_segments = 64 # build() reuses this scratch at 16 segments with proportionally more rows. max_num_tokens_3d = max( - min(self.max_seqs_64_segments, max_num_tokens_3d), + min(self.max_seqs_64_segments, max_num_seqs_3d) * max_query_len_3d, cdiv( max_num_tokens_3d, self.num_par_softmax_segments // NUM_PAR_SOFTMAX_SEGMENTS, ), ) - headdim_padded = next_power_of_2(self.headdim) self.softmax_segm_output = torch.empty( ( max_num_tokens_3d, diff --git a/vllm/v1/attention/ops/triton_attention_helpers.py b/vllm/v1/attention/ops/triton_attention_helpers.py index 7804b01317..f2399b7423 100644 --- a/vllm/v1/attention/ops/triton_attention_helpers.py +++ b/vllm/v1/attention/ops/triton_attention_helpers.py @@ -470,15 +470,15 @@ def softmax_step(S, M, L): # compute running maximum # m_j : (BLOCK_M,) m_j = tl.maximum(M, tl.max(S, axis=1)) - # For sliding window there's a chance the max is -inf due to masking of - # the entire row. In this case we need to set m_j 0 to avoid NaN - m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + # Keep an empty row's maximum at -inf for later tiles; use a finite shift + # only for the exponent arithmetic below. + safe_m_j = tl.where(m_j > float("-inf"), m_j, 0.0) # P : (BLOCK_M, TILE_SIZE) - P = tl.exp(S - m_j[:, None]) + P = tl.exp(S - safe_m_j[:, None]) # l_j : (BLOCK_M,) l_j = tl.sum(P, axis=1) # alpha : (BLOCK_M, ) - alpha = tl.exp(M - m_j) + alpha = tl.exp(M - safe_m_j) # update constants L_new = L * alpha + l_j return m_j, L_new, P, alpha diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index 63a9cdc7f9..d914e574f2 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -33,6 +33,8 @@ from vllm.v1.kv_cache_interface import KVQuantMode logger = init_logger(__name__) is_batch_invariant = envs.VLLM_BATCH_INVARIANT float8_info = torch.finfo(current_platform.fp8_dtype()) +# Longest per-request query (e.g. 1 + num_speculative_tokens) run by split-K. +MAX_3D_QUERY_LEN = 8 @triton.jit @@ -721,12 +723,18 @@ def reduce_segments( query_token_idx = tl.program_id(0) query_head_idx = tl.program_id(1) + # Graph padding must be rejected before sequence lookup or scratch access. + if query_token_idx >= tl.load(query_start_len_ptr + num_seqs): + return + seq_idx = find_seq_idx( query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False ) # sequence len for this particular sequence seq_len = tl.load(seq_lens_ptr + seq_idx) + if seq_len <= 0: + return # number of segments for this particular sequence num_segments = NUM_SEGMENTS_PER_SEQ @@ -746,11 +754,16 @@ def reduce_segments( + tl.arange(0, NUM_SEGMENTS_PER_SEQ) ) segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf")) + segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0) + # Empty segments must not set the maximum. Unvisited segments retain + # the M=-inf,L=1 initial state. + valid_segment = segm_mask & (segm_expsum > 0) & (segm_max != float("-inf")) + segm_max = tl.where(valid_segment, segm_max, float("-inf")) overall_max = tl.max(segm_max) # load and rescale segment exp sums - segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0) - segm_expsum = segm_expsum * tl.exp(segm_max - overall_max) + segment_scale = tl.where(valid_segment, tl.exp(segm_max - overall_max), 0.0) + segm_expsum = segm_expsum * segment_scale overall_expsum = tl.sum(segm_expsum) # load, rescale, and add segment attention outputs @@ -766,7 +779,7 @@ def reduce_segments( mask=segm_mask[:, None] & dim_mask[None, :], other=0.0, ) - segm_output *= tl.exp(segm_max - overall_max)[:, None] + segm_output *= segment_scale[:, None] acc_sum = tl.sum(segm_output, axis=0) # safely divide by overall_expsum, returning 0.0 if overall_expsum is 0 acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum) @@ -1047,9 +1060,12 @@ def unified_attention( ) # Launch the 2D kernel if - # 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or - # 2. The batch includes at least one prefill request, or - # 3. The number of sequences exceeds the configured threshold, or + # 1. No split-K threshold or scratch buffers, or the scratch has fewer + # rows than query tokens, or + # 2. max_seqlen_q > 1 and either it exceeds MAX_3D_QUERY_LEN, masking is + # non-causal, per-sequence causal, or mm-prefix, or the tuned + # large-head path applies, or + # 3. The number of Q blocks exceeds seq_threshold_3D, or # 4. Batch invariance is enabled use_3d = not ( seq_threshold_3D is None @@ -1057,10 +1073,22 @@ def unified_attention( or softmax_segm_output is None or softmax_segm_max is None or softmax_segm_expsum is None - or max_seqlen_q > 1 - or num_seqs > seq_threshold_3D + or ( + max_seqlen_q > 1 + and ( + max_seqlen_q > MAX_3D_QUERY_LEN + or not use_causal + or use_per_seq_causal + or use_mm_prefix + or tuned_large_head + ) + ) + or num_seqs * triton.cdiv(max_seqlen_q, BLOCK_Q) > seq_threshold_3D or is_batch_invariant ) + if use_3d and softmax_segm_max.shape[0] < q.shape[0]: + logger.debug_once("Split-K scratch rows < query tokens; using 2D.") + use_3d = False # The kernel signature is the same for 2D and 3D — only the launch # grid + a handful of constexpr toggles differ. Per-token-head scale