[Attention][Spec Decode] Multi-query Triton split-K for spec-decode verify, with empty-segment/padding hardening (#56148)

Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com>
Signed-off-by: Venky <23023424+venkywonka@users.noreply.github.com>
Co-authored-by: OpenAI GPT-6 Astra <noreply@openai.com>
Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Venky
2026-10-13 04:37:56 +04:00
committed by GitHub
co-authored by OpenAI GPT-6 Astra Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Claude Opus 5.5
parent 189bbfb9f4
commit 98f7211162
5 changed files with 500 additions and 25 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops import triton_unified_attention as attention
from vllm.v1.attention.ops.triton_attention_helpers import (
apply_softcap,
compute_tile_loop_bounds,
softmax_step,
)
from vllm.v1.attention.ops.triton_unified_attention import (
MAX_3D_QUERY_LEN,
reduce_segments,
unified_attention,
)
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
from vllm.v1.kv_cache_interface import KVQuantMode
pytestmark = pytest.mark.skip_global_cleanup
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
SEQ_THRESHOLD_3D_VALUES = [0, 8]
def _scratch(rows):
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
def _uses_splitk(
monkeypatch,
query_len,
scratch,
*,
causal=True,
threshold=8,
mm_prefix=None,
num_seqs=2,
):
query = torch.empty(
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
)
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
kernel = MagicMock()
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
unified_attention(
query,
cache,
cache,
torch.empty_like(query),
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
query_len,
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
128,
128**-0.5,
causal,
(-1, -1),
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
0.0,
None,
None,
None,
seq_threshold_3D=threshold,
num_par_softmax_segments=16,
softmax_segm_output=scratch[0],
softmax_segm_max=scratch[1],
softmax_segm_expsum=scratch[2],
mm_prefix_range=mm_prefix,
)
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
def _spy_reduce_segments(monkeypatch):
original = attention.reduce_segments
grids = []
def launch(grid):
grids.append(grid)
return original[grid]
spy = MagicMock()
spy.__getitem__.side_effect = launch
monkeypatch.setattr(attention, "reduce_segments", spy)
return grids
@pytest.mark.parametrize(
"query_len,options,expected",
[
(1, {}, True),
(1, {"causal": False}, True),
(5, {}, True),
(6, {"num_seqs": 1}, True),
(8, {}, True),
(9, {"num_seqs": 1}, False),
(5, {"causal": False}, False),
(5, {"causal": "per_sequence"}, False),
(5, {"threshold": 1}, False),
(5, {"batch_invariant": True}, False),
(5, {"mm_prefix": True}, False),
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
(4, {"num_seqs": 4}, True),
(5, {"num_seqs": 4}, False),
(5, {"scratch_rows": 9}, False),
],
)
def test_splitk_admission_preserves_decode_and_bounds_verification(
monkeypatch, query_len, options, expected
):
options = options.copy()
monkeypatch.setattr(
attention, "is_batch_invariant", options.pop("batch_invariant", False)
)
num_seqs = options.setdefault("num_seqs", 2)
if options.get("causal") == "per_sequence":
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
if options.get("mm_prefix"):
options["mm_prefix"] = torch.zeros(
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
)
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
"""Empty segments must not suppress strongly negative, nonempty segments."""
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
if empty_state != "all_empty":
partial[:, :, 0] = 3.0
maxima[:, :, 0] = -1000.0
if empty_state in ("masked", "all_empty"):
maxima[:, :, 1:] = 0.0
masses[:, :, 1:] = 0.0
reduce_segments[(1, 1)](
output,
partial,
maxima,
masses,
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
1,
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
16,
16,
False,
)
torch.testing.assert_close(
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
)
@triton.jit
def _softmax_after_empty_tile(output_ptr):
maximum = tl.full((1,), float("-inf"), tl.float32)
mass = tl.full((1,), 1.0, tl.float32)
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
scores = tl.full((1, 16), -1000.0, tl.float32)
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
"""A masked first tile must not suppress later, strongly negative logits."""
probabilities = torch.empty(16, device=DEVICE_TYPE)
_softmax_after_empty_tile[(1,)](probabilities)
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
@pytest.mark.parametrize(
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
)
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
"""A padded reducer row must return before reading sequence or scratch data."""
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
query_starts = torch.tensor(
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
).cumsum(0, dtype=torch.int32)
reduce_segments[(10, 1)](
output,
partial,
maxima,
masses,
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
len(seq_lens),
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
query_starts,
16,
16,
False,
)
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
torch.testing.assert_close(
output[:active_rows], torch.zeros_like(output[:active_rows])
)
torch.testing.assert_close(
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
)
@pytest.mark.parametrize(
("seq_lens", "num_heads", "q_dtype"),
[
*[
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
],
# Prefill chunk, verification, decode, padding, short query.
pytest.param(
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
),
*[
pytest.param(
[(6, 128), (6, 129), (6, 257)],
heads,
FP8_DTYPE,
id=f"w6-fp8q-kv{heads[1]}",
)
for heads in [(8, 1), (8, 8)]
],
],
)
@pytest.mark.parametrize("sliding_window", [None, 32])
def test_speculative_splitk_matches_reference(
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=128,
sliding_window=sliding_window,
dtype=torch.bfloat16,
block_size=128,
soft_cap=None,
num_blocks=8,
q_dtype=q_dtype,
seq_threshold_3D=32,
)
assert len(reduce_grids) == 1
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn_bf16_query_fp8_kv(
seq_lens=[(6, 128), (6, 129), (6, 4097)],
num_heads=(8, 1),
head_size=128,
block_size=128,
num_blocks=64,
seq_threshold_3D=16,
)
assert len(reduce_grids) == 1
@triton.jit
def _compute_clamped_mm_tile_bounds(
output_ptr,
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)