[Attention][Spec Decode] Multi-query Triton split-K for spec-decode verify, with empty-segment/padding hardening (#56148)
Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Signed-off-by: Venky <23023424+venkywonka@users.noreply.github.com> Co-authored-by: OpenAI GPT-6 Astra <noreply@openai.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI GPT-6 Astra
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Claude Opus 5.5
parent
189bbfb9f4
commit
98f7211162
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops import triton_unified_attention as attention
|
||||
from vllm.v1.attention.ops.triton_attention_helpers import (
|
||||
apply_softcap,
|
||||
compute_tile_loop_bounds,
|
||||
softmax_step,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import (
|
||||
MAX_3D_QUERY_LEN,
|
||||
reduce_segments,
|
||||
unified_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
|
||||
SEQ_THRESHOLD_3D_VALUES = [0, 8]
|
||||
|
||||
|
||||
def _scratch(rows):
|
||||
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
|
||||
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
|
||||
|
||||
|
||||
def _uses_splitk(
|
||||
monkeypatch,
|
||||
query_len,
|
||||
scratch,
|
||||
*,
|
||||
causal=True,
|
||||
threshold=8,
|
||||
mm_prefix=None,
|
||||
num_seqs=2,
|
||||
):
|
||||
query = torch.empty(
|
||||
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
|
||||
)
|
||||
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
|
||||
kernel = MagicMock()
|
||||
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
|
||||
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
|
||||
unified_attention(
|
||||
query,
|
||||
cache,
|
||||
cache,
|
||||
torch.empty_like(query),
|
||||
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
|
||||
query_len,
|
||||
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
|
||||
128,
|
||||
128**-0.5,
|
||||
causal,
|
||||
(-1, -1),
|
||||
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
|
||||
0.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
seq_threshold_3D=threshold,
|
||||
num_par_softmax_segments=16,
|
||||
softmax_segm_output=scratch[0],
|
||||
softmax_segm_max=scratch[1],
|
||||
softmax_segm_expsum=scratch[2],
|
||||
mm_prefix_range=mm_prefix,
|
||||
)
|
||||
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
|
||||
|
||||
|
||||
def _spy_reduce_segments(monkeypatch):
|
||||
original = attention.reduce_segments
|
||||
grids = []
|
||||
|
||||
def launch(grid):
|
||||
grids.append(grid)
|
||||
return original[grid]
|
||||
|
||||
spy = MagicMock()
|
||||
spy.__getitem__.side_effect = launch
|
||||
monkeypatch.setattr(attention, "reduce_segments", spy)
|
||||
return grids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,options,expected",
|
||||
[
|
||||
(1, {}, True),
|
||||
(1, {"causal": False}, True),
|
||||
(5, {}, True),
|
||||
(6, {"num_seqs": 1}, True),
|
||||
(8, {}, True),
|
||||
(9, {"num_seqs": 1}, False),
|
||||
(5, {"causal": False}, False),
|
||||
(5, {"causal": "per_sequence"}, False),
|
||||
(5, {"threshold": 1}, False),
|
||||
(5, {"batch_invariant": True}, False),
|
||||
(5, {"mm_prefix": True}, False),
|
||||
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
|
||||
(4, {"num_seqs": 4}, True),
|
||||
(5, {"num_seqs": 4}, False),
|
||||
(5, {"scratch_rows": 9}, False),
|
||||
],
|
||||
)
|
||||
def test_splitk_admission_preserves_decode_and_bounds_verification(
|
||||
monkeypatch, query_len, options, expected
|
||||
):
|
||||
options = options.copy()
|
||||
monkeypatch.setattr(
|
||||
attention, "is_batch_invariant", options.pop("batch_invariant", False)
|
||||
)
|
||||
num_seqs = options.setdefault("num_seqs", 2)
|
||||
if options.get("causal") == "per_sequence":
|
||||
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
|
||||
if options.get("mm_prefix"):
|
||||
options["mm_prefix"] = torch.zeros(
|
||||
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
|
||||
)
|
||||
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
|
||||
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
|
||||
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
|
||||
"""Empty segments must not suppress strongly negative, nonempty segments."""
|
||||
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
|
||||
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
if empty_state != "all_empty":
|
||||
partial[:, :, 0] = 3.0
|
||||
maxima[:, :, 0] = -1000.0
|
||||
if empty_state in ("masked", "all_empty"):
|
||||
maxima[:, :, 1:] = 0.0
|
||||
masses[:, :, 1:] = 0.0
|
||||
reduce_segments[(1, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
1,
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _softmax_after_empty_tile(output_ptr):
|
||||
maximum = tl.full((1,), float("-inf"), tl.float32)
|
||||
mass = tl.full((1,), 1.0, tl.float32)
|
||||
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
|
||||
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
|
||||
scores = tl.full((1, 16), -1000.0, tl.float32)
|
||||
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
|
||||
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
|
||||
|
||||
|
||||
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
|
||||
"""A masked first tile must not suppress later, strongly negative logits."""
|
||||
probabilities = torch.empty(16, device=DEVICE_TYPE)
|
||||
_softmax_after_empty_tile[(1,)](probabilities)
|
||||
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
|
||||
)
|
||||
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
|
||||
"""A padded reducer row must return before reading sequence or scratch data."""
|
||||
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
|
||||
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
query_starts = torch.tensor(
|
||||
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
).cumsum(0, dtype=torch.int32)
|
||||
reduce_segments[(10, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
len(seq_lens),
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
query_starts,
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
|
||||
torch.testing.assert_close(
|
||||
output[:active_rows], torch.zeros_like(output[:active_rows])
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_lens", "num_heads", "q_dtype"),
|
||||
[
|
||||
*[
|
||||
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
|
||||
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
|
||||
],
|
||||
# Prefill chunk, verification, decode, padding, short query.
|
||||
pytest.param(
|
||||
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
|
||||
),
|
||||
*[
|
||||
pytest.param(
|
||||
[(6, 128), (6, 129), (6, 257)],
|
||||
heads,
|
||||
FP8_DTYPE,
|
||||
id=f"w6-fp8q-kv{heads[1]}",
|
||||
)
|
||||
for heads in [(8, 1), (8, 8)]
|
||||
],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sliding_window", [None, 32])
|
||||
def test_speculative_splitk_matches_reference(
|
||||
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
|
||||
):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=128,
|
||||
sliding_window=sliding_window,
|
||||
dtype=torch.bfloat16,
|
||||
block_size=128,
|
||||
soft_cap=None,
|
||||
num_blocks=8,
|
||||
q_dtype=q_dtype,
|
||||
seq_threshold_3D=32,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
seq_lens=[(6, 128), (6, 129), (6, 4097)],
|
||||
num_heads=(8, 1),
|
||||
head_size=128,
|
||||
block_size=128,
|
||||
num_blocks=64,
|
||||
seq_threshold_3D=16,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_clamped_mm_tile_bounds(
|
||||
output_ptr,
|
||||
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user