[Attention][Spec Decode] Multi-query Triton split-K for spec-decode verify, with empty-segment/padding hardening (#56148)
Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Signed-off-by: Venky <23023424+venkywonka@users.noreply.github.com> Co-authored-by: OpenAI GPT-6 Astra <noreply@openai.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI GPT-6 Astra
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Claude Opus 5.5
parent
189bbfb9f4
commit
98f7211162
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops import triton_unified_attention as attention
|
||||
from vllm.v1.attention.ops.triton_attention_helpers import (
|
||||
apply_softcap,
|
||||
compute_tile_loop_bounds,
|
||||
softmax_step,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import (
|
||||
MAX_3D_QUERY_LEN,
|
||||
reduce_segments,
|
||||
unified_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
|
||||
SEQ_THRESHOLD_3D_VALUES = [0, 8]
|
||||
|
||||
|
||||
def _scratch(rows):
|
||||
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
|
||||
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
|
||||
|
||||
|
||||
def _uses_splitk(
|
||||
monkeypatch,
|
||||
query_len,
|
||||
scratch,
|
||||
*,
|
||||
causal=True,
|
||||
threshold=8,
|
||||
mm_prefix=None,
|
||||
num_seqs=2,
|
||||
):
|
||||
query = torch.empty(
|
||||
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
|
||||
)
|
||||
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
|
||||
kernel = MagicMock()
|
||||
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
|
||||
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
|
||||
unified_attention(
|
||||
query,
|
||||
cache,
|
||||
cache,
|
||||
torch.empty_like(query),
|
||||
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
|
||||
query_len,
|
||||
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
|
||||
128,
|
||||
128**-0.5,
|
||||
causal,
|
||||
(-1, -1),
|
||||
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
|
||||
0.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
seq_threshold_3D=threshold,
|
||||
num_par_softmax_segments=16,
|
||||
softmax_segm_output=scratch[0],
|
||||
softmax_segm_max=scratch[1],
|
||||
softmax_segm_expsum=scratch[2],
|
||||
mm_prefix_range=mm_prefix,
|
||||
)
|
||||
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
|
||||
|
||||
|
||||
def _spy_reduce_segments(monkeypatch):
|
||||
original = attention.reduce_segments
|
||||
grids = []
|
||||
|
||||
def launch(grid):
|
||||
grids.append(grid)
|
||||
return original[grid]
|
||||
|
||||
spy = MagicMock()
|
||||
spy.__getitem__.side_effect = launch
|
||||
monkeypatch.setattr(attention, "reduce_segments", spy)
|
||||
return grids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,options,expected",
|
||||
[
|
||||
(1, {}, True),
|
||||
(1, {"causal": False}, True),
|
||||
(5, {}, True),
|
||||
(6, {"num_seqs": 1}, True),
|
||||
(8, {}, True),
|
||||
(9, {"num_seqs": 1}, False),
|
||||
(5, {"causal": False}, False),
|
||||
(5, {"causal": "per_sequence"}, False),
|
||||
(5, {"threshold": 1}, False),
|
||||
(5, {"batch_invariant": True}, False),
|
||||
(5, {"mm_prefix": True}, False),
|
||||
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
|
||||
(4, {"num_seqs": 4}, True),
|
||||
(5, {"num_seqs": 4}, False),
|
||||
(5, {"scratch_rows": 9}, False),
|
||||
],
|
||||
)
|
||||
def test_splitk_admission_preserves_decode_and_bounds_verification(
|
||||
monkeypatch, query_len, options, expected
|
||||
):
|
||||
options = options.copy()
|
||||
monkeypatch.setattr(
|
||||
attention, "is_batch_invariant", options.pop("batch_invariant", False)
|
||||
)
|
||||
num_seqs = options.setdefault("num_seqs", 2)
|
||||
if options.get("causal") == "per_sequence":
|
||||
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
|
||||
if options.get("mm_prefix"):
|
||||
options["mm_prefix"] = torch.zeros(
|
||||
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
|
||||
)
|
||||
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
|
||||
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
|
||||
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
|
||||
"""Empty segments must not suppress strongly negative, nonempty segments."""
|
||||
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
|
||||
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
if empty_state != "all_empty":
|
||||
partial[:, :, 0] = 3.0
|
||||
maxima[:, :, 0] = -1000.0
|
||||
if empty_state in ("masked", "all_empty"):
|
||||
maxima[:, :, 1:] = 0.0
|
||||
masses[:, :, 1:] = 0.0
|
||||
reduce_segments[(1, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
1,
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _softmax_after_empty_tile(output_ptr):
|
||||
maximum = tl.full((1,), float("-inf"), tl.float32)
|
||||
mass = tl.full((1,), 1.0, tl.float32)
|
||||
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
|
||||
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
|
||||
scores = tl.full((1, 16), -1000.0, tl.float32)
|
||||
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
|
||||
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
|
||||
|
||||
|
||||
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
|
||||
"""A masked first tile must not suppress later, strongly negative logits."""
|
||||
probabilities = torch.empty(16, device=DEVICE_TYPE)
|
||||
_softmax_after_empty_tile[(1,)](probabilities)
|
||||
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
|
||||
)
|
||||
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
|
||||
"""A padded reducer row must return before reading sequence or scratch data."""
|
||||
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
|
||||
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
query_starts = torch.tensor(
|
||||
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
).cumsum(0, dtype=torch.int32)
|
||||
reduce_segments[(10, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
len(seq_lens),
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
query_starts,
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
|
||||
torch.testing.assert_close(
|
||||
output[:active_rows], torch.zeros_like(output[:active_rows])
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_lens", "num_heads", "q_dtype"),
|
||||
[
|
||||
*[
|
||||
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
|
||||
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
|
||||
],
|
||||
# Prefill chunk, verification, decode, padding, short query.
|
||||
pytest.param(
|
||||
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
|
||||
),
|
||||
*[
|
||||
pytest.param(
|
||||
[(6, 128), (6, 129), (6, 257)],
|
||||
heads,
|
||||
FP8_DTYPE,
|
||||
id=f"w6-fp8q-kv{heads[1]}",
|
||||
)
|
||||
for heads in [(8, 1), (8, 8)]
|
||||
],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sliding_window", [None, 32])
|
||||
def test_speculative_splitk_matches_reference(
|
||||
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
|
||||
):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=128,
|
||||
sliding_window=sliding_window,
|
||||
dtype=torch.bfloat16,
|
||||
block_size=128,
|
||||
soft_cap=None,
|
||||
num_blocks=8,
|
||||
q_dtype=q_dtype,
|
||||
seq_threshold_3D=32,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
seq_lens=[(6, 128), (6, 129), (6, 4097)],
|
||||
num_heads=(8, 1),
|
||||
head_size=128,
|
||||
block_size=128,
|
||||
num_blocks=64,
|
||||
seq_threshold_3D=16,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_clamped_mm_tile_bounds(
|
||||
output_ptr,
|
||||
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
|
||||
|
||||
|
||||
def _build(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
capability,
|
||||
num_kv_heads,
|
||||
query_lens,
|
||||
max_num_seqs=16,
|
||||
num_speculative_tokens=None,
|
||||
parallel_drafting=False,
|
||||
kv_quant_mode=KVQuantMode.NONE,
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
speculative_config=(
|
||||
None
|
||||
if num_speculative_tokens is None
|
||||
else SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=parallel_drafting,
|
||||
)
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * len(query_lens), query_lens=query_lens)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.softmax_segm_output.shape[2] == metadata.num_par_softmax_segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= sum(query_lens)
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
return builder, metadata
|
||||
|
||||
|
||||
BUILDERS = [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder]
|
||||
|
||||
|
||||
# Verify batches: the 64-segment scratch holds min(11, max_num_seqs) * (1 + K)
|
||||
# rows, and its 16-segment view covers every 3D-eligible sequence. 64 segments
|
||||
# follow the request count, not Q blocks: on SM12.0 they stay faster for verify
|
||||
# batches up to 11 requests (22 Q blocks at K=3).
|
||||
@pytest.mark.parametrize("builder_cls", BUILDERS)
|
||||
@pytest.mark.parametrize(
|
||||
"max_num_seqs,num_speculative_tokens,num_seqs,rows,segments",
|
||||
[
|
||||
(16, 3, 10, 44, 64),
|
||||
(16, 3, 11, 44, 64),
|
||||
(16, 3, 12, 44, 16),
|
||||
(16, 3, 16, 44, 16),
|
||||
(1, 5, 1, 6, 64),
|
||||
],
|
||||
)
|
||||
def test_split_k_segment_rows_cover_verify_queries(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
num_seqs,
|
||||
rows,
|
||||
segments,
|
||||
):
|
||||
builder, metadata = _build(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
(12, 0),
|
||||
1,
|
||||
[1 + num_speculative_tokens] * num_seqs,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
)
|
||||
assert builder.softmax_segm_max.shape[:2] == (rows, 8)
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
|
||||
|
||||
# Without SM12.0 64-segment reuse, the scratch holds exactly
|
||||
# min(seq_threshold_3D, max_num_seqs) * query_len rows for split-K query_lens.
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_tokens,parallel_drafting,max_num_seqs,rows",
|
||||
[
|
||||
(None, False, 8, 8),
|
||||
(4, False, 8, 40),
|
||||
(7, False, 8, 64),
|
||||
(8, False, 8, 64),
|
||||
(3, True, 8, 56),
|
||||
(4, True, 8, 64),
|
||||
(5, False, 1, 6),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_quant_mode", [KVQuantMode.NONE, KVQuantMode.INT4_PER_TOKEN_HEAD]
|
||||
)
|
||||
def test_split_k_scratch_rows_follow_speculative_config(
|
||||
monkeypatch,
|
||||
num_speculative_tokens,
|
||||
parallel_drafting,
|
||||
max_num_seqs,
|
||||
rows,
|
||||
kv_quant_mode,
|
||||
):
|
||||
# The INT4 launcher never runs multi-query split-K.
|
||||
if kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
rows = max_num_seqs
|
||||
builder, _ = _build(
|
||||
monkeypatch,
|
||||
TritonAttentionMetadataBuilder,
|
||||
(9, 0),
|
||||
1,
|
||||
[1],
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
parallel_drafting,
|
||||
kv_quant_mode,
|
||||
)
|
||||
assert builder.softmax_segm_output.shape == (rows, 8, 16, 128)
|
||||
assert builder.softmax_segm_max.shape == (rows, 8, 16)
|
||||
assert builder.softmax_segm_expsum.shape == (rows, 8, 16)
|
||||
@@ -39,7 +39,10 @@ from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash,
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
|
||||
from vllm.v1.attention.ops.triton_unified_attention import (
|
||||
MAX_3D_QUERY_LEN,
|
||||
unified_attention,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVQuantMode,
|
||||
@@ -144,7 +147,10 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
|
||||
)
|
||||
|
||||
self.num_par_softmax_segments = NUM_PAR_SOFTMAX_SEGMENTS
|
||||
# On SM120, batches whose 16-segment grid under-fills the SMs use 64.
|
||||
# On SM120, batches whose 16-segment decode grid under-fills the SMs use
|
||||
# 64. This counts requests, not Q blocks, on purpose: for 1..11 verify
|
||||
# requests (K=3/5, 8 q / 1 KV head, 2k-128k context) 64 segments were
|
||||
# faster or within 6% at 2k, and 1.1-3.6x faster from 8k up.
|
||||
self.max_seqs_64_segments = 0
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability(
|
||||
(12, 0)
|
||||
@@ -152,18 +158,34 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
|
||||
self.max_seqs_64_segments = (current_platform.num_compute_units() - 1) // (
|
||||
NUM_PAR_SOFTMAX_SEGMENTS * self.num_heads_kv
|
||||
)
|
||||
max_num_tokens_3d = self.seq_threshold_3D
|
||||
headdim_padded = next_power_of_2(self.headdim)
|
||||
max_query_len_3d = 1
|
||||
speculative_config = vllm_config.speculative_config
|
||||
# The INT4 launcher runs multi-query batches in 2D only.
|
||||
if (
|
||||
speculative_config is not None
|
||||
and kv_cache_spec.kv_quant_mode != KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
):
|
||||
num_speculative_tokens = speculative_config.num_speculative_tokens or 0
|
||||
query_len = 1 + num_speculative_tokens * (
|
||||
2 if speculative_config.parallel_drafting else 1
|
||||
)
|
||||
max_query_len_3d = min(query_len, MAX_3D_QUERY_LEN)
|
||||
# Scratch is indexed by query token, including verification tokens.
|
||||
max_num_seqs_3d = min(
|
||||
self.seq_threshold_3D, vllm_config.scheduler_config.max_num_seqs
|
||||
)
|
||||
max_num_tokens_3d = max_num_seqs_3d * max_query_len_3d
|
||||
if self.max_seqs_64_segments > 0:
|
||||
self.num_par_softmax_segments = 64
|
||||
# build() reuses this scratch at 16 segments with proportionally more rows.
|
||||
max_num_tokens_3d = max(
|
||||
min(self.max_seqs_64_segments, max_num_tokens_3d),
|
||||
min(self.max_seqs_64_segments, max_num_seqs_3d) * max_query_len_3d,
|
||||
cdiv(
|
||||
max_num_tokens_3d,
|
||||
self.num_par_softmax_segments // NUM_PAR_SOFTMAX_SEGMENTS,
|
||||
),
|
||||
)
|
||||
headdim_padded = next_power_of_2(self.headdim)
|
||||
self.softmax_segm_output = torch.empty(
|
||||
(
|
||||
max_num_tokens_3d,
|
||||
|
||||
@@ -470,15 +470,15 @@ def softmax_step(S, M, L):
|
||||
# compute running maximum
|
||||
# m_j : (BLOCK_M,)
|
||||
m_j = tl.maximum(M, tl.max(S, axis=1))
|
||||
# For sliding window there's a chance the max is -inf due to masking of
|
||||
# the entire row. In this case we need to set m_j 0 to avoid NaN
|
||||
m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
|
||||
# Keep an empty row's maximum at -inf for later tiles; use a finite shift
|
||||
# only for the exponent arithmetic below.
|
||||
safe_m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
|
||||
# P : (BLOCK_M, TILE_SIZE)
|
||||
P = tl.exp(S - m_j[:, None])
|
||||
P = tl.exp(S - safe_m_j[:, None])
|
||||
# l_j : (BLOCK_M,)
|
||||
l_j = tl.sum(P, axis=1)
|
||||
# alpha : (BLOCK_M, )
|
||||
alpha = tl.exp(M - m_j)
|
||||
alpha = tl.exp(M - safe_m_j)
|
||||
# update constants
|
||||
L_new = L * alpha + l_j
|
||||
return m_j, L_new, P, alpha
|
||||
|
||||
@@ -33,6 +33,8 @@ from vllm.v1.kv_cache_interface import KVQuantMode
|
||||
logger = init_logger(__name__)
|
||||
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
float8_info = torch.finfo(current_platform.fp8_dtype())
|
||||
# Longest per-request query (e.g. 1 + num_speculative_tokens) run by split-K.
|
||||
MAX_3D_QUERY_LEN = 8
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -721,12 +723,18 @@ def reduce_segments(
|
||||
query_token_idx = tl.program_id(0)
|
||||
query_head_idx = tl.program_id(1)
|
||||
|
||||
# Graph padding must be rejected before sequence lookup or scratch access.
|
||||
if query_token_idx >= tl.load(query_start_len_ptr + num_seqs):
|
||||
return
|
||||
|
||||
seq_idx = find_seq_idx(
|
||||
query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False
|
||||
)
|
||||
|
||||
# sequence len for this particular sequence
|
||||
seq_len = tl.load(seq_lens_ptr + seq_idx)
|
||||
if seq_len <= 0:
|
||||
return
|
||||
|
||||
# number of segments for this particular sequence
|
||||
num_segments = NUM_SEGMENTS_PER_SEQ
|
||||
@@ -746,11 +754,16 @@ def reduce_segments(
|
||||
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)
|
||||
)
|
||||
segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf"))
|
||||
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
|
||||
# Empty segments must not set the maximum. Unvisited segments retain
|
||||
# the M=-inf,L=1 initial state.
|
||||
valid_segment = segm_mask & (segm_expsum > 0) & (segm_max != float("-inf"))
|
||||
segm_max = tl.where(valid_segment, segm_max, float("-inf"))
|
||||
overall_max = tl.max(segm_max)
|
||||
|
||||
# load and rescale segment exp sums
|
||||
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
|
||||
segm_expsum = segm_expsum * tl.exp(segm_max - overall_max)
|
||||
segment_scale = tl.where(valid_segment, tl.exp(segm_max - overall_max), 0.0)
|
||||
segm_expsum = segm_expsum * segment_scale
|
||||
overall_expsum = tl.sum(segm_expsum)
|
||||
|
||||
# load, rescale, and add segment attention outputs
|
||||
@@ -766,7 +779,7 @@ def reduce_segments(
|
||||
mask=segm_mask[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
segm_output *= tl.exp(segm_max - overall_max)[:, None]
|
||||
segm_output *= segment_scale[:, None]
|
||||
acc_sum = tl.sum(segm_output, axis=0)
|
||||
# safely divide by overall_expsum, returning 0.0 if overall_expsum is 0
|
||||
acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum)
|
||||
@@ -1047,9 +1060,12 @@ def unified_attention(
|
||||
)
|
||||
|
||||
# Launch the 2D kernel if
|
||||
# 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or
|
||||
# 2. The batch includes at least one prefill request, or
|
||||
# 3. The number of sequences exceeds the configured threshold, or
|
||||
# 1. No split-K threshold or scratch buffers, or the scratch has fewer
|
||||
# rows than query tokens, or
|
||||
# 2. max_seqlen_q > 1 and either it exceeds MAX_3D_QUERY_LEN, masking is
|
||||
# non-causal, per-sequence causal, or mm-prefix, or the tuned
|
||||
# large-head path applies, or
|
||||
# 3. The number of Q blocks exceeds seq_threshold_3D, or
|
||||
# 4. Batch invariance is enabled
|
||||
use_3d = not (
|
||||
seq_threshold_3D is None
|
||||
@@ -1057,10 +1073,22 @@ def unified_attention(
|
||||
or softmax_segm_output is None
|
||||
or softmax_segm_max is None
|
||||
or softmax_segm_expsum is None
|
||||
or max_seqlen_q > 1
|
||||
or num_seqs > seq_threshold_3D
|
||||
or (
|
||||
max_seqlen_q > 1
|
||||
and (
|
||||
max_seqlen_q > MAX_3D_QUERY_LEN
|
||||
or not use_causal
|
||||
or use_per_seq_causal
|
||||
or use_mm_prefix
|
||||
or tuned_large_head
|
||||
)
|
||||
)
|
||||
or num_seqs * triton.cdiv(max_seqlen_q, BLOCK_Q) > seq_threshold_3D
|
||||
or is_batch_invariant
|
||||
)
|
||||
if use_3d and softmax_segm_max.shape[0] < q.shape[0]:
|
||||
logger.debug_once("Split-K scratch rows < query tokens; using 2D.")
|
||||
use_3d = False
|
||||
|
||||
# The kernel signature is the same for 2D and 3D — only the launch
|
||||
# grid + a handful of constexpr toggles differ. Per-token-head scale
|
||||
|
||||
Reference in New Issue
Block a user