[Dependency] Upgrade FlashInfer version to 0.7.0 (#58069)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Yongye Zhu
Kevin H. Luu
parent
a1b6763df6
commit
44bf011f44
@@ -509,3 +509,59 @@ def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
|
||||
@pytest.mark.parametrize("page_padding", [0, 64])
|
||||
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
|
||||
"""Decode and prefill must read storage rows after vLLM remaps packed pages."""
|
||||
import importlib
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse")
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
storage = torch.full(
|
||||
(3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache = storage[:, :block_size]
|
||||
cache[1].fill_(2)
|
||||
cache[2].fill_(3)
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.bfloat16
|
||||
attn.attn_sink = torch.full((64,), -float("inf"), device=device)
|
||||
attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device)
|
||||
decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device)
|
||||
decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device)
|
||||
query_start = torch.tensor([0, 1, 3], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor([67, 130, 131], device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(3, dtype=torch.bool, device=device),
|
||||
replay_start=torch.zeros(2, dtype=torch.int32, device=device),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=2,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device)
|
||||
output = torch.empty_like(query)
|
||||
attn._forward(query, None, cache, metadata, None, True, output)
|
||||
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
|
||||
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def skip_unsupported_flashinfer_sampler():
|
||||
# CI runs this suite with FlashInfer both explicitly enabled and disabled.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and envs.is_set("VLLM_USE_FLASHINFER_SAMPLER")
|
||||
and envs.VLLM_USE_FLASHINFER_SAMPLER
|
||||
and current_platform.num_compute_units(torch.accelerator.current_device_index())
|
||||
<= 16
|
||||
):
|
||||
pytest.skip("FlashInfer top-k masking requires more than 16 SMs")
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
|
||||
"""Tests for the FlashInfer SM90 sparse MLA backend wiring and index packing.
|
||||
|
||||
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
|
||||
the tests pin the contract between the impl and the kernel API: page_size=1
|
||||
@@ -65,12 +65,18 @@ class FakeState:
|
||||
def __init__(self, width, max_tokens=64):
|
||||
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
|
||||
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
|
||||
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
|
||||
self.wrapper = FakeWrapper()
|
||||
self.plan_calls = []
|
||||
|
||||
def plan(self, num_tokens, kv_lens):
|
||||
self.plan_calls.append((num_tokens, kv_lens))
|
||||
|
||||
def pack_indices(self, slots):
|
||||
for row in range(slots.shape[0]):
|
||||
start, end = self.kv_indptr[row : row + 2].tolist()
|
||||
self.kv_indices[start:end] = slots[row, : end - start].clamp(min=0)
|
||||
|
||||
|
||||
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
|
||||
impl = object.__new__(FlashInferMLASparseSM90Impl)
|
||||
@@ -113,6 +119,9 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
[6, 5, 4] + [-1] * (TOPK - 3),
|
||||
[2] + [-1] * (TOPK - 1),
|
||||
]
|
||||
impl.topk_indices_buffer.copy_(torch.tensor(topk_rows, dtype=torch.int32))
|
||||
state.kv_indptr[1:5] = torch.tensor([8, 13, 16, 17], dtype=torch.int32)
|
||||
state.kv_indptr[5:] = 17
|
||||
meta = make_batch(rows, topk_rows, [3])
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
@@ -133,11 +142,13 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
)
|
||||
width = TOPK
|
||||
got_slots = state.kv_indices[: rows * width].view(rows, width)
|
||||
offset = 0
|
||||
for t in range(rows):
|
||||
k = int(ref_counts[t])
|
||||
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
|
||||
assert (
|
||||
state.kv_indices[offset : offset + k].tolist() == ref_slots[t, :k].tolist()
|
||||
)
|
||||
offset += k
|
||||
assert state.plan_calls == []
|
||||
|
||||
assert state.wrapper.run_args is not None
|
||||
@@ -221,13 +232,51 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
args, kwargs = wrapper.plan_args
|
||||
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
|
||||
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
|
||||
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
|
||||
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
|
||||
assert kv.tolist() == [0, 2, 7, 14, 14]
|
||||
assert kv_len.tolist() == [2, 5, 7, 0]
|
||||
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
|
||||
assert scale == 576**-0.5
|
||||
assert kwargs["q_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
|
||||
# Replanning a smaller batch must clear the previous rows' lengths.
|
||||
state.plan(1, torch.tensor([TOPK], dtype=torch.int32))
|
||||
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
|
||||
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
|
||||
state.plan(0, torch.empty(0, dtype=torch.int32))
|
||||
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
|
||||
assert state._lens_cpu.tolist() == [0, 0, 0, 0]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize("width", [128, 2048, 2176])
|
||||
def test_pack_indices_replays_with_updated_offsets(width):
|
||||
"""Graph replay packs exact prefixes after row lengths and slots change."""
|
||||
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
|
||||
state.kv_indptr = torch.zeros(5, dtype=torch.int32, device="cuda")
|
||||
state.kv_indices = torch.full((4 * width,), -99, dtype=torch.int32, device="cuda")
|
||||
# Exercise a contiguous view with an unaligned starting address.
|
||||
slots = torch.arange(4 * width + 1, dtype=torch.int32, device="cuda")[1:]
|
||||
slots = slots.view(4, width)
|
||||
state.kv_indptr.copy_(torch.tensor([0, 1, 4, 4, 4]))
|
||||
state.pack_indices(slots)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
state.pack_indices(slots)
|
||||
|
||||
for lengths in ([1, 3, 0, 0], [width, 0, 17, width - 1], [0, 0, 0, 0]):
|
||||
offsets = torch.tensor([0, *torch.tensor(lengths).cumsum(0).tolist()])
|
||||
state.kv_indptr.copy_(offsets)
|
||||
state.kv_indices.fill_(-99)
|
||||
slots.add_(1)
|
||||
graph.replay()
|
||||
expected = torch.cat(
|
||||
[slots[row, :length] for row, length in enumerate(lengths)]
|
||||
)
|
||||
torch.testing.assert_close(state.kv_indices[: expected.numel()], expected)
|
||||
assert (state.kv_indices[expected.numel() :] == -99).all()
|
||||
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
|
||||
@@ -25,7 +25,7 @@ def _flashinfer_topk_topp_supported() -> bool:
|
||||
|
||||
Mirrors the gate in `TopKTopPSampler.__init__`: CUDA + flashinfer
|
||||
importable + GPU compute capability supported by the FlashInfer
|
||||
backend.
|
||||
backend + more than 16 SMs.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
@@ -38,7 +38,10 @@ def _flashinfer_topk_topp_supported() -> bool:
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
return False
|
||||
return FlashInferBackend.supports_compute_capability(capability)
|
||||
return FlashInferBackend.supports_compute_capability(capability) and (
|
||||
current_platform.num_compute_units(torch.accelerator.current_device_index())
|
||||
> 16
|
||||
)
|
||||
|
||||
|
||||
FLASHINFER_TOPK_TOPP_SUPPORTED = _flashinfer_topk_topp_supported()
|
||||
|
||||
Reference in New Issue
Block a user