[Perf][Attention] Bound FlashInfer prefill dequantization scratch (#57918)

Signed-off-by: Yueh-Ting Chen <yueh.ting.chen@gmail.com>
This commit is contained in:
Yueh-Ting (eop) Chen
2026-09-24 10:42:08 -04:00
committed by GitHub
parent 3c42385d7d
commit 26f49e336a
2 changed files with 94 additions and 2 deletions
@@ -9,6 +9,7 @@ import pytest
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
from tests.kernels.attention.test_trtllm_kvfp8_dequant import make_random_kv_cache
from tests.v1.attention.test_attention_backends import create_and_prepopulate_kv_cache
from tests.v1.attention.utils import (
BatchSpec,
@@ -17,7 +18,11 @@ from tests.v1.attention.utils import (
)
from vllm.config import set_current_vllm_config
from vllm.platforms import current_platform
from vllm.utils.torch_utils import nvfp4_kv_cache_full_dim, set_random_seed
from vllm.utils.torch_utils import (
is_strictly_contiguous,
nvfp4_kv_cache_full_dim,
set_random_seed,
)
from vllm.v1.attention.backends.utils import PerLayerParameters
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
@@ -33,6 +38,7 @@ from vllm.v1.attention.backends.flashinfer import ( # noqa: E402
FlashInferMetadataBuilder,
FlashInferTrtllmAPIDecode,
TRTLLMPrefill,
trtllm_prefill_attn_kvfp8_dequant,
)
@@ -83,6 +89,70 @@ def _mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
}
@pytest.mark.parametrize(
"block_size,num_prefills,has_host_lengths,quantize_query",
[
(16, 1, True, False),
(64, 2, True, False),
(128, 2, True, False),
(16, 2, False, False),
(16, 2, True, True),
],
)
@torch.inference_mode()
def test_prefill_dequant_scratch_ignores_unused_capacity(
block_size, num_prefills, has_host_lengths, quantize_query, monkeypatch
):
"""Bound scratch by attended prefill KV, including cached prefixes."""
device = torch.device("cuda")
config = create_vllm_config(model_name=MODEL, max_model_len=32768)
config.cache_config.cache_dtype = "fp8"
config.cache_config.kv_cache_layout = "LBHNC"
config.attention_config.use_trtllm_attention = True
config.attention_config.disable_flashinfer_q_quantization = not quantize_query
monkeypatch.setattr(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
_mock_get_per_layer_parameters,
)
spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=64,
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
)
kv_cache, scale = make_random_kv_cache(32, 2, block_size, 64)
batch = BatchSpec(
seq_lens=[1024] + [17, 65][-num_prefills:],
query_lens=[1] + [2, 8][-num_prefills:],
)
common = create_common_attn_metadata(batch, block_size, device)
if not has_host_lengths:
common.seq_lens_cpu_upper_bound = None
live_pages = ((65 if has_host_lengths else 1024) + block_size - 1) // block_size
with set_current_vllm_config(config):
builder = FlashInferMetadataBuilder(spec, ["layer.0"], config, device)
for capacity in (64, 256):
# Unused columns can contain nonzero page IDs.
common.block_table_tensor = torch.randint(
1, 32, (batch.batch_size, capacity), dtype=torch.int32, device=device
)
metadata = builder.build(0, common)
assert isinstance(metadata.prefill, TRTLLMPrefill)
table = metadata.prefill.block_tables
assert is_strictly_contiguous(table)
assert (metadata.q_data_type_prefill == current_platform.fp8_dtype()) == (
quantize_query
)
if not quantize_query:
converted, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache, table, scale, scale, metadata.q_data_type_prefill
)
assert converted.shape[0] == num_prefills * live_pages + 1
width = capacity if quantize_query else live_pages
torch.testing.assert_close(table, common.block_table_tensor[1:, :width])
def _create_nvfp4_hnd_kv_cache(
k_contexts,
v_contexts,
+23 -1
View File
@@ -1563,8 +1563,30 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
qo_indptr_prefill_cpu[1:] - qo_indptr_prefill_cpu[:-1]
)
max_q_len_prefill = int(query_lens_prefill_cpu.max().item())
prefill_block_tables = block_table_tensor[prefill_start:]
if (
self.q_data_type_prefill != FP8_DTYPE
and self.cache_dtype.startswith("fp8")
):
seq_lens_cpu_upper_bound = (
common_attn_metadata.seq_lens_cpu_upper_bound
)
max_prefill_seq_len = max_seq_len
if seq_lens_cpu_upper_bound is not None:
max_prefill_seq_len = int(
seq_lens_cpu_upper_bound[prefill_start:num_reqs]
.max()
.item()
)
# Dequantization allocates one page per table entry and
# indexes rows by their logical width.
prefill_block_tables = canonicalize_singleton_dim_strides(
prefill_block_tables[
:, : cdiv(max_prefill_seq_len, page_size)
].contiguous()
)
attn_metadata.prefill = TRTLLMPrefill(
block_tables=block_table_tensor[prefill_start:],
block_tables=prefill_block_tables,
seq_lens=prefill_seq_lens,
cum_seq_lens_q=qo_indptr_prefill_gpu,
cum_seq_lens_kv=paged_kv_indptr_prefill_gpu,