diff --git a/tests/v1/attention/test_trtllm_attention_integration.py b/tests/v1/attention/test_trtllm_attention_integration.py index 1a49a3895c..42dd8f9f48 100644 --- a/tests/v1/attention/test_trtllm_attention_integration.py +++ b/tests/v1/attention/test_trtllm_attention_integration.py @@ -9,6 +9,7 @@ import pytest import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention +from tests.kernels.attention.test_trtllm_kvfp8_dequant import make_random_kv_cache from tests.v1.attention.test_attention_backends import create_and_prepopulate_kv_cache from tests.v1.attention.utils import ( BatchSpec, @@ -17,7 +18,11 @@ from tests.v1.attention.utils import ( ) from vllm.config import set_current_vllm_config from vllm.platforms import current_platform -from vllm.utils.torch_utils import nvfp4_kv_cache_full_dim, set_random_seed +from vllm.utils.torch_utils import ( + is_strictly_contiguous, + nvfp4_kv_cache_full_dim, + set_random_seed, +) from vllm.v1.attention.backends.utils import PerLayerParameters from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode @@ -33,6 +38,7 @@ from vllm.v1.attention.backends.flashinfer import ( # noqa: E402 FlashInferMetadataBuilder, FlashInferTrtllmAPIDecode, TRTLLMPrefill, + trtllm_prefill_attn_kvfp8_dequant, ) @@ -83,6 +89,70 @@ def _mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): } +@pytest.mark.parametrize( + "block_size,num_prefills,has_host_lengths,quantize_query", + [ + (16, 1, True, False), + (64, 2, True, False), + (128, 2, True, False), + (16, 2, False, False), + (16, 2, True, True), + ], +) +@torch.inference_mode() +def test_prefill_dequant_scratch_ignores_unused_capacity( + block_size, num_prefills, has_host_lengths, quantize_query, monkeypatch +): + """Bound scratch by attended prefill KV, including cached prefixes.""" + device = torch.device("cuda") + config = create_vllm_config(model_name=MODEL, max_model_len=32768) + config.cache_config.cache_dtype = "fp8" + config.cache_config.kv_cache_layout = "LBHNC" + config.attention_config.use_trtllm_attention = True + config.attention_config.disable_flashinfer_q_quantization = not quantize_query + monkeypatch.setattr( + "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", + _mock_get_per_layer_parameters, + ) + spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=64, + dtype=torch.uint8, + kv_quant_mode=KVQuantMode.FP8_PER_TENSOR, + ) + kv_cache, scale = make_random_kv_cache(32, 2, block_size, 64) + batch = BatchSpec( + seq_lens=[1024] + [17, 65][-num_prefills:], + query_lens=[1] + [2, 8][-num_prefills:], + ) + common = create_common_attn_metadata(batch, block_size, device) + if not has_host_lengths: + common.seq_lens_cpu_upper_bound = None + live_pages = ((65 if has_host_lengths else 1024) + block_size - 1) // block_size + with set_current_vllm_config(config): + builder = FlashInferMetadataBuilder(spec, ["layer.0"], config, device) + for capacity in (64, 256): + # Unused columns can contain nonzero page IDs. + common.block_table_tensor = torch.randint( + 1, 32, (batch.batch_size, capacity), dtype=torch.int32, device=device + ) + metadata = builder.build(0, common) + assert isinstance(metadata.prefill, TRTLLMPrefill) + table = metadata.prefill.block_tables + assert is_strictly_contiguous(table) + assert (metadata.q_data_type_prefill == current_platform.fp8_dtype()) == ( + quantize_query + ) + if not quantize_query: + converted, _ = trtllm_prefill_attn_kvfp8_dequant( + kv_cache, table, scale, scale, metadata.q_data_type_prefill + ) + assert converted.shape[0] == num_prefills * live_pages + 1 + width = capacity if quantize_query else live_pages + torch.testing.assert_close(table, common.block_table_tensor[1:, :width]) + + def _create_nvfp4_hnd_kv_cache( k_contexts, v_contexts, diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 2cfaef8166..08920fa3ca 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1563,8 +1563,30 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): qo_indptr_prefill_cpu[1:] - qo_indptr_prefill_cpu[:-1] ) max_q_len_prefill = int(query_lens_prefill_cpu.max().item()) + prefill_block_tables = block_table_tensor[prefill_start:] + if ( + self.q_data_type_prefill != FP8_DTYPE + and self.cache_dtype.startswith("fp8") + ): + seq_lens_cpu_upper_bound = ( + common_attn_metadata.seq_lens_cpu_upper_bound + ) + max_prefill_seq_len = max_seq_len + if seq_lens_cpu_upper_bound is not None: + max_prefill_seq_len = int( + seq_lens_cpu_upper_bound[prefill_start:num_reqs] + .max() + .item() + ) + # Dequantization allocates one page per table entry and + # indexes rows by their logical width. + prefill_block_tables = canonicalize_singleton_dim_strides( + prefill_block_tables[ + :, : cdiv(max_prefill_seq_len, page_size) + ].contiguous() + ) attn_metadata.prefill = TRTLLMPrefill( - block_tables=block_table_tensor[prefill_start:], + block_tables=prefill_block_tables, seq_lens=prefill_seq_lens, cum_seq_lens_q=qo_indptr_prefill_gpu, cum_seq_lens_kv=paged_kv_indptr_prefill_gpu,