[Perf][Pooling] Avoid blocking seq_lens GPU-to-CPU copy for pooling in FlashInfer metadata builder (#57214)

Signed-off-by: frankwang28 <frank.wbb@hotmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
Frank Wang
2026-09-27 16:06:38 +00:00
committed by GitHub
co-authored by Misha Goin mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Vadim Gimpelson
parent 44af287ebe
commit 0376f81530
2 changed files with 97 additions and 3 deletions
+87 -1
View File
@@ -19,7 +19,12 @@ from tests.v1.attention.utils import (
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
from vllm.config import ModelConfig, set_current_vllm_config
from vllm.config import (
DiffusionConfig,
ModelConfig,
SpeculativeConfig,
set_current_vllm_config,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
@@ -1069,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
not current_platform.is_cuda()
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer with CUDA is required.",
)
@pytest.mark.parametrize(
"runner_type,spec_kind,has_upper_bound,expected_copies",
[
("generate", None, True, 0),
("pooling", None, True, 0),
("generate", None, False, 1),
("generate", "spec_decode", True, 1),
("generate", "diffusion", True, 1),
],
)
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
):
"""Planning reuses the host seq_lens bound only when it is exact.
Without speculative token accounting (no spec-decode drafts, no dLLM
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
is exact regardless of runner type. Any speculative accounting keeps the
bound optimistic (rejected tokens are in-flight) and forces the copy.
"""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
monkeypatch.setattr(
flashinfer_backend,
"get_per_layer_parameters",
lambda *args: {
"layer.0": PerLayerParameters(
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
)
},
)
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
)
config.model_config.runner_type = runner_type
config.attention_config.use_trtllm_attention = False
if spec_kind == "spec_decode":
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
elif spec_kind == "diffusion":
config.diffusion_config = DiffusionConfig(canvas_length=8)
device = torch.device(f"{DEVICE_TYPE}:0")
# Include a one-token final chunk and both partial and full KV pages.
common_attn_metadata = create_common_attn_metadata(
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
)
if not has_upper_bound:
common_attn_metadata.seq_lens_cpu_upper_bound = None
elif spec_kind is not None:
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
with set_current_vllm_config(config):
builder = flashinfer_backend.FlashInferMetadataBuilder(
create_standard_kv_cache_spec(config), ["layer.0"], config, device
)
with unittest.mock.patch.object(
common_attn_metadata.seq_lens,
"cpu",
wraps=common_attn_metadata.seq_lens.cpu,
) as copy_to_cpu:
metadata = builder.build(0, common_attn_metadata)
assert copy_to_cpu.call_count == expected_copies
assert metadata.num_decodes == 1
assert metadata.num_prefills == 2
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
+10 -2
View File
@@ -700,6 +700,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.cache_config = vllm_config.cache_config
self.model_config = vllm_config.model_config
self.attention_config = vllm_config.attention_config
self._num_speculative_tokens = vllm_config.num_speculative_tokens
self._workspace_buffer = None
self._prefill_wrapper: (
BatchPrefillWithPagedKVCacheWrapper | BatchDCPPrefillWrapper | None
@@ -1446,8 +1447,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# (block_tables, seq_lens) directly.
needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm
if needs_seq_lens_cpu:
with gpu_sync_allowed():
seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
if (
self._num_speculative_tokens == 0
and common_attn_metadata.seq_lens_cpu_upper_bound is not None
):
# No speculative token accounting, so the upper bound is exact.
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
else:
with gpu_sync_allowed():
seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
else:
seq_lens_cpu = None