[Perf][Pooling] Avoid blocking seq_lens GPU-to-CPU copy for pooling in FlashInfer metadata builder (#57214)
Signed-off-by: frankwang28 <frank.wbb@hotmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
Misha Goin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Vadim Gimpelson
parent
44af287ebe
commit
0376f81530
@@ -19,7 +19,12 @@ from tests.v1.attention.utils import (
|
||||
try_backend_includes_kv_cache_update,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.config import ModelConfig, set_current_vllm_config
|
||||
from vllm.config import (
|
||||
DiffusionConfig,
|
||||
ModelConfig,
|
||||
SpeculativeConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -1069,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer with CUDA is required.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"runner_type,spec_kind,has_upper_bound,expected_copies",
|
||||
[
|
||||
("generate", None, True, 0),
|
||||
("pooling", None, True, 0),
|
||||
("generate", None, False, 1),
|
||||
("generate", "spec_decode", True, 1),
|
||||
("generate", "diffusion", True, 1),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
|
||||
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
|
||||
):
|
||||
"""Planning reuses the host seq_lens bound only when it is exact.
|
||||
|
||||
Without speculative token accounting (no spec-decode drafts, no dLLM
|
||||
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
|
||||
is exact regardless of runner type. Any speculative accounting keeps the
|
||||
bound optimistic (rejected tokens are in-flight) and forces the copy.
|
||||
"""
|
||||
import unittest.mock
|
||||
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend,
|
||||
"get_per_layer_parameters",
|
||||
lambda *args: {
|
||||
"layer.0": PerLayerParameters(
|
||||
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
|
||||
)
|
||||
config.model_config.runner_type = runner_type
|
||||
config.attention_config.use_trtllm_attention = False
|
||||
if spec_kind == "spec_decode":
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=3
|
||||
)
|
||||
elif spec_kind == "diffusion":
|
||||
config.diffusion_config = DiffusionConfig(canvas_length=8)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
# Include a one-token final chunk and both partial and full KV pages.
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
|
||||
)
|
||||
if not has_upper_bound:
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound = None
|
||||
elif spec_kind is not None:
|
||||
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
|
||||
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
||||
create_standard_kv_cache_spec(config), ["layer.0"], config, device
|
||||
)
|
||||
with unittest.mock.patch.object(
|
||||
common_attn_metadata.seq_lens,
|
||||
"cpu",
|
||||
wraps=common_attn_metadata.seq_lens.cpu,
|
||||
) as copy_to_cpu:
|
||||
metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
assert copy_to_cpu.call_count == expected_copies
|
||||
assert metadata.num_decodes == 1
|
||||
assert metadata.num_prefills == 2
|
||||
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
|
||||
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
|
||||
@@ -700,6 +700,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.attention_config = vllm_config.attention_config
|
||||
self._num_speculative_tokens = vllm_config.num_speculative_tokens
|
||||
self._workspace_buffer = None
|
||||
self._prefill_wrapper: (
|
||||
BatchPrefillWithPagedKVCacheWrapper | BatchDCPPrefillWrapper | None
|
||||
@@ -1446,8 +1447,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# (block_tables, seq_lens) directly.
|
||||
needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm
|
||||
if needs_seq_lens_cpu:
|
||||
with gpu_sync_allowed():
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
|
||||
if (
|
||||
self._num_speculative_tokens == 0
|
||||
and common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
):
|
||||
# No speculative token accounting, so the upper bound is exact.
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
else:
|
||||
with gpu_sync_allowed():
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
|
||||
else:
|
||||
seq_lens_cpu = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user