[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
@@ -5,9 +5,11 @@
from dataclasses import dataclass
from unittest.mock import patch
import numpy as np
import pytest
import torch
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.pooler.seqwise.methods import (
CLSPool,
LastPool,
@@ -52,6 +54,8 @@ def _make_pooling_cursor(
prompt_lens_cpu=prompt_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
)
@@ -101,6 +105,38 @@ def _make_metadata(
)
def test_build_pooling_cursor_caches_cpu_status() -> None:
metadata = PoolingMetadata(
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
pooling_states=[PoolingStates() for _ in range(3)],
)
metadata.build_pooling_cursor(
np.array([3, 2, 2], dtype=np.int32),
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
device=_CPU,
)
cursor = metadata.get_pooling_cursor()
assert cursor.is_partial_prefill() is True
assert cursor.get_finished_mask() == [True, False, False]
assert cursor[:1].is_partial_prefill() is False
assert cursor[1:2].is_partial_prefill() is True
assert cursor[1:].get_finished_mask() == [False, False]
def test_pooling_params_update_combines_token_id_requirements() -> None:
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
params = PoolingParams(task="embed")
update.apply(params)
assert params.requires_token_ids is True
# ---------------------------------------------------------------------------
# CLSPool
# ---------------------------------------------------------------------------
@@ -12,9 +12,7 @@ from vllm.model_executor.models.bert import (
BertMLMHead,
SPLADESparsePooler,
)
from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -119,18 +117,17 @@ def test_pooling_runner_gathers_required_token_ids() -> None:
req_states = MagicMock(spec=RequestState)
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
metadata = runner._get_pooling_metadata(
input_batch, req_states, torch.device(current_platform.device_type)
input_batch, req_states, torch.device("cpu")
)
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
assert metadata.prompt_token_ids_cpu is not None
assert metadata.prompt_token_ids is not None
assert metadata.prompt_token_ids_cpu.is_pinned() == PIN_MEMORY
assert metadata.prompt_token_ids is None
assert metadata.prompt_token_ids_cpu.is_pinned() is False
torch.testing.assert_close(
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
)
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
torch.testing.assert_close(metadata.prompt_token_ids.cpu(), expected)
def test_pooling_runner_stores_only_required_token_ids() -> None:
+5 -11
View File
@@ -475,15 +475,14 @@ def test_placeholder_spec_token_ids_written_verbatim():
@pytest.mark.parametrize(
("pooling_params", "expect_device_prompt_token_ids", "expect_cpu_prompt_token_ids"),
("pooling_params", "expect_cpu_prompt_token_ids"),
[
({"task": "classify"}, False, False),
({"task": "classify", "requires_token_ids": True}, True, True),
({"task": "classify"}, False),
({"task": "classify", "requires_token_ids": True}, True),
],
)
def test_pooling_metadata_token_id_buffers(
pooling_params: dict[str, object],
expect_device_prompt_token_ids: bool,
expect_cpu_prompt_token_ids: bool,
):
from vllm.pooling_params import PoolingParams
@@ -504,13 +503,8 @@ def test_pooling_metadata_token_id_buffers(
input_batch.refresh_metadata()
metadata = input_batch.get_pooling_metadata()
if expect_device_prompt_token_ids:
assert input_batch.sampling_metadata.prompt_token_ids is not None
assert metadata.prompt_token_ids is not None
assert metadata.get_prompt_token_ids()[0].tolist() == req.prompt_token_ids
else:
assert input_batch.sampling_metadata.prompt_token_ids is None
assert metadata.prompt_token_ids is None
assert input_batch.sampling_metadata.prompt_token_ids is None
assert metadata.prompt_token_ids is None
if expect_cpu_prompt_token_ids:
assert metadata.prompt_token_ids_cpu is not None