[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -5,9 +5,11 @@
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
CLSPool,
|
||||
LastPool,
|
||||
@@ -52,6 +54,8 @@ def _make_pooling_cursor(
|
||||
prompt_lens_cpu=prompt_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
|
||||
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
|
||||
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
|
||||
)
|
||||
|
||||
|
||||
@@ -101,6 +105,38 @@ def _make_metadata(
|
||||
)
|
||||
|
||||
|
||||
def test_build_pooling_cursor_caches_cpu_status() -> None:
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=None,
|
||||
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
|
||||
pooling_states=[PoolingStates() for _ in range(3)],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
np.array([3, 2, 2], dtype=np.int32),
|
||||
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
|
||||
device=_CPU,
|
||||
)
|
||||
|
||||
cursor = metadata.get_pooling_cursor()
|
||||
|
||||
assert cursor.is_partial_prefill() is True
|
||||
assert cursor.get_finished_mask() == [True, False, False]
|
||||
assert cursor[:1].is_partial_prefill() is False
|
||||
assert cursor[1:2].is_partial_prefill() is True
|
||||
assert cursor[1:].get_finished_mask() == [False, False]
|
||||
|
||||
|
||||
def test_pooling_params_update_combines_token_id_requirements() -> None:
|
||||
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
|
||||
params = PoolingParams(task="embed")
|
||||
|
||||
update.apply(params)
|
||||
|
||||
assert params.requires_token_ids is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLSPool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -12,9 +12,7 @@ from vllm.model_executor.models.bert import (
|
||||
BertMLMHead,
|
||||
SPLADESparsePooler,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
@@ -119,18 +117,17 @@ def test_pooling_runner_gathers_required_token_ids() -> None:
|
||||
req_states = MagicMock(spec=RequestState)
|
||||
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
|
||||
metadata = runner._get_pooling_metadata(
|
||||
input_batch, req_states, torch.device(current_platform.device_type)
|
||||
input_batch, req_states, torch.device("cpu")
|
||||
)
|
||||
|
||||
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
|
||||
assert metadata.prompt_token_ids_cpu is not None
|
||||
assert metadata.prompt_token_ids is not None
|
||||
assert metadata.prompt_token_ids_cpu.is_pinned() == PIN_MEMORY
|
||||
assert metadata.prompt_token_ids is None
|
||||
assert metadata.prompt_token_ids_cpu.is_pinned() is False
|
||||
torch.testing.assert_close(
|
||||
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids.cpu(), expected)
|
||||
|
||||
|
||||
def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
|
||||
@@ -475,15 +475,14 @@ def test_placeholder_spec_token_ids_written_verbatim():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pooling_params", "expect_device_prompt_token_ids", "expect_cpu_prompt_token_ids"),
|
||||
("pooling_params", "expect_cpu_prompt_token_ids"),
|
||||
[
|
||||
({"task": "classify"}, False, False),
|
||||
({"task": "classify", "requires_token_ids": True}, True, True),
|
||||
({"task": "classify"}, False),
|
||||
({"task": "classify", "requires_token_ids": True}, True),
|
||||
],
|
||||
)
|
||||
def test_pooling_metadata_token_id_buffers(
|
||||
pooling_params: dict[str, object],
|
||||
expect_device_prompt_token_ids: bool,
|
||||
expect_cpu_prompt_token_ids: bool,
|
||||
):
|
||||
from vllm.pooling_params import PoolingParams
|
||||
@@ -504,13 +503,8 @@ def test_pooling_metadata_token_id_buffers(
|
||||
input_batch.refresh_metadata()
|
||||
|
||||
metadata = input_batch.get_pooling_metadata()
|
||||
if expect_device_prompt_token_ids:
|
||||
assert input_batch.sampling_metadata.prompt_token_ids is not None
|
||||
assert metadata.prompt_token_ids is not None
|
||||
assert metadata.get_prompt_token_ids()[0].tolist() == req.prompt_token_ids
|
||||
else:
|
||||
assert input_batch.sampling_metadata.prompt_token_ids is None
|
||||
assert metadata.prompt_token_ids is None
|
||||
assert input_batch.sampling_metadata.prompt_token_ids is None
|
||||
assert metadata.prompt_token_ids is None
|
||||
|
||||
if expect_cpu_prompt_token_ids:
|
||||
assert metadata.prompt_token_ids_cpu is not None
|
||||
|
||||
Reference in New Issue
Block a user