[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
@@ -5,9 +5,11 @@
from dataclasses import dataclass
from unittest.mock import patch
import numpy as np
import pytest
import torch
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.pooler.seqwise.methods import (
CLSPool,
LastPool,
@@ -52,6 +54,8 @@ def _make_pooling_cursor(
prompt_lens_cpu=prompt_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
)
@@ -101,6 +105,38 @@ def _make_metadata(
)
def test_build_pooling_cursor_caches_cpu_status() -> None:
metadata = PoolingMetadata(
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
pooling_states=[PoolingStates() for _ in range(3)],
)
metadata.build_pooling_cursor(
np.array([3, 2, 2], dtype=np.int32),
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
device=_CPU,
)
cursor = metadata.get_pooling_cursor()
assert cursor.is_partial_prefill() is True
assert cursor.get_finished_mask() == [True, False, False]
assert cursor[:1].is_partial_prefill() is False
assert cursor[1:2].is_partial_prefill() is True
assert cursor[1:].get_finished_mask() == [False, False]
def test_pooling_params_update_combines_token_id_requirements() -> None:
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
params = PoolingParams(task="embed")
update.apply(params)
assert params.requires_token_ids is True
# ---------------------------------------------------------------------------
# CLSPool
# ---------------------------------------------------------------------------