[Model][6/N] Improve all pooling task | Support chunked prefill with ALL pooling (#27145)

Signed-off-by: wang.yuqi <noooop@126.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2025-12-04 13:44:15 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 1b7c7f5159
commit 74c4d80c6c
15 changed files with 224 additions and 93 deletions
+21 -2
View File
@@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams, SamplingType
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.utils.collection_utils import swap_dict_values
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.pool.metadata import PoolingMetadata
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.sample.logits_processor import (
BatchUpdateBuilder,
LogitsProcessors,
@@ -33,7 +33,6 @@ class CachedRequestState:
prompt_token_ids: list[int] | None
mm_features: list[MultiModalFeatureSpec]
sampling_params: SamplingParams | None
pooling_params: PoolingParams | None
generator: torch.Generator | None
block_ids: tuple[list[int], ...]
@@ -51,11 +50,18 @@ class CachedRequestState:
# Used when both async_scheduling and spec_decode are enabled.
prev_num_draft_len: int = 0
# for pooling models
pooling_params: PoolingParams | None = None
pooling_states: PoolingStates | None = None
def __post_init__(self):
self.num_prompt_tokens = length_from_prompt_token_ids_or_embeds(
self.prompt_token_ids, self.prompt_embeds
)
if self.pooling_params is not None:
self.pooling_states = PoolingStates()
@property
def num_tokens(self) -> int:
return self.num_prompt_tokens + len(self.output_token_ids)
@@ -255,7 +261,9 @@ class InputBatch:
# This is updated each time the batch constituents change.
self.sampling_metadata = self._make_sampling_metadata()
# for pooling models
self.pooling_params: dict[str, PoolingParams] = {}
self.pooling_states: dict[str, PoolingStates] = {}
# Cached reference to the GPU tensor of previously sampled tokens
self.prev_sampled_token_ids: torch.Tensor | None = None
@@ -413,7 +421,11 @@ class InputBatch:
sampling_params.bad_words_token_ids
)
elif pooling_params := request.pooling_params:
pooling_states = request.pooling_states
assert pooling_states is not None
self.pooling_params[req_id] = pooling_params
self.pooling_states[req_id] = pooling_states
self.logits_processing_needs_token_ids[req_index] = (
pooling_params.requires_token_ids
)
@@ -469,6 +481,7 @@ class InputBatch:
if self.is_pooling_model:
self.pooling_params.pop(req_id, None)
self.pooling_states.pop(req_id, None)
return req_index
self.greedy_reqs.discard(req_id)
@@ -837,13 +850,19 @@ class InputBatch:
assert len(self.req_ids) == len(self.pooling_params)
return [self.pooling_params[req_id] for req_id in self.req_ids]
def get_pooling_states(self) -> list[PoolingStates]:
assert len(self.req_ids) == len(self.pooling_states)
return [self.pooling_states[req_id] for req_id in self.req_ids]
def get_pooling_metadata(self) -> PoolingMetadata:
pooling_params = self.get_pooling_params()
pooling_states = self.get_pooling_states()
return PoolingMetadata(
prompt_lens=torch.from_numpy(self.num_prompt_tokens[: self.num_reqs]),
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
pooling_params=pooling_params,
pooling_states=pooling_states,
)
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
+15 -35
View File
@@ -131,7 +131,7 @@ from vllm.v1.outputs import (
SamplerOutput,
make_empty_encoder_model_runner_output,
)
from vllm.v1.pool.metadata import PoolingMetadata
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.sample.logits_processor import LogitsProcessors, build_logitsprocs
from vllm.v1.sample.logits_processor.interface import LogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
@@ -2291,20 +2291,6 @@ class GPUModelRunner(
supported_tasks = list(model.pooler.get_supported_tasks())
if self.scheduler_config.enable_chunked_prefill:
if "token_embed" in supported_tasks:
supported_tasks.remove("token_embed")
if "token_classify" in supported_tasks:
supported_tasks.remove("token_classify")
logger.debug_once(
"Chunked prefill is not supported with "
"token_embed and token_classify tasks "
"which using ALL pooling. "
"Please turn off chunked prefill by "
"`--no-enable-chunked-prefill` before using it."
)
if "score" in supported_tasks:
num_labels = getattr(self.model_config.hf_config, "num_labels", 0)
if num_labels != 1:
@@ -2381,11 +2367,12 @@ class GPUModelRunner(
)
hidden_states = hidden_states[:num_scheduled_tokens]
seq_lens_cpu = self.seq_lens.cpu[: self.input_batch.num_reqs]
pooling_metadata = self.input_batch.get_pooling_metadata()
pooling_metadata.build_pooling_cursor(
num_scheduled_tokens_np.tolist(), device=hidden_states.device
num_scheduled_tokens_np.tolist(), seq_lens_cpu, device=hidden_states.device
)
seq_lens_cpu = self.seq_lens.cpu[: self.input_batch.num_reqs]
model = cast(VllmModelForPooling, self.model)
raw_pooler_output: PoolerOutput = model.pooler(
@@ -2393,7 +2380,7 @@ class GPUModelRunner(
pooling_metadata=pooling_metadata,
)
raw_pooler_output = json_map_leaves(
lambda x: x.to("cpu", non_blocking=True),
lambda x: x.to("cpu", non_blocking=True) if x is not None else x,
raw_pooler_output,
)
self._sync_device()
@@ -4248,10 +4235,13 @@ class GPUModelRunner(
prompt_lens=dummy_prompt_lens,
prompt_token_ids=dummy_token_ids,
pooling_params=[dummy_pooling_params] * num_reqs,
pooling_states=[PoolingStates() for i in range(num_reqs)],
)
dummy_metadata.build_pooling_cursor(
num_scheduled_tokens_list, device=hidden_states.device
num_scheduled_tokens_list,
seq_lens_cpu=dummy_prompt_lens,
device=hidden_states.device,
)
try:
@@ -4278,22 +4268,12 @@ class GPUModelRunner(
supported_pooling_tasks = self.get_supported_pooling_tasks()
if not supported_pooling_tasks:
if self.scheduler_config.enable_chunked_prefill:
raise RuntimeError(
f"Model {self.model_config.model} does not support "
"any pooling tasks with chunked prefill enabled. "
"Please add --no-enable-chunked-prefill to your "
"config or CLI args. See "
"https://docs.vllm.ai/en/latest/models/pooling_models.html "
"to learn more."
)
else:
raise RuntimeError(
f"Model {self.model_config.model} does not support "
"any pooling tasks. See "
"https://docs.vllm.ai/en/latest/models/pooling_models.html "
"to learn more."
)
raise RuntimeError(
f"Model {self.model_config.model} does not support "
"any pooling tasks. See "
"https://docs.vllm.ai/en/latest/models/pooling_models.html "
"to learn more."
)
output_size = dict[PoolingTask, float]()
for task in supported_pooling_tasks: