[Core] Set pooling params based on task and model (#21128)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-07-18 05:41:17 -07:00
committed by GitHub
parent 4adc66f64d
commit 45badd05d0
24 changed files with 509 additions and 241 deletions
+27 -18
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, Literal, Optional
import msgspec
@@ -10,12 +10,14 @@ from vllm.sampling_params import RequestOutputKind
if TYPE_CHECKING:
from vllm.config import ModelConfig
PoolingTask = Literal["encode", "embed", "classify", "score"]
class PoolingParams(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
array_like=True): # type: ignore[call-arg]
"""API parameters for pooling models. This
"""API parameters for pooling models.
Attributes:
dimensions: Reduce the dimensions of embeddings
@@ -24,24 +26,33 @@ class PoolingParams(
dimensions: Optional[int] = None
use_cross_encoder: bool = False
"""Internal use only."""
logits_processing_needs_token_ids: bool = False
"""Internal use only."""
output_kind: RequestOutputKind = RequestOutputKind.FINAL_ONLY
task: Optional[PoolingTask] = None
"""Internal use only."""
requires_token_ids: bool = False
"""Internal use only."""
def clone(self) -> "PoolingParams":
"""Returns a deep copy of the PoolingParams instance."""
return PoolingParams(
dimensions=self.dimensions,
use_cross_encoder=self.use_cross_encoder,
logits_processing_needs_token_ids=self.
logits_processing_needs_token_ids,
task=self.task,
requires_token_ids=self.requires_token_ids,
)
def verify(self, model_config: "ModelConfig") -> None:
def verify(self, task: PoolingTask, model_config: "ModelConfig") -> None:
if self.task is None:
self.task = task
elif self.task != task:
msg = f"You cannot overwrite {self.task=!r} with {task=!r}!"
raise ValueError(msg)
# NOTE: Task validation needs to done against the model instance,
# which is not available in model config. So, it's not included
# in this method
if self.dimensions is not None:
if not model_config.is_matryoshka:
raise ValueError(
@@ -61,12 +72,10 @@ class PoolingParams(
raise ValueError("Dimensions must be greater than 0")
def __repr__(self) -> str:
return (
f"PoolingParams("
f"dimensions={self.dimensions}, "
f"use_cross_encoder={self.use_cross_encoder}, "
f"logits_processing_needs_token_ids={self.logits_processing_needs_token_ids})"
)
return (f"PoolingParams("
f"dimensions={self.dimensions}, "
f"task={self.task}, "
f"requires_token_ids={self.requires_token_ids})")
def __post_init__(self) -> None:
assert self.output_kind == RequestOutputKind.FINAL_ONLY,\