[Core] Set pooling params based on task and model (#21128)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+27
-18
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING, Literal, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -10,12 +10,14 @@ from vllm.sampling_params import RequestOutputKind
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
PoolingTask = Literal["encode", "embed", "classify", "score"]
|
||||
|
||||
|
||||
class PoolingParams(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
array_like=True): # type: ignore[call-arg]
|
||||
"""API parameters for pooling models. This
|
||||
"""API parameters for pooling models.
|
||||
|
||||
Attributes:
|
||||
dimensions: Reduce the dimensions of embeddings
|
||||
@@ -24,24 +26,33 @@ class PoolingParams(
|
||||
|
||||
dimensions: Optional[int] = None
|
||||
|
||||
use_cross_encoder: bool = False
|
||||
"""Internal use only."""
|
||||
|
||||
logits_processing_needs_token_ids: bool = False
|
||||
"""Internal use only."""
|
||||
|
||||
output_kind: RequestOutputKind = RequestOutputKind.FINAL_ONLY
|
||||
|
||||
task: Optional[PoolingTask] = None
|
||||
"""Internal use only."""
|
||||
|
||||
requires_token_ids: bool = False
|
||||
"""Internal use only."""
|
||||
|
||||
def clone(self) -> "PoolingParams":
|
||||
"""Returns a deep copy of the PoolingParams instance."""
|
||||
return PoolingParams(
|
||||
dimensions=self.dimensions,
|
||||
use_cross_encoder=self.use_cross_encoder,
|
||||
logits_processing_needs_token_ids=self.
|
||||
logits_processing_needs_token_ids,
|
||||
task=self.task,
|
||||
requires_token_ids=self.requires_token_ids,
|
||||
)
|
||||
|
||||
def verify(self, model_config: "ModelConfig") -> None:
|
||||
def verify(self, task: PoolingTask, model_config: "ModelConfig") -> None:
|
||||
if self.task is None:
|
||||
self.task = task
|
||||
elif self.task != task:
|
||||
msg = f"You cannot overwrite {self.task=!r} with {task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
# NOTE: Task validation needs to done against the model instance,
|
||||
# which is not available in model config. So, it's not included
|
||||
# in this method
|
||||
|
||||
if self.dimensions is not None:
|
||||
if not model_config.is_matryoshka:
|
||||
raise ValueError(
|
||||
@@ -61,12 +72,10 @@ class PoolingParams(
|
||||
raise ValueError("Dimensions must be greater than 0")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"PoolingParams("
|
||||
f"dimensions={self.dimensions}, "
|
||||
f"use_cross_encoder={self.use_cross_encoder}, "
|
||||
f"logits_processing_needs_token_ids={self.logits_processing_needs_token_ids})"
|
||||
)
|
||||
return (f"PoolingParams("
|
||||
f"dimensions={self.dimensions}, "
|
||||
f"task={self.task}, "
|
||||
f"requires_token_ids={self.requires_token_ids})")
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert self.output_kind == RequestOutputKind.FINAL_ONLY,\
|
||||
|
||||
Reference in New Issue
Block a user