[Model] Pooling model activation supports per request control by PoolingParams (#20538)
Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
+19
-3
@@ -1189,6 +1189,8 @@ class LLM:
|
||||
/,
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams,
|
||||
Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""
|
||||
@@ -1207,7 +1209,8 @@ class LLM:
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
Returns:
|
||||
A list of `ClassificationRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
@@ -1220,6 +1223,7 @@ class LLM:
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
pooling_task="classify",
|
||||
)
|
||||
@@ -1272,6 +1276,7 @@ class LLM:
|
||||
text_2: list[Union[str, TextPrompt, TokensPrompt]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
|
||||
@@ -1280,6 +1285,7 @@ class LLM:
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
pooling_params=pooling_params,
|
||||
pooling_task="embed",
|
||||
)
|
||||
|
||||
@@ -1306,6 +1312,7 @@ class LLM:
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
model_config = self.llm_engine.model_config
|
||||
@@ -1317,7 +1324,12 @@ class LLM:
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * len(data_2)
|
||||
|
||||
pooling_params = PoolingParams(task="score")
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams(task="score")
|
||||
|
||||
model_config = self.llm_engine.model_config
|
||||
pooling_params.verify("score", model_config)
|
||||
|
||||
tokenization_kwargs: dict[str, Any] = {}
|
||||
|
||||
_validate_truncation_size(model_config.max_model_len,
|
||||
@@ -1379,6 +1391,7 @@ class LLM:
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
"""Generate similarity scores for all pairs `<text,text_pair>` or
|
||||
@@ -1410,7 +1423,8 @@ class LLM:
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
Returns:
|
||||
A list of `ScoringRequestOutput` objects containing the
|
||||
generated scores in the same order as the input prompts.
|
||||
@@ -1494,6 +1508,7 @@ class LLM:
|
||||
data_2, # type: ignore[arg-type]
|
||||
truncate_prompt_tokens,
|
||||
use_tqdm,
|
||||
pooling_params,
|
||||
lora_request)
|
||||
else:
|
||||
return self._embedding_score(
|
||||
@@ -1502,6 +1517,7 @@ class LLM:
|
||||
data_2, # type: ignore[arg-type]
|
||||
truncate_prompt_tokens,
|
||||
use_tqdm,
|
||||
pooling_params,
|
||||
lora_request)
|
||||
|
||||
def start_profile(self) -> None:
|
||||
|
||||
@@ -1274,11 +1274,13 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."),
|
||||
)
|
||||
normalize: Optional[bool] = None
|
||||
|
||||
# --8<-- [end:embedding-extra-params]
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(dimensions=self.dimensions)
|
||||
return PoolingParams(dimensions=self.dimensions,
|
||||
normalize=self.normalize)
|
||||
|
||||
|
||||
class EmbeddingChatRequest(OpenAIBaseModel):
|
||||
@@ -1332,6 +1334,7 @@ class EmbeddingChatRequest(OpenAIBaseModel):
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."),
|
||||
)
|
||||
normalize: Optional[bool] = None
|
||||
# --8<-- [end:chat-embedding-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@@ -1344,7 +1347,8 @@ class EmbeddingChatRequest(OpenAIBaseModel):
|
||||
return data
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(dimensions=self.dimensions)
|
||||
return PoolingParams(dimensions=self.dimensions,
|
||||
normalize=self.normalize)
|
||||
|
||||
|
||||
EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]
|
||||
@@ -1375,10 +1379,12 @@ class ScoreRequest(OpenAIBaseModel):
|
||||
"if the served model does not use priority scheduling."),
|
||||
)
|
||||
|
||||
activation: Optional[bool] = None
|
||||
|
||||
# --8<-- [end:score-extra-params]
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams()
|
||||
return PoolingParams(activation=self.activation)
|
||||
|
||||
|
||||
class RerankRequest(OpenAIBaseModel):
|
||||
@@ -1403,10 +1409,12 @@ class RerankRequest(OpenAIBaseModel):
|
||||
"if the served model does not use priority scheduling."),
|
||||
)
|
||||
|
||||
activation: Optional[bool] = None
|
||||
|
||||
# --8<-- [end:rerank-extra-params]
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams()
|
||||
return PoolingParams(activation=self.activation)
|
||||
|
||||
|
||||
class RerankDocument(BaseModel):
|
||||
@@ -1553,10 +1561,12 @@ class ClassificationRequest(OpenAIBaseModel):
|
||||
"if the served model does not use priority scheduling."),
|
||||
)
|
||||
|
||||
activation: Optional[bool] = None
|
||||
|
||||
# --8<-- [end:classification-extra-params]
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams()
|
||||
return PoolingParams(activation=self.activation)
|
||||
|
||||
|
||||
class ClassificationData(OpenAIBaseModel):
|
||||
|
||||
Reference in New Issue
Block a user