diff --git a/vllm/entrypoints/pooling/base/protocol.py b/vllm/entrypoints/pooling/base/protocol.py index 2ce89e4bf2..9e410a2b54 100644 --- a/vllm/entrypoints/pooling/base/protocol.py +++ b/vllm/entrypoints/pooling/base/protocol.py @@ -6,13 +6,14 @@ from typing import Annotated, Any, Literal from pydantic import Field, model_validator +from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel from vllm.exceptions import VLLMValidationError -from vllm.renderers import ChatParams, merge_kwargs +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs from vllm.utils import random_uuid from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness @@ -68,6 +69,88 @@ class PoolingBasicRequestMixin(OpenAIBaseModel): ) # --8<-- [end:pooling-common-extra-params] + def _build_pooling_tok_params( + self, + model_config: ModelConfig, + *, + add_special_tokens: bool, + max_total_tokens: int | None, + max_output_tokens: int, + max_total_tokens_param: str = "max_model_len", + max_output_tokens_param: str | None = None, + ) -> TokenizeParams: + encoder_config = model_config.encoder_config or {} + if max_output_tokens_param is None: + return TokenizeParams( + max_total_tokens=max_total_tokens, + max_output_tokens=max_output_tokens, + truncate_prompt_tokens=self.truncate_prompt_tokens, + truncation_side=self.truncation_side, + do_lower_case=encoder_config.get("do_lower_case", False), + add_special_tokens=add_special_tokens, + max_total_tokens_param=max_total_tokens_param, + ) + + return TokenizeParams( + max_total_tokens=max_total_tokens, + max_output_tokens=max_output_tokens, + truncate_prompt_tokens=self.truncate_prompt_tokens, + truncation_side=self.truncation_side, + do_lower_case=encoder_config.get("do_lower_case", False), + add_special_tokens=add_special_tokens, + max_total_tokens_param=max_total_tokens_param, + max_output_tokens_param=max_output_tokens_param, + ) + + +class PoolingTokenizeParamsMixin: + add_special_tokens: bool + + def _build_pooling_tok_params( + self, + model_config: ModelConfig, + *, + add_special_tokens: bool, + max_total_tokens: int | None, + max_output_tokens: int, + max_total_tokens_param: str = "max_model_len", + max_output_tokens_param: str | None = None, + ) -> TokenizeParams: + raise NotImplementedError + + +class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin): + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: + return self._build_pooling_tok_params( + model_config, + add_special_tokens=self.add_special_tokens, + max_total_tokens=model_config.max_model_len, + max_output_tokens=0, + ) + + +class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin): + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: + default_max_total_tokens = model_config.max_model_len + max_total_tokens: int | None = default_max_total_tokens + max_output_tokens = 0 + + pooler_config = model_config.pooler_config + if pooler_config is not None: + if pooler_config.enable_chunked_processing: + max_total_tokens = None + else: + max_embed_len = pooler_config.max_embed_len or default_max_total_tokens + max_output_tokens = default_max_total_tokens - max_embed_len + + return self._build_pooling_tok_params( + model_config, + add_special_tokens=self.add_special_tokens, + max_total_tokens=max_total_tokens, + max_output_tokens=max_output_tokens, + max_output_tokens_param="max_model_len - max_embed_len", + ) + class CompletionRequestMixin(OpenAIBaseModel): # --8<-- [start:completion-params] diff --git a/vllm/entrypoints/pooling/classify/protocol.py b/vllm/entrypoints/pooling/classify/protocol.py index b5948e406b..39cacdd835 100644 --- a/vllm/entrypoints/pooling/classify/protocol.py +++ b/vllm/entrypoints/pooling/classify/protocol.py @@ -7,16 +7,15 @@ from typing import TypeAlias from pydantic import Field from vllm import PoolingParams -from vllm.config import ModelConfig from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo from vllm.logger import init_logger -from vllm.renderers import TokenizeParams from vllm.utils import random_uuid from ..base.protocol import ( ChatRequestMixin, ClassifyRequestMixin, CompletionRequestMixin, + FixedMaxLenTokenizeParamsMixin, PoolingBasicRequestMixin, ) @@ -24,21 +23,11 @@ logger = init_logger(__name__) class ClassificationCompletionRequest( - PoolingBasicRequestMixin, CompletionRequestMixin, ClassifyRequestMixin + PoolingBasicRequestMixin, + CompletionRequestMixin, + ClassifyRequestMixin, + FixedMaxLenTokenizeParamsMixin, ): - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - ) - def to_pooling_params(self): return PoolingParams( task="classify", @@ -47,21 +36,11 @@ class ClassificationCompletionRequest( class ClassificationChatRequest( - PoolingBasicRequestMixin, ChatRequestMixin, ClassifyRequestMixin + PoolingBasicRequestMixin, + ChatRequestMixin, + ClassifyRequestMixin, + FixedMaxLenTokenizeParamsMixin, ): - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - ) - def to_pooling_params(self): return PoolingParams( task="classify", diff --git a/vllm/entrypoints/pooling/embed/protocol.py b/vllm/entrypoints/pooling/embed/protocol.py index 64087e2333..d886e3199f 100644 --- a/vllm/entrypoints/pooling/embed/protocol.py +++ b/vllm/entrypoints/pooling/embed/protocol.py @@ -16,62 +16,24 @@ import pybase64 as base64 from pydantic import BaseModel, Field from vllm import PoolingParams -from vllm.config import ModelConfig from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo -from vllm.renderers import TokenizeParams from vllm.utils import random_uuid from ..base.protocol import ( ChatRequestMixin, CompletionRequestMixin, + EmbeddingTokenizeParamsMixin, EmbedRequestMixin, PoolingBasicRequestMixin, ) -# --------------------------------------------------------------------------- -# OpenAI /v1/embeddings — request models -# --------------------------------------------------------------------------- - - -def _get_max_total_output_tokens( - model_config: ModelConfig, -) -> tuple[int | None, int]: - max_total_tokens = model_config.max_model_len - pooler_config = model_config.pooler_config - - if pooler_config is None: - return max_total_tokens, 0 - - if pooler_config.enable_chunked_processing: - return None, 0 - - max_embed_len = pooler_config.max_embed_len or max_total_tokens - max_output_tokens = max_total_tokens - max_embed_len - return max_total_tokens, max_output_tokens - class EmbeddingCompletionRequest( - PoolingBasicRequestMixin, CompletionRequestMixin, EmbedRequestMixin + PoolingBasicRequestMixin, + CompletionRequestMixin, + EmbedRequestMixin, + EmbeddingTokenizeParamsMixin, ): - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - ( - max_total_tokens, - max_output_tokens, - ) = _get_max_total_output_tokens(model_config) - - return TokenizeParams( - max_total_tokens=max_total_tokens, - max_output_tokens=max_output_tokens, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - max_output_tokens_param="max_model_len - max_embed_len", - ) - def to_pooling_params(self): return PoolingParams( task="embed", @@ -81,27 +43,11 @@ class EmbeddingCompletionRequest( class EmbeddingChatRequest( - PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin + PoolingBasicRequestMixin, + ChatRequestMixin, + EmbedRequestMixin, + EmbeddingTokenizeParamsMixin, ): - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - ( - max_total_tokens, - max_output_tokens, - ) = _get_max_total_output_tokens(model_config) - - return TokenizeParams( - max_total_tokens=max_total_tokens, - max_output_tokens=max_output_tokens, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - max_output_tokens_param="max_model_len - max_embed_len", - ) - def to_pooling_params(self): return PoolingParams( task="embed", diff --git a/vllm/entrypoints/pooling/pooling/protocol.py b/vllm/entrypoints/pooling/pooling/protocol.py index ab9b132fee..b2a43b1935 100644 --- a/vllm/entrypoints/pooling/pooling/protocol.py +++ b/vllm/entrypoints/pooling/pooling/protocol.py @@ -18,6 +18,7 @@ from ..base.protocol import ( CompletionRequestMixin, EmbedRequestMixin, EncodingRequestMixin, + FixedMaxLenTokenizeParamsMixin, PoolingBasicRequestMixin, ) @@ -27,22 +28,10 @@ class PoolingCompletionRequest( CompletionRequestMixin, EmbedRequestMixin, ClassifyRequestMixin, + FixedMaxLenTokenizeParamsMixin, ): task: PoolingTask | None = None - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - ) - def to_pooling_params(self): return PoolingParams( task=self.task, @@ -52,23 +41,14 @@ class PoolingCompletionRequest( class PoolingChatRequest( - PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin, ClassifyRequestMixin + PoolingBasicRequestMixin, + ChatRequestMixin, + EmbedRequestMixin, + ClassifyRequestMixin, + FixedMaxLenTokenizeParamsMixin, ): task: PoolingTask | None = None - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=self.add_special_tokens, - max_total_tokens_param="max_model_len", - ) - def to_pooling_params(self): return PoolingParams( task=self.task, @@ -85,16 +65,11 @@ class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic task: PoolingTask = "plugin" def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( + return self._build_pooling_tok_params( + model_config, + add_special_tokens=not model_config.is_encoder_decoder, max_total_tokens=model_config.max_model_len, max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=not model_config.is_encoder_decoder, - max_total_tokens_param="max_model_len", ) def to_pooling_params(self): diff --git a/vllm/entrypoints/pooling/scoring/protocol.py b/vllm/entrypoints/pooling/scoring/protocol.py index d32a3660d0..b2fa0f2194 100644 --- a/vllm/entrypoints/pooling/scoring/protocol.py +++ b/vllm/entrypoints/pooling/scoring/protocol.py @@ -38,15 +38,11 @@ class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin): # --8<-- [end:scoring-common-params] def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( + return self._build_pooling_tok_params( + model_config, + add_special_tokens=True, max_total_tokens=model_config.max_model_len, max_output_tokens=0, - truncate_prompt_tokens=self.truncate_prompt_tokens, - truncation_side=self.truncation_side, - do_lower_case=encoder_config.get("do_lower_case", False), - max_total_tokens_param="max_model_len", ) def to_pooling_params(self, task: PoolingTask = "classify"):