[Refactor] Clean up pooling models build_tok_params logic (#42341)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -6,13 +6,14 @@ from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams, merge_kwargs
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
|
||||
|
||||
@@ -68,6 +69,88 @@ class PoolingBasicRequestMixin(OpenAIBaseModel):
|
||||
)
|
||||
# --8<-- [end:pooling-common-extra-params]
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
if max_output_tokens_param is None:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
)
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
max_output_tokens_param=max_output_tokens_param,
|
||||
)
|
||||
|
||||
|
||||
class PoolingTokenizeParamsMixin:
|
||||
add_special_tokens: bool
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
default_max_total_tokens = model_config.max_model_len
|
||||
max_total_tokens: int | None = default_max_total_tokens
|
||||
max_output_tokens = 0
|
||||
|
||||
pooler_config = model_config.pooler_config
|
||||
if pooler_config is not None:
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = None
|
||||
else:
|
||||
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
|
||||
max_output_tokens = default_max_total_tokens - max_embed_len
|
||||
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
|
||||
class CompletionRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:completion-params]
|
||||
|
||||
@@ -7,16 +7,15 @@ from typing import TypeAlias
|
||||
from pydantic import Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
)
|
||||
|
||||
@@ -24,21 +23,11 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ClassificationCompletionRequest(
|
||||
PoolingBasicRequestMixin, CompletionRequestMixin, ClassifyRequestMixin
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
@@ -47,21 +36,11 @@ class ClassificationCompletionRequest(
|
||||
|
||||
|
||||
class ClassificationChatRequest(
|
||||
PoolingBasicRequestMixin, ChatRequestMixin, ClassifyRequestMixin
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
|
||||
@@ -16,62 +16,24 @@ import pybase64 as base64
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
EmbedRequestMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI /v1/embeddings — request models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_max_total_output_tokens(
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[int | None, int]:
|
||||
max_total_tokens = model_config.max_model_len
|
||||
pooler_config = model_config.pooler_config
|
||||
|
||||
if pooler_config is None:
|
||||
return max_total_tokens, 0
|
||||
|
||||
if pooler_config.enable_chunked_processing:
|
||||
return None, 0
|
||||
|
||||
max_embed_len = pooler_config.max_embed_len or max_total_tokens
|
||||
max_output_tokens = max_total_tokens - max_embed_len
|
||||
return max_total_tokens, max_output_tokens
|
||||
|
||||
|
||||
class EmbeddingCompletionRequest(
|
||||
PoolingBasicRequestMixin, CompletionRequestMixin, EmbedRequestMixin
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
(
|
||||
max_total_tokens,
|
||||
max_output_tokens,
|
||||
) = _get_max_total_output_tokens(model_config)
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
@@ -81,27 +43,11 @@ class EmbeddingCompletionRequest(
|
||||
|
||||
|
||||
class EmbeddingChatRequest(
|
||||
PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
(
|
||||
max_total_tokens,
|
||||
max_output_tokens,
|
||||
) = _get_max_total_output_tokens(model_config)
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
|
||||
@@ -18,6 +18,7 @@ from ..base.protocol import (
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EncodingRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
)
|
||||
|
||||
@@ -27,22 +28,10 @@ class PoolingCompletionRequest(
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
@@ -52,23 +41,14 @@ class PoolingCompletionRequest(
|
||||
|
||||
|
||||
class PoolingChatRequest(
|
||||
PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin, ClassifyRequestMixin
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
@@ -85,16 +65,11 @@ class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic
|
||||
task: PoolingTask = "plugin"
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=not model_config.is_encoder_decoder,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=not model_config.is_encoder_decoder,
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
|
||||
@@ -38,15 +38,11 @@ class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
# --8<-- [end:scoring-common-params]
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
|
||||
return TokenizeParams(
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=True,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
|
||||
Reference in New Issue
Block a user