[Frontend] Use new Renderer for Completions and Tokenize API (#32863)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,7 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .params import ChatParams, TokenizeParams, merge_kwargs
|
||||
from .protocol import RendererLike
|
||||
from .registry import RendererRegistry, renderer_from_config
|
||||
|
||||
__all__ = ["RendererLike", "RendererRegistry", "renderer_from_config"]
|
||||
__all__ = [
|
||||
"RendererLike",
|
||||
"RendererRegistry",
|
||||
"renderer_from_config",
|
||||
"ChatParams",
|
||||
"TokenizeParams",
|
||||
"merge_kwargs",
|
||||
]
|
||||
|
||||
@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
|
||||
|
||||
from .params import ChatParams
|
||||
from .protocol import RendererLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -61,8 +62,8 @@ class DeepseekV32Renderer(RendererLike):
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -74,26 +75,22 @@ class DeepseekV32Renderer(RendererLike):
|
||||
prompt_raw = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
messages=messages,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -105,17 +102,13 @@ class DeepseekV32Renderer(RendererLike):
|
||||
prompt_raw = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
messages=messages,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
def safe_load_prompt_embeds(
|
||||
model_config: "ModelConfig",
|
||||
embed: bytes,
|
||||
) -> torch.Tensor:
|
||||
if not model_config.enable_prompt_embeds:
|
||||
raise VLLMValidationError(
|
||||
"You must set `--enable-prompt-embeds` to input `prompt_embeds`.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Enable sparse tensor integrity checks to prevent out-of-bounds
|
||||
# writes from maliciously crafted tensors
|
||||
with torch.sparse.check_sparse_tensor_invariants():
|
||||
tensor = torch.load(
|
||||
BytesIO(pybase64.b64decode(embed, validate=True)),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
)
|
||||
tensor = tensor.to_dense()
|
||||
|
||||
if tensor.dim() > 2:
|
||||
tensor = tensor.squeeze(0)
|
||||
assert tensor.dim() == 2
|
||||
|
||||
return tensor
|
||||
+12
-19
@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.grok2 import Grok2Tokenizer
|
||||
|
||||
from .params import ChatParams
|
||||
from .protocol import RendererLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -61,8 +62,8 @@ class Grok2Renderer(RendererLike):
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -74,26 +75,22 @@ class Grok2Renderer(RendererLike):
|
||||
prompt_raw = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
messages=messages,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -105,17 +102,13 @@ class Grok2Renderer(RendererLike):
|
||||
prompt_raw = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
messages=messages,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
+20
-31
@@ -25,7 +25,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer
|
||||
@@ -33,6 +33,7 @@ from vllm.transformers_utils.chat_templates import get_chat_template_fallback_pa
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
from .params import ChatParams
|
||||
from .protocol import RendererLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -632,9 +633,8 @@ class HfRenderer(RendererLike):
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
model_config = self.config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
@@ -642,9 +642,9 @@ class HfRenderer(RendererLike):
|
||||
messages,
|
||||
model_config,
|
||||
content_format=resolve_chat_template_content_format(
|
||||
chat_template=kwargs.get("chat_template"),
|
||||
tools=kwargs.get("tools"),
|
||||
given_format=chat_template_content_format,
|
||||
chat_template=params.chat_template,
|
||||
tools=params.chat_template_kwargs.get("tools"),
|
||||
given_format=params.chat_template_content_format,
|
||||
tokenizer=tokenizer,
|
||||
model_config=model_config,
|
||||
),
|
||||
@@ -654,7 +654,7 @@ class HfRenderer(RendererLike):
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -666,7 +666,7 @@ class HfRenderer(RendererLike):
|
||||
):
|
||||
mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)
|
||||
|
||||
# get video placehoder, replace it with runtime video-chunk prompts
|
||||
# get video placeholder, replace it with runtime video-chunk prompts
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
@@ -676,24 +676,19 @@ class HfRenderer(RendererLike):
|
||||
video_placeholder,
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
model_config = self.config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
@@ -701,9 +696,9 @@ class HfRenderer(RendererLike):
|
||||
messages,
|
||||
model_config,
|
||||
content_format=resolve_chat_template_content_format(
|
||||
chat_template=kwargs.get("chat_template"),
|
||||
tools=kwargs.get("tools"),
|
||||
given_format=chat_template_content_format,
|
||||
chat_template=params.chat_template,
|
||||
tools=params.chat_template_kwargs.get("tools"),
|
||||
given_format=params.chat_template_content_format,
|
||||
tokenizer=tokenizer,
|
||||
model_config=model_config,
|
||||
),
|
||||
@@ -713,7 +708,7 @@ class HfRenderer(RendererLike):
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**kwargs,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -723,9 +718,7 @@ class HfRenderer(RendererLike):
|
||||
and mm_uuids is not None
|
||||
and mm_data is not None
|
||||
):
|
||||
mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)
|
||||
|
||||
# get video placehoder, replace it with runtime video-chunk prompts
|
||||
# get video placeholder, replace it with runtime video-chunk prompts
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
@@ -735,14 +728,10 @@ class HfRenderer(RendererLike):
|
||||
video_placeholder,
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
+18
-19
@@ -10,12 +10,13 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.utils.async_utils import make_async
|
||||
|
||||
from .params import ChatParams
|
||||
from .protocol import RendererLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -95,8 +96,8 @@ class MistralRenderer(RendererLike):
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -104,25 +105,25 @@ class MistralRenderer(RendererLike):
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
prompt_raw = safe_apply_chat_template(tokenizer, messages, **kwargs)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
tokenizer,
|
||||
messages,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -131,17 +132,15 @@ class MistralRenderer(RendererLike):
|
||||
)
|
||||
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
tokenizer, messages, **kwargs
|
||||
tokenizer,
|
||||
messages,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
else TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
)
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt # type: ignore[return-value]
|
||||
return conversation, prompt
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
else:
|
||||
torch = LazyLoader("torch", globals(), "torch")
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_S = TypeVar("_S", list[int], "torch.Tensor")
|
||||
|
||||
|
||||
def merge_kwargs(
|
||||
defaults: dict[str, Any] | None,
|
||||
overrides: dict[str, Any] | None,
|
||||
/,
|
||||
*,
|
||||
unset_values: tuple[object, ...] = (None, "auto"),
|
||||
) -> dict[str, Any]:
|
||||
if defaults is None:
|
||||
defaults = {}
|
||||
if overrides is None:
|
||||
overrides = {}
|
||||
|
||||
return defaults | {k: v for k, v in overrides.items() if v not in unset_values}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChatParams:
|
||||
"""Configuration to control how to parse chat messages."""
|
||||
|
||||
chat_template: str | None = None
|
||||
"""The chat template to apply."""
|
||||
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
||||
"""The format of the chat template."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
"""The kwargs to pass to the chat template."""
|
||||
|
||||
def with_defaults(self, default_chat_template_kwargs: dict[str, Any] | None):
|
||||
if not default_chat_template_kwargs:
|
||||
return self
|
||||
|
||||
return ChatParams(
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
default_chat_template_kwargs,
|
||||
self.chat_template_kwargs,
|
||||
),
|
||||
)
|
||||
|
||||
def get_apply_chat_template_kwargs(self) -> dict[str, Any]:
|
||||
"""The arguments to pass to `tokenizer.apply_chat_template`."""
|
||||
return merge_kwargs(
|
||||
self.chat_template_kwargs,
|
||||
dict(chat_template=self.chat_template),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TokenizeParams:
|
||||
"""Configuration to control how prompts are tokenized."""
|
||||
|
||||
max_total_tokens: int | None
|
||||
"""
|
||||
Maximum allowed number of input + output tokens.
|
||||
|
||||
Usually, this refers to the model's context length.
|
||||
"""
|
||||
|
||||
max_output_tokens: int = 0
|
||||
"""Maximum requested number of output tokens."""
|
||||
|
||||
pad_prompt_tokens: int | None = None
|
||||
"""
|
||||
Number of tokens to pad to:
|
||||
- `None` means no padding.
|
||||
- `-1` maps to `max_input_tokens`.
|
||||
"""
|
||||
|
||||
truncate_prompt_tokens: int | None = None
|
||||
"""
|
||||
Number of tokens to keep:
|
||||
- `None` means no truncation.
|
||||
- `-1` maps to `max_input_tokens`.
|
||||
"""
|
||||
|
||||
do_lower_case: bool = False
|
||||
"""Whether to normalize text to lower case before tokenization."""
|
||||
|
||||
add_special_tokens: bool = True
|
||||
"""Whether to add special tokens."""
|
||||
|
||||
needs_detokenization: bool = False
|
||||
"""
|
||||
Whether the tokenized prompt needs to contain the original text.
|
||||
|
||||
Not to be confused with `SamplingParams.detokenize` which deals
|
||||
with the output generated by the model.
|
||||
"""
|
||||
|
||||
max_total_tokens_param: str = "max_total_tokens"
|
||||
"""Override this to edit the message for validation errors."""
|
||||
|
||||
max_output_tokens_param: str = "max_output_tokens"
|
||||
"""Override this to edit the message for validation errors."""
|
||||
|
||||
truncate_prompt_tokens_param: str = "truncate_prompt_tokens"
|
||||
"""Override this to edit the message for validation errors."""
|
||||
|
||||
@property
|
||||
def max_input_tokens(self) -> int | None:
|
||||
"""Maximum allowed number of input tokens."""
|
||||
if self.max_total_tokens is None:
|
||||
return None
|
||||
|
||||
return self.max_total_tokens - self.max_output_tokens
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
max_total_tokens = self.max_total_tokens
|
||||
max_output_tokens = self.max_output_tokens
|
||||
max_input_tokens = self.max_input_tokens
|
||||
truncate_prompt_tokens = self.truncate_prompt_tokens
|
||||
|
||||
if (
|
||||
max_output_tokens is not None
|
||||
and max_total_tokens is not None
|
||||
and max_output_tokens > max_total_tokens
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
f"{self.max_output_tokens_param}={max_output_tokens}"
|
||||
f"cannot be greater than "
|
||||
f"{self.max_total_tokens_param}={max_total_tokens=}. "
|
||||
f"Please request fewer output tokens.",
|
||||
parameter=self.max_output_tokens_param,
|
||||
value=max_output_tokens,
|
||||
)
|
||||
|
||||
if (
|
||||
max_input_tokens is not None
|
||||
and truncate_prompt_tokens is not None
|
||||
and truncate_prompt_tokens > max_input_tokens
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} "
|
||||
f"cannot be greater than {self.max_total_tokens_param} - "
|
||||
f"{self.max_output_tokens_param} = {max_input_tokens}. "
|
||||
f"Please request a smaller truncation size.",
|
||||
parameter=self.truncate_prompt_tokens_param,
|
||||
value=truncate_prompt_tokens,
|
||||
)
|
||||
|
||||
def with_kwargs(self, tokenization_kwargs: dict[str, Any] | None):
|
||||
if tokenization_kwargs is None:
|
||||
tokenization_kwargs = {}
|
||||
|
||||
max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)
|
||||
pad_prompt_tokens = tokenization_kwargs.pop(
|
||||
"pad_prompt_tokens", self.pad_prompt_tokens
|
||||
)
|
||||
truncate_prompt_tokens = tokenization_kwargs.pop(
|
||||
"truncate_prompt_tokens", self.truncate_prompt_tokens
|
||||
)
|
||||
do_lower_case = tokenization_kwargs.pop("do_lower_case", self.do_lower_case)
|
||||
add_special_tokens = tokenization_kwargs.pop(
|
||||
"add_special_tokens", self.add_special_tokens
|
||||
)
|
||||
needs_detokenization = tokenization_kwargs.pop(
|
||||
"needs_detokenization", self.needs_detokenization
|
||||
)
|
||||
|
||||
# https://huggingface.co/docs/transformers/en/pad_truncation
|
||||
if padding := tokenization_kwargs.pop("padding", None):
|
||||
if padding == "max_length":
|
||||
pad_prompt_tokens = max_length
|
||||
elif padding in (False, "do_not_pad"):
|
||||
pad_prompt_tokens = None
|
||||
else:
|
||||
# To emit the below warning
|
||||
tokenization_kwargs["padding"] = padding
|
||||
|
||||
if truncation := tokenization_kwargs.pop("truncation", None):
|
||||
if truncation in (True, "longest_first"):
|
||||
truncate_prompt_tokens = max_length
|
||||
elif truncation in (False, "do_not_truncate"):
|
||||
truncate_prompt_tokens = None
|
||||
else:
|
||||
# To emit the below warning
|
||||
tokenization_kwargs["truncation"] = truncation
|
||||
|
||||
if tokenization_kwargs:
|
||||
logger.warning(
|
||||
"The following tokenization arguments are not supported "
|
||||
"by vLLM Renderer and will be ignored: %s",
|
||||
tokenization_kwargs,
|
||||
)
|
||||
|
||||
max_total_tokens = self.max_total_tokens
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=(
|
||||
0
|
||||
if max_total_tokens is None or max_length is None
|
||||
else max_total_tokens - max_length
|
||||
),
|
||||
pad_prompt_tokens=pad_prompt_tokens,
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
do_lower_case=do_lower_case,
|
||||
add_special_tokens=add_special_tokens,
|
||||
needs_detokenization=needs_detokenization,
|
||||
)
|
||||
|
||||
def get_encode_kwargs(self) -> dict[str, Any]:
|
||||
"""The arguments to pass to `tokenizer.encode`."""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
|
||||
return dict(
|
||||
truncation=self.truncate_prompt_tokens is not None,
|
||||
max_length=max_length,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
)
|
||||
|
||||
def _apply_lowercase(self, tokenizer: TokenizerLike | None, text: str) -> str:
|
||||
if self.do_lower_case:
|
||||
text = text.lower()
|
||||
|
||||
return text
|
||||
|
||||
def _validate_text(self, tokenizer: TokenizerLike | None, text: str) -> str:
|
||||
"""Apply all validators to prompt text."""
|
||||
# TODO: Implement https://github.com/vllm-project/vllm/pull/31366
|
||||
for validator in (self._apply_lowercase,):
|
||||
text = validator(tokenizer, text)
|
||||
|
||||
return text
|
||||
|
||||
def apply_pre_tokenization(
|
||||
self,
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: TextPrompt,
|
||||
) -> TextPrompt:
|
||||
"""
|
||||
Ensure that the prompt meets the requirements set out by this config.
|
||||
If that is not possible, raise a `VLLMValidationError`.
|
||||
|
||||
This method is run before tokenization occurs.
|
||||
"""
|
||||
prompt["prompt"] = self._validate_text(tokenizer, prompt["prompt"])
|
||||
|
||||
return prompt
|
||||
|
||||
def _apply_padding(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||
"""Apply padding to a token sequence."""
|
||||
pad_length = self.pad_prompt_tokens
|
||||
if pad_length is not None and pad_length < 0:
|
||||
pad_length = self.max_input_tokens
|
||||
|
||||
if pad_length is None or pad_length <= len(tokens):
|
||||
return tokens
|
||||
|
||||
if tokenizer is None:
|
||||
raise ValueError("Cannot pad tokens when `skip_tokenizer_init=True`")
|
||||
if not isinstance(tokens, list):
|
||||
raise ValueError("Cannot pad tokens for embedding inputs")
|
||||
|
||||
return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens))
|
||||
|
||||
def _apply_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||
"""Apply truncation to a token sequence."""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
|
||||
if max_length is None or max_length >= len(tokens):
|
||||
return tokens
|
||||
if max_length == 0:
|
||||
return tokens[:0]
|
||||
|
||||
if getattr(tokenizer, "truncation_side", "left") == "left":
|
||||
return tokens[-max_length:]
|
||||
|
||||
return tokens[:max_length]
|
||||
|
||||
def _apply_length_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||
"""Apply length checks to a token sequence."""
|
||||
max_input_tokens = self.max_input_tokens
|
||||
|
||||
if max_input_tokens is not None and len(tokens) > max_input_tokens:
|
||||
raise VLLMValidationError(
|
||||
f"You passed {len(tokens)} input tokens and "
|
||||
f"requested {self.max_output_tokens} output tokens. "
|
||||
f"However, the model's context length is only "
|
||||
f"{self.max_total_tokens}, resulting in a maximum "
|
||||
f"input length of {max_input_tokens}. "
|
||||
f"Please reduce the length of the input messages.",
|
||||
parameter="input_tokens",
|
||||
value=len(tokens),
|
||||
)
|
||||
|
||||
return tokens
|
||||
|
||||
def _validate_tokens(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||
"""Apply all validators to a token sequence."""
|
||||
for validator in (
|
||||
self._apply_padding,
|
||||
self._apply_truncation,
|
||||
self._apply_length_check,
|
||||
):
|
||||
tokens = validator(tokenizer, tokens)
|
||||
|
||||
return tokens
|
||||
|
||||
def apply_post_tokenization(
|
||||
self,
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: TokensPrompt | EmbedsPrompt,
|
||||
) -> TokensPrompt | EmbedsPrompt:
|
||||
"""
|
||||
Ensure that the prompt meets the requirements set out by this config.
|
||||
If that is not possible, raise a `VLLMValidationError`.
|
||||
|
||||
This method is run after tokenization occurs.
|
||||
"""
|
||||
if "prompt_token_ids" in prompt:
|
||||
prompt["prompt_token_ids"] = self._validate_tokens( # type: ignore[typeddict-unknown-key]
|
||||
tokenizer,
|
||||
prompt["prompt_token_ids"], # type: ignore[typeddict-item]
|
||||
)
|
||||
if "prompt_embeds" in prompt:
|
||||
prompt["prompt_embeds"] = self._validate_tokens( # type: ignore[typeddict-unknown-key]
|
||||
tokenizer,
|
||||
prompt["prompt_embeds"], # type: ignore[typeddict-item]
|
||||
)
|
||||
|
||||
return prompt
|
||||
+146
-6
@@ -1,9 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import AsyncMicrobatchTokenizer
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
|
||||
from .embed_utils import safe_load_prompt_embeds
|
||||
from .params import ChatParams, TokenizeParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -14,6 +20,9 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class RendererLike(Protocol):
|
||||
config: "ModelConfig"
|
||||
_async_tokenizer: AsyncMicrobatchTokenizer
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls,
|
||||
@@ -33,16 +42,147 @@ class RendererLike(Protocol):
|
||||
|
||||
return tokenizer
|
||||
|
||||
def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:
|
||||
# Lazy initialization since offline LLM doesn't use async
|
||||
if not hasattr(self, "_async_tokenizer"):
|
||||
self._async_tokenizer = AsyncMicrobatchTokenizer(self.get_tokenizer())
|
||||
|
||||
return self._async_tokenizer
|
||||
|
||||
# Step 1: Convert raw inputs to prompts
|
||||
def render_completion(
|
||||
self,
|
||||
prompt_raw: str | list[int] | bytes,
|
||||
) -> TextPrompt | TokensPrompt | EmbedsPrompt:
|
||||
error_msg = "Each prompt must be a string or an array of tokens"
|
||||
|
||||
if isinstance(prompt_raw, str):
|
||||
return TextPrompt(prompt=prompt_raw)
|
||||
|
||||
if isinstance(prompt_raw, list):
|
||||
if not is_list_of(prompt_raw, int):
|
||||
raise TypeError(error_msg)
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
|
||||
if isinstance(prompt_raw, bytes):
|
||||
embeds = safe_load_prompt_embeds(self.config, prompt_raw)
|
||||
return EmbedsPrompt(prompt_embeds=embeds)
|
||||
|
||||
raise TypeError(error_msg)
|
||||
|
||||
def render_completions(
|
||||
self,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
prompts_raw = list[str | list[int] | bytes]()
|
||||
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
if isinstance(prompt_embeds, bytes):
|
||||
prompts_raw.append(prompt_embeds)
|
||||
else:
|
||||
prompts_raw.extend(prompt_embeds)
|
||||
|
||||
if prompt_input is not None:
|
||||
if isinstance(prompt_input, str) or (
|
||||
len(prompt_input) > 0 and is_list_of(prompt_input, int)
|
||||
):
|
||||
prompts_raw.append(prompt_input) # type: ignore[arg-type]
|
||||
else:
|
||||
prompts_raw.extend(prompt_input) # type: ignore[arg-type]
|
||||
|
||||
if len(prompts_raw) == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
return [self.render_completion(prompt) for prompt in prompts_raw]
|
||||
|
||||
async def render_completions_async(
|
||||
self,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
return self.render_completions(prompt_input, prompt_embeds)
|
||||
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list["ChatCompletionMessageParam"],
|
||||
**kwargs,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list["ChatCompletionMessageParam"],
|
||||
**kwargs,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:
|
||||
return self.render_messages(messages, **kwargs)
|
||||
params: ChatParams,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
return self.render_messages(messages, params)
|
||||
|
||||
# Step 2: Tokenize prompts if necessary
|
||||
def tokenize_prompt(
|
||||
self,
|
||||
prompt: TextPrompt | TokensPrompt | EmbedsPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt | EmbedsPrompt:
|
||||
if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt)
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt_token_ids = tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt_text = tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
|
||||
prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
def tokenize_prompts(
|
||||
self,
|
||||
prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],
|
||||
params: TokenizeParams,
|
||||
) -> list[TokensPrompt | EmbedsPrompt]:
|
||||
return [self.tokenize_prompt(prompt, params) for prompt in prompts]
|
||||
|
||||
async def tokenize_prompt_async(
|
||||
self,
|
||||
prompt: TextPrompt | TokensPrompt | EmbedsPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt | EmbedsPrompt:
|
||||
if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt)
|
||||
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt_token_ids = await tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt_text = await tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
|
||||
prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
async def tokenize_prompts_async(
|
||||
self,
|
||||
prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],
|
||||
params: TokenizeParams,
|
||||
) -> list[TokensPrompt | EmbedsPrompt]:
|
||||
return await asyncio.gather(
|
||||
*(self.tokenize_prompt_async(prompt, params) for prompt in prompts)
|
||||
)
|
||||
|
||||
@@ -9,10 +9,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .params import ChatParams
|
||||
from .protocol import RendererLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -45,8 +46,8 @@ class TerratorchRenderer(RendererLike):
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
model_config = self.config
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
@@ -55,7 +56,7 @@ class TerratorchRenderer(RendererLike):
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=[1])
|
||||
prompt = self.render_completion([1]) # Dummy token IDs
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -66,8 +67,8 @@ class TerratorchRenderer(RendererLike):
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
**kwargs,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
model_config = self.config
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
@@ -76,7 +77,7 @@ class TerratorchRenderer(RendererLike):
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=[1]) # Dummy token IDs
|
||||
prompt = self.render_completion([1]) # Dummy token IDs
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
Reference in New Issue
Block a user