[Refactor] Consolidate sequence normalization and enc-dec parsing (#33928)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
from .protocol import BaseRenderer
|
||||
|
||||
@@ -61,7 +62,7 @@ class DeepseekV32Renderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -75,7 +76,7 @@ class DeepseekV32Renderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -87,7 +88,7 @@ class DeepseekV32Renderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -101,7 +102,7 @@ class DeepseekV32Renderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
@@ -9,11 +9,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.grok2 import Grok2Tokenizer
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
from .protocol import BaseRenderer
|
||||
|
||||
@@ -61,7 +62,7 @@ class Grok2Renderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -75,7 +76,7 @@ class Grok2Renderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -87,7 +88,7 @@ class Grok2Renderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -101,7 +102,7 @@ class Grok2Renderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
@@ -25,7 +25,6 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer
|
||||
@@ -33,6 +32,8 @@ from vllm.transformers_utils.chat_templates import get_chat_template_fallback_pa
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
from .protocol import BaseRenderer
|
||||
|
||||
@@ -632,7 +633,7 @@ class HfRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
model_config = self.config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
@@ -674,7 +675,7 @@ class HfRenderer(BaseRenderer):
|
||||
video_placeholder,
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -686,7 +687,7 @@ class HfRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
model_config = self.config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
@@ -726,7 +727,7 @@ class HfRenderer(BaseRenderer):
|
||||
video_placeholder,
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .preprocess import (
|
||||
DecoderDictPrompt,
|
||||
DecoderOnlyDictPrompt,
|
||||
DictPrompt,
|
||||
EncoderDecoderDictPrompt,
|
||||
EncoderDictPrompt,
|
||||
SingletonDictPrompt,
|
||||
)
|
||||
from .tokenize import (
|
||||
DecoderOnlyTokPrompt,
|
||||
DecoderTokPrompt,
|
||||
EncoderDecoderTokPrompt,
|
||||
EncoderTokPrompt,
|
||||
SingletonTokPrompt,
|
||||
TokPrompt,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DecoderOnlyDictPrompt",
|
||||
"EncoderDictPrompt",
|
||||
"DecoderDictPrompt",
|
||||
"EncoderDecoderDictPrompt",
|
||||
"SingletonDictPrompt",
|
||||
"DictPrompt",
|
||||
"DecoderOnlyTokPrompt",
|
||||
"EncoderTokPrompt",
|
||||
"DecoderTokPrompt",
|
||||
"EncoderDecoderTokPrompt",
|
||||
"SingletonTokPrompt",
|
||||
"TokPrompt",
|
||||
]
|
||||
@@ -0,0 +1,255 @@
|
||||
"""
|
||||
Schemas and utilites for preprocessing inputs.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, NamedTuple, TypeAlias, TypedDict, overload
|
||||
|
||||
from vllm.inputs import (
|
||||
EmbedsPrompt,
|
||||
ExplicitEncoderDecoderPrompt,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
TextPrompt,
|
||||
TokensPrompt,
|
||||
)
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
|
||||
|
||||
@overload
|
||||
def prompt_to_seq(
|
||||
prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],
|
||||
) -> Sequence[SingletonPrompt]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def prompt_to_seq( # type: ignore[misc]
|
||||
prompt_or_prompts: ExplicitEncoderDecoderPrompt
|
||||
| Sequence[ExplicitEncoderDecoderPrompt],
|
||||
) -> Sequence[ExplicitEncoderDecoderPrompt]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def prompt_to_seq( # type: ignore[misc]
|
||||
prompt_or_prompts: PromptType | Sequence[PromptType],
|
||||
) -> Sequence[PromptType]: ...
|
||||
|
||||
|
||||
def prompt_to_seq(
|
||||
prompt_or_prompts: PromptType | bytes | Sequence[PromptType | bytes],
|
||||
) -> Sequence[PromptType]:
|
||||
if isinstance(prompt_or_prompts, (dict, str, bytes)) or (
|
||||
len(prompt_or_prompts) > 0 and is_list_of(prompt_or_prompts, int)
|
||||
):
|
||||
return [prompt_or_prompts] # type: ignore[list-item]
|
||||
|
||||
return prompt_or_prompts # type: ignore[return-value]
|
||||
|
||||
|
||||
def conversation_to_seq(
|
||||
conversation_or_conversations: list["ChatCompletionMessageParam"]
|
||||
| Sequence[list["ChatCompletionMessageParam"]],
|
||||
) -> Sequence[list["ChatCompletionMessageParam"]]:
|
||||
if len(conversation_or_conversations) > 0 and is_list_of(
|
||||
conversation_or_conversations, dict
|
||||
):
|
||||
return [conversation_or_conversations] # type: ignore[list-item]
|
||||
|
||||
return conversation_or_conversations # type: ignore[return-value]
|
||||
|
||||
|
||||
DecoderOnlyDictPrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt
|
||||
"""
|
||||
A [`DecoderOnlyPrompt`][vllm.inputs.data.DecoderOnlyPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
EncoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt
|
||||
"""
|
||||
A [`EncoderPrompt`][vllm.inputs.data.EncoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
DecoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt
|
||||
"""
|
||||
A [`DecoderPrompt`][vllm.inputs.data.DecoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
class EncoderDecoderDictPrompt(TypedDict):
|
||||
"""
|
||||
A [`EncoderDecoderPrompt`][vllm.inputs.data.EncoderDecoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
encoder_prompt: EncoderDictPrompt
|
||||
|
||||
decoder_prompt: DecoderDictPrompt | None
|
||||
|
||||
|
||||
SingletonDictPrompt: TypeAlias = (
|
||||
DecoderOnlyDictPrompt | EncoderDictPrompt | DecoderDictPrompt
|
||||
)
|
||||
"""
|
||||
A [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
DictPrompt: TypeAlias = DecoderOnlyDictPrompt | EncoderDecoderDictPrompt
|
||||
"""
|
||||
A [`PromptType`][vllm.inputs.data.PromptType]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
def parse_dec_only_prompt(prompt: object) -> DecoderOnlyDictPrompt:
|
||||
"""
|
||||
Parse a prompt for a decoder-only model and normalize it to a dictionary.
|
||||
"""
|
||||
if isinstance(prompt, str):
|
||||
return TextPrompt(prompt=prompt)
|
||||
|
||||
if isinstance(prompt, list):
|
||||
if not is_list_of(prompt, int):
|
||||
raise TypeError("Token prompt should be a list of integers")
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt)
|
||||
|
||||
if isinstance(prompt, dict):
|
||||
if "encoder_prompt" in prompt:
|
||||
raise TypeError("Cannot pass encoder-decoder prompt to decoder-only models")
|
||||
|
||||
if (
|
||||
"prompt" in prompt
|
||||
or "prompt_token_ids" in prompt
|
||||
or "prompt_embeds" in prompt
|
||||
):
|
||||
return prompt # type: ignore[return-value]
|
||||
|
||||
raise TypeError("Prompt dictionary must contain text, tokens, or embeddings")
|
||||
|
||||
raise TypeError("Prompt should be a string, list of tokens, or dictionary")
|
||||
|
||||
|
||||
def _parse_enc_prompt(prompt: object) -> EncoderDictPrompt:
|
||||
if isinstance(prompt, str):
|
||||
return TextPrompt(prompt=prompt)
|
||||
|
||||
if isinstance(prompt, list):
|
||||
if not is_list_of(prompt, int):
|
||||
raise TypeError("Token prompt should be a list of integers")
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt)
|
||||
|
||||
if isinstance(prompt, dict):
|
||||
if "prompt_embeds" in prompt:
|
||||
raise TypeError("Cannot pass embeddings prompt to encoder-decoder models")
|
||||
|
||||
if "prompt" in prompt or "prompt_token_ids" in prompt:
|
||||
return prompt # type: ignore[return-value]
|
||||
|
||||
raise TypeError("Prompt dictionary must contain text or tokens")
|
||||
|
||||
raise TypeError("Prompt should be a string, list of tokens, or dictionary")
|
||||
|
||||
|
||||
def _parse_dec_prompt(prompt: object) -> DecoderDictPrompt:
|
||||
if isinstance(prompt, str):
|
||||
return TextPrompt(prompt=prompt)
|
||||
|
||||
if isinstance(prompt, list):
|
||||
if not is_list_of(prompt, int):
|
||||
raise TypeError("Token prompt should be a list of integers")
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt)
|
||||
|
||||
if isinstance(prompt, dict):
|
||||
if "prompt_embeds" in prompt:
|
||||
raise TypeError("Cannot pass embeddings prompt to encoder-decoder models")
|
||||
|
||||
if (
|
||||
"multi_modal_data" in prompt
|
||||
or "mm_processor_kwargs" in prompt
|
||||
or "multi_modal_uuids" in prompt
|
||||
):
|
||||
raise TypeError("Cannot pass multi-modal inputs to decoder prompt")
|
||||
|
||||
if "prompt" in prompt or "prompt_token_ids" in prompt:
|
||||
return prompt # type: ignore[return-value]
|
||||
|
||||
raise TypeError("Prompt dictionary must contain text or tokens")
|
||||
|
||||
raise TypeError("Prompt should be a string, list of tokens, or dictionary")
|
||||
|
||||
|
||||
def parse_enc_dec_prompt(prompt: object) -> EncoderDecoderDictPrompt:
|
||||
"""
|
||||
Parse a prompt for an encoder-decoder model and normalize it to a dictionary.
|
||||
"""
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
enc_prompt: object = prompt["encoder_prompt"] # type: ignore[typeddict-item]
|
||||
dec_prompt: object | None = prompt["decoder_prompt"] # type: ignore[typeddict-item]
|
||||
else:
|
||||
enc_prompt = prompt
|
||||
dec_prompt = None
|
||||
|
||||
return EncoderDecoderDictPrompt(
|
||||
encoder_prompt=_parse_enc_prompt(enc_prompt),
|
||||
decoder_prompt=None if dec_prompt is None else _parse_dec_prompt(dec_prompt),
|
||||
)
|
||||
|
||||
|
||||
def parse_model_prompt(model_config: "ModelConfig", prompt: object):
|
||||
if model_config.is_encoder_decoder:
|
||||
return parse_enc_dec_prompt(prompt)
|
||||
|
||||
return parse_dec_only_prompt(prompt)
|
||||
|
||||
|
||||
class PromptComponents(NamedTuple):
|
||||
text: str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
embeds: "torch.Tensor | None" = None
|
||||
|
||||
|
||||
def extract_prompt_components(
|
||||
model_config: "ModelConfig",
|
||||
prompt: object,
|
||||
) -> PromptComponents:
|
||||
target_prompt = (
|
||||
parse_enc_dec_prompt(prompt)["encoder_prompt"]
|
||||
if model_config.is_encoder_decoder
|
||||
else parse_dec_only_prompt(prompt)
|
||||
)
|
||||
|
||||
return PromptComponents(
|
||||
text=target_prompt.get("prompt"),
|
||||
token_ids=target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]
|
||||
embeds=target_prompt.get("prompt_embeds"),
|
||||
)
|
||||
|
||||
|
||||
def extract_prompt_len(model_config: "ModelConfig", prompt: object):
|
||||
target_prompt = (
|
||||
parse_enc_dec_prompt(prompt)["encoder_prompt"]
|
||||
if model_config.is_encoder_decoder
|
||||
else parse_dec_only_prompt(prompt)
|
||||
)
|
||||
|
||||
return length_from_prompt_token_ids_or_embeds(
|
||||
target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]
|
||||
target_prompt.get("prompt_embeds"),
|
||||
)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
Schemas and utilites for tokenization inputs.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeAlias, TypedDict
|
||||
|
||||
from vllm.inputs import EmbedsPrompt, TokensPrompt
|
||||
|
||||
DecoderOnlyTokPrompt: TypeAlias = TokensPrompt | EmbedsPrompt
|
||||
"""
|
||||
A [`DecoderOnlyDictPrompt`][vllm.renderers.inputs.preprocess.DecoderOnlyDictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
|
||||
|
||||
EncoderTokPrompt: TypeAlias = TokensPrompt
|
||||
"""
|
||||
A [`EncoderDictPrompt`][vllm.renderers.inputs.preprocess.EncoderDictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
|
||||
|
||||
DecoderTokPrompt: TypeAlias = TokensPrompt
|
||||
"""
|
||||
A [`DecoderDictPrompt`][vllm.renderers.inputs.preprocess.DecoderDictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
|
||||
|
||||
class EncoderDecoderTokPrompt(TypedDict):
|
||||
"""
|
||||
A
|
||||
[`EncoderDecoderDictPrompt`][vllm.renderers.inputs.preprocess.EncoderDecoderDictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
|
||||
encoder_prompt: EncoderTokPrompt
|
||||
|
||||
decoder_prompt: DecoderTokPrompt | None
|
||||
|
||||
|
||||
SingletonTokPrompt: TypeAlias = (
|
||||
DecoderOnlyTokPrompt | EncoderTokPrompt | DecoderTokPrompt
|
||||
)
|
||||
"""
|
||||
A [`SingletonDictPrompt`][vllm.renderers.inputs.preprocess.SingletonDictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
|
||||
|
||||
TokPrompt: TypeAlias = DecoderOnlyTokPrompt | EncoderDecoderTokPrompt
|
||||
"""
|
||||
A [`DictPrompt`][vllm.renderers.inputs.preprocess.DictPrompt]
|
||||
that has been tokenized.
|
||||
"""
|
||||
@@ -10,12 +10,13 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.utils.async_utils import make_async
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
from .protocol import BaseRenderer
|
||||
|
||||
@@ -95,7 +96,7 @@ class MistralRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
@@ -109,7 +110,7 @@ class MistralRenderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -121,7 +122,7 @@ class MistralRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
tokenizer = self.get_tokenizer()
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
@@ -135,7 +136,7 @@ class MistralRenderer(BaseRenderer):
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
)
|
||||
|
||||
prompt = self.render_completion(prompt_raw)
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
+162
-79
@@ -2,14 +2,20 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, overload
|
||||
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import AsyncMicrobatchTokenizer
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
|
||||
from .embed_utils import safe_load_prompt_embeds
|
||||
from .inputs import (
|
||||
DictPrompt,
|
||||
EncoderDecoderDictPrompt,
|
||||
EncoderDecoderTokPrompt,
|
||||
TokPrompt,
|
||||
)
|
||||
from .params import ChatParams, TokenizeParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -57,140 +63,217 @@ class BaseRenderer(ABC):
|
||||
return self._async_tokenizer
|
||||
|
||||
# Step 1: Convert raw inputs to prompts
|
||||
def render_completion(
|
||||
def render_prompt(
|
||||
self,
|
||||
prompt_raw: str | list[int] | bytes,
|
||||
) -> TextPrompt | TokensPrompt | EmbedsPrompt:
|
||||
error_msg = "Each prompt must be a string or an array of tokens"
|
||||
prompt: DictPrompt | bytes,
|
||||
) -> DictPrompt:
|
||||
if isinstance(prompt, bytes):
|
||||
embeds = safe_load_prompt_embeds(self.config, prompt)
|
||||
prompt = EmbedsPrompt(prompt_embeds=embeds)
|
||||
|
||||
if isinstance(prompt_raw, str):
|
||||
return TextPrompt(prompt=prompt_raw)
|
||||
return prompt
|
||||
|
||||
if isinstance(prompt_raw, list):
|
||||
if not is_list_of(prompt_raw, int):
|
||||
raise TypeError(error_msg)
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt_raw)
|
||||
|
||||
if isinstance(prompt_raw, bytes):
|
||||
embeds = safe_load_prompt_embeds(self.config, prompt_raw)
|
||||
return EmbedsPrompt(prompt_embeds=embeds)
|
||||
|
||||
raise TypeError(error_msg)
|
||||
|
||||
def render_completions(
|
||||
def render_prompts(
|
||||
self,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
prompts_raw = list[str | list[int] | bytes]()
|
||||
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
if isinstance(prompt_embeds, bytes):
|
||||
prompts_raw.append(prompt_embeds)
|
||||
else:
|
||||
prompts_raw.extend(prompt_embeds)
|
||||
|
||||
if prompt_input is not None:
|
||||
if isinstance(prompt_input, str) or (
|
||||
len(prompt_input) > 0 and is_list_of(prompt_input, int)
|
||||
):
|
||||
prompts_raw.append(prompt_input) # type: ignore[arg-type]
|
||||
else:
|
||||
prompts_raw.extend(prompt_input) # type: ignore[arg-type]
|
||||
|
||||
if len(prompts_raw) == 0:
|
||||
prompts: Sequence[DictPrompt | bytes],
|
||||
) -> list[DictPrompt]:
|
||||
if len(prompts) == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
return [self.render_completion(prompt) for prompt in prompts_raw]
|
||||
return [self.render_prompt(prompt) for prompt in prompts]
|
||||
|
||||
async def render_completions_async(
|
||||
async def render_prompts_async(
|
||||
self,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
) -> list[TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
return self.render_completions(prompt_input, prompt_embeds)
|
||||
prompts: Sequence[DictPrompt | bytes],
|
||||
) -> list[DictPrompt]:
|
||||
return self.render_prompts(prompts)
|
||||
|
||||
@abstractmethod
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list["ChatCompletionMessageParam"],
|
||||
params: ChatParams,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list["ConversationMessage"], DictPrompt]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list["ChatCompletionMessageParam"],
|
||||
params: ChatParams,
|
||||
) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list["ConversationMessage"], DictPrompt]:
|
||||
return self.render_messages(messages, params)
|
||||
|
||||
# Step 2: Tokenize prompts if necessary
|
||||
def _tokenize_prompt(
|
||||
self,
|
||||
prompt: TextPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt:
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt_token_ids = tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
|
||||
async def _tokenize_prompt_async(
|
||||
self,
|
||||
prompt: TextPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt:
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt_token_ids = await tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
return TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
|
||||
def _detokenize_prompt(self, prompt: TokensPrompt) -> TokensPrompt:
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt["prompt"] = tokenizer.decode(prompt["prompt_token_ids"])
|
||||
|
||||
return prompt
|
||||
|
||||
async def _detokenize_prompt_async(self, prompt: TokensPrompt) -> TokensPrompt:
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt["prompt"] = await tokenizer.decode(prompt["prompt_token_ids"])
|
||||
|
||||
return prompt
|
||||
|
||||
def _tokenize_enc_dec_prompt(
|
||||
self,
|
||||
prompt: EncoderDecoderDictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EncoderDecoderTokPrompt:
|
||||
enc_prompt, dec_prompt = (
|
||||
self.tokenize_prompt(prompt["encoder_prompt"], params),
|
||||
(
|
||||
None
|
||||
if prompt["decoder_prompt"] is None
|
||||
else self.tokenize_prompt(prompt["decoder_prompt"], params)
|
||||
),
|
||||
)
|
||||
|
||||
return EncoderDecoderTokPrompt(
|
||||
encoder_prompt=enc_prompt,
|
||||
decoder_prompt=dec_prompt,
|
||||
)
|
||||
|
||||
async def _tokenize_enc_dec_prompt_async(
|
||||
self,
|
||||
prompt: EncoderDecoderDictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EncoderDecoderTokPrompt:
|
||||
enc_prompt, dec_prompt = await asyncio.gather(
|
||||
self.tokenize_prompt_async(prompt["encoder_prompt"], params),
|
||||
(
|
||||
asyncio.sleep(0)
|
||||
if prompt["decoder_prompt"] is None
|
||||
else self.tokenize_prompt_async(prompt["decoder_prompt"], params)
|
||||
),
|
||||
)
|
||||
|
||||
return EncoderDecoderTokPrompt(
|
||||
encoder_prompt=enc_prompt,
|
||||
decoder_prompt=dec_prompt,
|
||||
)
|
||||
|
||||
@overload
|
||||
def tokenize_prompt(
|
||||
self,
|
||||
prompt: TextPrompt | TokensPrompt | EmbedsPrompt,
|
||||
prompt: TextPrompt | TokensPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt | EmbedsPrompt:
|
||||
) -> TokensPrompt: ...
|
||||
|
||||
@overload
|
||||
def tokenize_prompt( # type: ignore[misc]
|
||||
self,
|
||||
prompt: EmbedsPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EmbedsPrompt: ...
|
||||
|
||||
@overload
|
||||
def tokenize_prompt( # type: ignore[misc]
|
||||
self,
|
||||
prompt: EncoderDecoderDictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EncoderDecoderTokPrompt: ...
|
||||
|
||||
def tokenize_prompt(
|
||||
self,
|
||||
prompt: DictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokPrompt:
|
||||
if "encoder_prompt" in prompt:
|
||||
return self._tokenize_enc_dec_prompt(prompt, params) # type: ignore[arg-type]
|
||||
|
||||
if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt)
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt_token_ids = tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
prompt = self._tokenize_prompt(prompt, params)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
prompt_text = tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
|
||||
prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]
|
||||
prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
def tokenize_prompts(
|
||||
self,
|
||||
prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],
|
||||
prompts: Sequence[DictPrompt],
|
||||
params: TokenizeParams,
|
||||
) -> list[TokensPrompt | EmbedsPrompt]:
|
||||
) -> list[TokPrompt]:
|
||||
return [self.tokenize_prompt(prompt, params) for prompt in prompts]
|
||||
|
||||
@overload
|
||||
async def tokenize_prompt_async(
|
||||
self,
|
||||
prompt: TextPrompt | TokensPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt: ...
|
||||
|
||||
@overload
|
||||
async def tokenize_prompt_async( # type: ignore[misc]
|
||||
self,
|
||||
prompt: EmbedsPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EmbedsPrompt: ...
|
||||
|
||||
@overload
|
||||
async def tokenize_prompt_async( # type: ignore[misc]
|
||||
self,
|
||||
prompt: EncoderDecoderDictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> EncoderDecoderTokPrompt: ...
|
||||
|
||||
async def tokenize_prompt_async(
|
||||
self,
|
||||
prompt: TextPrompt | TokensPrompt | EmbedsPrompt,
|
||||
prompt: DictPrompt,
|
||||
params: TokenizeParams,
|
||||
) -> TokensPrompt | EmbedsPrompt:
|
||||
) -> TokPrompt:
|
||||
if "encoder_prompt" in prompt:
|
||||
return await self._tokenize_enc_dec_prompt_async(prompt, params) # type: ignore[arg-type]
|
||||
|
||||
if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
|
||||
prompt = params.apply_pre_tokenization(self.tokenizer, prompt)
|
||||
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt_token_ids = await tokenizer.encode(
|
||||
prompt["prompt"],
|
||||
**params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, **prompt)
|
||||
prompt = await self._tokenize_prompt_async(prompt, params)
|
||||
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
tokenizer = self.get_async_tokenizer()
|
||||
prompt_text = await tokenizer.decode(prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
|
||||
prompt["prompt"] = prompt_text # type: ignore[typeddict-unknown-key]
|
||||
prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
async def tokenize_prompts_async(
|
||||
self,
|
||||
prompts: list[TextPrompt | TokensPrompt | EmbedsPrompt],
|
||||
prompts: Sequence[DictPrompt],
|
||||
params: TokenizeParams,
|
||||
) -> list[TokensPrompt | EmbedsPrompt]:
|
||||
) -> list[TokPrompt]:
|
||||
return await asyncio.gather(
|
||||
*(self.tokenize_prompt_async(prompt, params) for prompt in prompts)
|
||||
)
|
||||
|
||||
@@ -9,10 +9,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
from .protocol import BaseRenderer
|
||||
|
||||
@@ -45,7 +46,7 @@ class TerratorchRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
model_config = self.config
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
@@ -54,7 +55,7 @@ class TerratorchRenderer(BaseRenderer):
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
prompt = self.render_completion([1]) # Dummy token IDs
|
||||
prompt = parse_dec_only_prompt([1]) # Dummy token IDs
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -66,7 +67,7 @@ class TerratorchRenderer(BaseRenderer):
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt | EmbedsPrompt]:
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
model_config = self.config
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
@@ -75,7 +76,7 @@ class TerratorchRenderer(BaseRenderer):
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
prompt = self.render_completion([1]) # Dummy token IDs
|
||||
prompt = parse_dec_only_prompt([1]) # Dummy token IDs
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
|
||||
Reference in New Issue
Block a user