[Frontend] Use new Renderer for Completions and Tokenize API (#32863)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+12
-42
@@ -21,12 +21,7 @@ else:
|
||||
MultiModalUUIDDict = object
|
||||
|
||||
|
||||
class TextPrompt(TypedDict):
|
||||
"""Schema for a text prompt."""
|
||||
|
||||
prompt: str
|
||||
"""The input text to be tokenized before passing to the model."""
|
||||
|
||||
class _CommonKeys(TypedDict):
|
||||
multi_modal_data: NotRequired[MultiModalDataDict | None]
|
||||
"""
|
||||
Optional multi-modal data to pass to the model,
|
||||
@@ -56,7 +51,14 @@ class TextPrompt(TypedDict):
|
||||
"""
|
||||
|
||||
|
||||
class TokensPrompt(TypedDict):
|
||||
class TextPrompt(_CommonKeys):
|
||||
"""Schema for a text prompt."""
|
||||
|
||||
prompt: str
|
||||
"""The input text to be tokenized before passing to the model."""
|
||||
|
||||
|
||||
class TokensPrompt(_CommonKeys):
|
||||
"""Schema for a tokenized prompt."""
|
||||
|
||||
prompt_token_ids: list[int]
|
||||
@@ -68,47 +70,15 @@ class TokensPrompt(TypedDict):
|
||||
token_type_ids: NotRequired[list[int]]
|
||||
"""A list of token type IDs to pass to the cross encoder model."""
|
||||
|
||||
multi_modal_data: NotRequired[MultiModalDataDict | None]
|
||||
"""
|
||||
Optional multi-modal data to pass to the model,
|
||||
if the model supports it.
|
||||
"""
|
||||
|
||||
mm_processor_kwargs: NotRequired[dict[str, Any] | None]
|
||||
"""
|
||||
Optional multi-modal processor kwargs to be forwarded to the
|
||||
multimodal input mapper & processor. Note that if multiple modalities
|
||||
have registered mappers etc for the model being considered, we attempt
|
||||
to pass the mm_processor_kwargs to each of them.
|
||||
"""
|
||||
|
||||
multi_modal_uuids: NotRequired[MultiModalUUIDDict]
|
||||
"""
|
||||
Optional user-specified UUIDs for multimodal items, mapped by modality.
|
||||
Lists must match the number of items per modality and may contain `None`.
|
||||
For `None` entries, the hasher will compute IDs automatically; non-None
|
||||
entries override the default hashes for caching.
|
||||
"""
|
||||
|
||||
cache_salt: NotRequired[str]
|
||||
"""
|
||||
Optional cache salt to be used for prefix caching.
|
||||
"""
|
||||
|
||||
|
||||
class EmbedsPrompt(TypedDict):
|
||||
class EmbedsPrompt(_CommonKeys):
|
||||
"""Schema for a prompt provided via token embeddings."""
|
||||
|
||||
prompt_embeds: torch.Tensor
|
||||
"""The embeddings of the prompt."""
|
||||
|
||||
cache_salt: NotRequired[str]
|
||||
"""
|
||||
Optional cache salt to be used for prefix caching.
|
||||
"""
|
||||
|
||||
|
||||
class DataPrompt(TypedDict):
|
||||
class DataPrompt(_CommonKeys):
|
||||
"""Represents generic inputs handled by IO processor plugins."""
|
||||
|
||||
data: Any
|
||||
@@ -197,7 +167,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
|
||||
mm_processor_kwargs: NotRequired[dict[str, Any]]
|
||||
|
||||
|
||||
PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt
|
||||
PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt[Any, Any]
|
||||
"""
|
||||
Set of possible schemas for an LLM input, including
|
||||
both decoder-only and encoder/decoder input types:
|
||||
|
||||
+9
-47
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict, cast
|
||||
from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict
|
||||
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
|
||||
from .data import (
|
||||
EmbedsPrompt,
|
||||
@@ -22,50 +21,6 @@ if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
|
||||
def parse_raw_prompts(
|
||||
prompt: str | list[str] | list[int] | list[list[int]],
|
||||
) -> Sequence[TextPrompt] | Sequence[TokensPrompt]:
|
||||
if isinstance(prompt, str):
|
||||
# case 1: a string
|
||||
return [TextPrompt(prompt=prompt)]
|
||||
|
||||
if isinstance(prompt, list):
|
||||
if len(prompt) == 0:
|
||||
raise ValueError("please provide at least one prompt")
|
||||
|
||||
# case 2: array of strings
|
||||
if is_list_of(prompt, str):
|
||||
prompt = cast(list[str], prompt)
|
||||
return [TextPrompt(prompt=elem) for elem in prompt]
|
||||
|
||||
# case 3: array of tokens
|
||||
if is_list_of(prompt, int):
|
||||
prompt = cast(list[int], prompt)
|
||||
return [TokensPrompt(prompt_token_ids=prompt)]
|
||||
|
||||
# case 4: array of token arrays
|
||||
if is_list_of(prompt, list):
|
||||
if len(prompt) == 1 and isinstance(prompt[0], list) and len(prompt[0]) == 0:
|
||||
raise ValueError("please provide at least one prompt")
|
||||
for elem in prompt:
|
||||
if not isinstance(elem, list):
|
||||
raise TypeError(
|
||||
"prompt must be a list of lists, but found a non-list element."
|
||||
)
|
||||
if not is_list_of(elem, int):
|
||||
raise TypeError(
|
||||
"Nested lists of tokens must contain only integers."
|
||||
)
|
||||
|
||||
prompt = cast(list[list[int]], prompt)
|
||||
return [TokensPrompt(prompt_token_ids=elem) for elem in prompt]
|
||||
|
||||
raise TypeError(
|
||||
"prompt must be a string, array of strings, "
|
||||
"array of tokens, or array of token arrays"
|
||||
)
|
||||
|
||||
|
||||
class ParsedStrPrompt(TypedDict):
|
||||
type: Literal["str"]
|
||||
content: str
|
||||
@@ -145,3 +100,10 @@ def get_prompt_components(prompt: PromptType) -> PromptComponents:
|
||||
token_ids=prompt.get("prompt_token_ids"), # type: ignore[arg-type]
|
||||
embeds=prompt.get("prompt_embeds"),
|
||||
)
|
||||
|
||||
|
||||
def get_prompt_len(prompt: TokensPrompt | EmbedsPrompt):
|
||||
return length_from_prompt_token_ids_or_embeds(
|
||||
prompt.get("prompt_token_ids"), # type: ignore[arg-type]
|
||||
prompt.get("prompt_embeds"), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user