[Core] Support serving encoder/decoder models (#7258)
This commit is contained in:
+74
-137
@@ -1,71 +1,12 @@
|
||||
from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
|
||||
TypedDict, Union, cast, overload)
|
||||
from typing import (TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple,
|
||||
Union)
|
||||
|
||||
from typing_extensions import NotRequired
|
||||
from typing_extensions import NotRequired, TypedDict, TypeVar
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
|
||||
|
||||
class ParsedText(TypedDict):
|
||||
content: str
|
||||
is_tokens: Literal[False]
|
||||
|
||||
|
||||
class ParsedTokens(TypedDict):
|
||||
content: List[int]
|
||||
is_tokens: Literal[True]
|
||||
|
||||
|
||||
# https://github.com/vllm-project/vllm/pull/4028
|
||||
@overload
|
||||
def parse_and_batch_prompt(
|
||||
prompt: Union[str, List[str]]) -> Sequence[ParsedText]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def parse_and_batch_prompt(
|
||||
prompt: Union[List[int], List[List[int]]]) -> Sequence[ParsedTokens]:
|
||||
...
|
||||
|
||||
|
||||
def parse_and_batch_prompt(
|
||||
prompt: Union[str, List[str], List[int], List[List[int]]],
|
||||
) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
|
||||
if isinstance(prompt, str):
|
||||
# case 1: a string
|
||||
return [ParsedText(content=prompt, is_tokens=False)]
|
||||
|
||||
if isinstance(prompt, list):
|
||||
if len(prompt) == 0:
|
||||
raise ValueError("please provide at least one prompt")
|
||||
|
||||
if isinstance(prompt[0], str):
|
||||
# case 2: array of strings
|
||||
return [
|
||||
ParsedText(content=elem, is_tokens=False)
|
||||
for elem in cast(List[str], prompt)
|
||||
]
|
||||
if isinstance(prompt[0], int):
|
||||
# case 3: array of tokens
|
||||
elem = cast(List[int], prompt)
|
||||
return [ParsedTokens(content=elem, is_tokens=True)]
|
||||
if isinstance(prompt[0], list):
|
||||
if len(prompt[0]) == 0:
|
||||
raise ValueError("please provide at least one prompt")
|
||||
|
||||
if isinstance(prompt[0][0], int):
|
||||
# case 4: array of token arrays
|
||||
return [
|
||||
ParsedTokens(content=elem, is_tokens=True)
|
||||
for elem in cast(List[List[int]], prompt)
|
||||
]
|
||||
|
||||
raise ValueError("prompt must be a string, array of strings, "
|
||||
"array of tokens, or array of token arrays")
|
||||
|
||||
|
||||
class TextPrompt(TypedDict):
|
||||
"""Schema for a text prompt."""
|
||||
|
||||
@@ -103,39 +44,49 @@ Note that "singleton" is as opposed to a data structure
|
||||
which encapsulates multiple prompts, i.e. of the sort
|
||||
which may be utilized for encoder/decoder models when
|
||||
the user desires to express both the encoder & decoder
|
||||
prompts explicitly, i.e. ExplicitEncoderDecoderPrompt
|
||||
prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
|
||||
|
||||
A prompt of type SingletonPromptInputs may be employed
|
||||
A prompt of type :class:`SingletonPromptInputs` may be employed
|
||||
as (1) input to a decoder-only model, (2) input to
|
||||
the encoder of an encoder/decoder model, in the scenario
|
||||
where the decoder-prompt is not specified explicitly, or
|
||||
(3) as a member of a larger data structure encapsulating
|
||||
more than one prompt, i.e. ExplicitEncoderDecoderPrompt
|
||||
more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
|
||||
"""
|
||||
|
||||
_T1_co = TypeVar("_T1_co",
|
||||
bound=SingletonPromptInputs,
|
||||
default=SingletonPromptInputs,
|
||||
covariant=True)
|
||||
_T2_co = TypeVar("_T2_co",
|
||||
bound=SingletonPromptInputs,
|
||||
default=SingletonPromptInputs,
|
||||
covariant=True)
|
||||
|
||||
class ExplicitEncoderDecoderPrompt(TypedDict):
|
||||
|
||||
# TODO: Make fields ReadOnly once mypy supports it
|
||||
class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
|
||||
"""Represents an encoder/decoder model input prompt,
|
||||
comprising an explicit encoder prompt and a
|
||||
decoder prompt.
|
||||
|
||||
The encoder and decoder prompts, respectively,
|
||||
may formatted according to any of the
|
||||
SingletonPromptInputs schemas, and are not
|
||||
:class:`SingletonPromptInputs` schemas, and are not
|
||||
required to have the same schema.
|
||||
|
||||
Only the encoder prompt may have multi-modal data.
|
||||
|
||||
Note that an ExplicitEncoderDecoderPrompt may not
|
||||
Note that an :class:`ExplicitEncoderDecoderPrompt` may not
|
||||
be used as an input to a decoder-only model,
|
||||
and that the `encoder_prompt` and `decoder_prompt`
|
||||
fields of this data structure may not themselves
|
||||
must be SingletonPromptInputs instances.
|
||||
fields of this data structure themselves must be
|
||||
:class:`SingletonPromptInputs` instances.
|
||||
"""
|
||||
|
||||
encoder_prompt: SingletonPromptInputs
|
||||
encoder_prompt: _T1_co
|
||||
|
||||
decoder_prompt: SingletonPromptInputs
|
||||
decoder_prompt: Optional[_T2_co]
|
||||
|
||||
|
||||
PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
|
||||
@@ -150,60 +101,12 @@ both decoder-only and encoder/decoder input types:
|
||||
"""
|
||||
|
||||
|
||||
def _has_required_keys(
|
||||
d: dict,
|
||||
required_keys: set,
|
||||
) -> bool:
|
||||
return required_keys.issubset(d.keys())
|
||||
|
||||
|
||||
def get_prompt_type(prompt: Optional[PromptInputs]) -> Optional[str]:
|
||||
"""
|
||||
Get the type-name of the prompt argument instance, given that
|
||||
isinstance() cannot apply to TypedDict subclasses directly.
|
||||
If the prompt is None, return 'None' as the type name.
|
||||
|
||||
Arguments:
|
||||
|
||||
* prompt: LLM input prompt or None
|
||||
|
||||
Returns:
|
||||
|
||||
* String representation of prompt type
|
||||
"""
|
||||
|
||||
if prompt is None:
|
||||
return 'None'
|
||||
|
||||
required_keys_dict = {
|
||||
'TextPrompt': {'prompt'},
|
||||
'TokensPrompt': {'prompt_token_ids'},
|
||||
'ExplicitEncoderDecoder': {'encoder_prompt', 'decoder_prompt'},
|
||||
}
|
||||
|
||||
if isinstance(prompt, dict):
|
||||
for (ptype, required_keys) in required_keys_dict.items():
|
||||
# Ignore type checking in the conditional below because type
|
||||
# checker does not understand that is_dict(prompt) narrows
|
||||
# down the possible types
|
||||
if _has_required_keys(
|
||||
prompt, # type: ignore
|
||||
required_keys):
|
||||
return ptype
|
||||
|
||||
raise ValueError(f"Invalid prompt {prompt}, valid types are "
|
||||
"required_keys_dict={required_keys_dict}")
|
||||
|
||||
if isinstance(prompt, str):
|
||||
return "str"
|
||||
|
||||
raise ValueError(f"Invalid prompt {prompt}")
|
||||
|
||||
|
||||
class LLMInputs(TypedDict):
|
||||
"""
|
||||
The inputs in :class:`~vllm.LLMEngine` before they are
|
||||
passed to the model executor.
|
||||
|
||||
This specifies the data required for decoder-only models.
|
||||
"""
|
||||
prompt_token_ids: List[int]
|
||||
"""The token IDs of the prompt."""
|
||||
@@ -213,7 +116,21 @@ class LLMInputs(TypedDict):
|
||||
The original prompt text corresponding to the token IDs, if available.
|
||||
"""
|
||||
|
||||
encoder_prompt_token_ids: NotRequired[List[int]]
|
||||
multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
|
||||
"""
|
||||
Optional multi-modal data to pass to the model,
|
||||
if the model supports it.
|
||||
"""
|
||||
|
||||
|
||||
class EncoderDecoderLLMInputs(LLMInputs):
|
||||
"""
|
||||
The inputs in :class:`~vllm.LLMEngine` before they are
|
||||
passed to the model executor.
|
||||
|
||||
This specifies the required data for encoder-decoder models.
|
||||
"""
|
||||
encoder_prompt_token_ids: List[int]
|
||||
"""The token IDs of the encoder prompt."""
|
||||
|
||||
encoder_prompt: NotRequired[Optional[str]]
|
||||
@@ -222,20 +139,40 @@ class LLMInputs(TypedDict):
|
||||
available.
|
||||
"""
|
||||
|
||||
multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
|
||||
"""
|
||||
Optional multi-modal data to pass to the model,
|
||||
if the model supports it.
|
||||
"""
|
||||
|
||||
_T1 = TypeVar("_T1",
|
||||
bound=SingletonPromptInputs,
|
||||
default=SingletonPromptInputs)
|
||||
_T2 = TypeVar("_T2",
|
||||
bound=SingletonPromptInputs,
|
||||
default=SingletonPromptInputs)
|
||||
|
||||
|
||||
def is_valid_encoder_decoder_llm_inputs(inputs: LLMInputs) -> bool:
|
||||
"""
|
||||
Return True if the LLMInputs instance has the correct configuration
|
||||
for encoder/decoder.
|
||||
"""
|
||||
def build_explicit_enc_dec_prompt(
|
||||
encoder_prompt: _T1,
|
||||
decoder_prompt: Optional[_T2],
|
||||
) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
|
||||
return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
|
||||
decoder_prompt=decoder_prompt)
|
||||
|
||||
# True if encoder prompt token ids field exists &
|
||||
# is not None
|
||||
return ('encoder_prompt_token_ids' in inputs
|
||||
and inputs['encoder_prompt_token_ids'] is not None)
|
||||
|
||||
def zip_enc_dec_prompts(
|
||||
enc_prompts: Iterable[_T1],
|
||||
dec_prompts: Iterable[Optional[_T2]],
|
||||
) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
|
||||
"""
|
||||
Zip encoder and decoder prompts together into a list of
|
||||
:class:`ExplicitEncoderDecoderPrompt` instances.
|
||||
"""
|
||||
return [
|
||||
build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
|
||||
for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)
|
||||
]
|
||||
|
||||
|
||||
def to_enc_dec_tuple_list(
|
||||
enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]],
|
||||
) -> List[Tuple[_T1, Optional[_T2]]]:
|
||||
return [(enc_dec_prompt["encoder_prompt"],
|
||||
enc_dec_prompt["decoder_prompt"])
|
||||
for enc_dec_prompt in enc_dec_prompts]
|
||||
|
||||
Reference in New Issue
Block a user