[Frontend] Refactor prompt processing (#4028)
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
parent
89c1c6a196
commit
739b61a348
+13
-24
@@ -6,8 +6,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
from vllm.inputs import (PromptInputs, PromptStrictInputs, TextPrompt,
|
||||
TextTokensPrompt, TokensPrompt,
|
||||
from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
|
||||
parse_and_batch_prompt)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -238,7 +237,7 @@ class LLM:
|
||||
@overload
|
||||
def generate(
|
||||
self,
|
||||
inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
inputs: Union[PromptInputs, Sequence[PromptInputs]],
|
||||
/, # We may enable `inputs` keyword after removing the old API
|
||||
*,
|
||||
sampling_params: Optional[Union[SamplingParams,
|
||||
@@ -255,7 +254,7 @@ class LLM:
|
||||
"instead.")
|
||||
def generate(
|
||||
self,
|
||||
prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
|
||||
Optional[Union[str, List[str]]]] = None,
|
||||
sampling_params: Optional[Union[SamplingParams,
|
||||
Sequence[SamplingParams]]] = None,
|
||||
@@ -302,9 +301,7 @@ class LLM:
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
else:
|
||||
inputs = cast(
|
||||
Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
prompts)
|
||||
inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
|
||||
|
||||
if sampling_params is None:
|
||||
# Use default sampling params.
|
||||
@@ -383,7 +380,7 @@ class LLM:
|
||||
@overload
|
||||
def encode(
|
||||
self,
|
||||
inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
inputs: Union[PromptInputs, Sequence[PromptInputs]],
|
||||
/, # We may enable `inputs` keyword after removing the old API
|
||||
*,
|
||||
pooling_params: Optional[Union[PoolingParams,
|
||||
@@ -400,7 +397,7 @@ class LLM:
|
||||
"instead.")
|
||||
def encode(
|
||||
self,
|
||||
prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
|
||||
Optional[Union[str, List[str]]]] = None,
|
||||
pooling_params: Optional[Union[PoolingParams,
|
||||
Sequence[PoolingParams]]] = None,
|
||||
@@ -417,7 +414,7 @@ class LLM:
|
||||
|
||||
Args:
|
||||
inputs: The inputs to the LLM. You may pass a sequence of inputs for
|
||||
batch inference. See :class:`~vllm.inputs.PromptStrictInputs`
|
||||
batch inference. See :class:`~vllm.inputs.PromptInputs`
|
||||
for more details about the format of each input.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
@@ -446,9 +443,7 @@ class LLM:
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
else:
|
||||
inputs = cast(
|
||||
Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
prompts)
|
||||
inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
|
||||
|
||||
if pooling_params is None:
|
||||
# Use default pooling params.
|
||||
@@ -496,17 +491,11 @@ class LLM:
|
||||
inputs: List[PromptInputs] = []
|
||||
for i in range(num_requests):
|
||||
if prompts is not None:
|
||||
if prompt_token_ids is not None:
|
||||
item = TextTokensPrompt(
|
||||
prompt=prompts[i],
|
||||
prompt_token_ids=prompt_token_ids[i])
|
||||
else:
|
||||
item = TextPrompt(prompt=prompts[i])
|
||||
item = TextPrompt(prompt=prompts[i])
|
||||
elif prompt_token_ids is not None:
|
||||
item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
|
||||
else:
|
||||
if prompt_token_ids is not None:
|
||||
item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
|
||||
else:
|
||||
raise AssertionError
|
||||
raise AssertionError
|
||||
|
||||
inputs.append(item)
|
||||
|
||||
@@ -514,7 +503,7 @@ class LLM:
|
||||
|
||||
def _validate_and_add_requests(
|
||||
self,
|
||||
inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
|
||||
inputs: Union[PromptInputs, Sequence[PromptInputs]],
|
||||
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
|
||||
Sequence[PoolingParams]],
|
||||
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
|
||||
|
||||
Reference in New Issue
Block a user