[Refactor] Consolidate sequence normalization and enc-dec parsing (#33928)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -67,12 +67,13 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
|
||||
from vllm.inputs.data import TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.mistral import (
|
||||
@@ -218,10 +219,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
) -> (
|
||||
tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]
|
||||
| ErrorResponse
|
||||
):
|
||||
) -> tuple[list[ConversationMessage], list[TokPrompt]] | ErrorResponse:
|
||||
"""
|
||||
render chat request by validating and preprocessing inputs.
|
||||
|
||||
@@ -380,7 +378,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
try:
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
prompt_text = engine_prompt.get("prompt")
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
# If we are creating sub requests for multiple prompts, ensure that they
|
||||
# have unique request ids.
|
||||
@@ -389,10 +387,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
|
||||
max_tokens = get_max_tokens(
|
||||
max_model_len=self.max_model_len,
|
||||
request=request,
|
||||
prompt=engine_prompt,
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
self.max_model_len,
|
||||
request,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
sampling_params: SamplingParams | BeamSearchParams
|
||||
|
||||
@@ -34,10 +34,10 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
@@ -78,7 +78,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
) -> list[TokensPrompt | EmbedsPrompt] | ErrorResponse:
|
||||
) -> list[TokPrompt] | ErrorResponse:
|
||||
"""
|
||||
render completion request by validating and preprocessing inputs.
|
||||
|
||||
@@ -160,13 +160,13 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
try:
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
prompt_text = engine_prompt.get("prompt")
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
max_tokens = get_max_tokens(
|
||||
max_model_len=self.max_model_len,
|
||||
request=request,
|
||||
prompt=engine_prompt,
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
self.max_model_len,
|
||||
request,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
sampling_params: SamplingParams | BeamSearchParams
|
||||
@@ -277,7 +277,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
# with the inputs token IDs
|
||||
if final_res.prompt is None:
|
||||
engine_prompt = engine_prompts[i]
|
||||
final_res.prompt = engine_prompt.get("prompt")
|
||||
final_res.prompt = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
|
||||
|
||||
@@ -313,7 +313,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def completion_stream_generator(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
engine_prompts: list[TokensPrompt | EmbedsPrompt],
|
||||
engine_prompts: list[TokPrompt],
|
||||
result_generator: AsyncIterator[tuple[int, RequestOutput]],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
@@ -347,7 +347,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
prompt_text = res.prompt
|
||||
if prompt_text is None:
|
||||
engine_prompt = engine_prompts[prompt_idx]
|
||||
prompt_text = engine_prompt.get("prompt")
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
# Prompt details are excluded from later streamed outputs
|
||||
if prompt_token_ids is not None:
|
||||
|
||||
@@ -96,11 +96,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.utils import get_max_tokens, sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import EmbedsPrompt, PromptType, TokensPrompt
|
||||
from vllm.inputs.parse import (
|
||||
get_prompt_components,
|
||||
is_explicit_encoder_decoder_prompt,
|
||||
)
|
||||
from vllm.inputs.data import PromptType, SingletonPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -108,6 +104,14 @@ from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
SingletonDictPrompt,
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
@@ -203,7 +207,7 @@ class ServeContext(Generic[RequestT]):
|
||||
request_id: str
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
engine_prompts: list[TokensPrompt | EmbedsPrompt] | None = None
|
||||
engine_prompts: list[TokPrompt] | None = None
|
||||
|
||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||
None
|
||||
@@ -247,7 +251,7 @@ class OpenAIServing:
|
||||
|
||||
async def beam_search(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
prompt: TokPrompt,
|
||||
request_id: str,
|
||||
params: BeamSearchParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -271,20 +275,12 @@ class OpenAIServing:
|
||||
|
||||
eos_token_id: int = tokenizer.eos_token_id # type: ignore
|
||||
|
||||
if is_explicit_encoder_decoder_prompt(prompt):
|
||||
raise NotImplementedError
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
raise NotImplementedError("Encoder-decoder prompt not supported")
|
||||
|
||||
prompt_text: str | None
|
||||
prompt_token_ids: list[int]
|
||||
multi_modal_data: MultiModalDataDict | None
|
||||
if isinstance(prompt, str):
|
||||
prompt_text = prompt
|
||||
prompt_token_ids = []
|
||||
multi_modal_data = None
|
||||
else:
|
||||
prompt_text = prompt.get("prompt") # type: ignore
|
||||
prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
|
||||
multi_modal_data = prompt.get("multi_modal_data") # type: ignore
|
||||
prompt_text: str | None = prompt.get("prompt") # type: ignore
|
||||
prompt_token_ids: list[int] = prompt.get("prompt_token_ids", []) # type: ignore
|
||||
multi_modal_data: MultiModalDataDict | None = prompt.get("multi_modal_data") # type: ignore
|
||||
|
||||
mm_processor_kwargs: dict[str, Any] | None = None
|
||||
|
||||
@@ -963,22 +959,40 @@ class OpenAIServing:
|
||||
request: RendererRequest,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
) -> list[TokensPrompt | EmbedsPrompt]:
|
||||
) -> list[TokPrompt]:
|
||||
renderer = self.renderer
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
model_config = self.model_config
|
||||
|
||||
in_prompts = await renderer.render_completions_async(
|
||||
prompt_input, prompt_embeds
|
||||
)
|
||||
engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
in_prompts = await renderer.render_prompts_async(parsed_prompts)
|
||||
|
||||
extra_items = {
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
}
|
||||
for prompt in engine_prompts:
|
||||
prompt.update(extra_items) # type: ignore
|
||||
for in_prompt in in_prompts:
|
||||
target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore
|
||||
"encoder_prompt", in_prompt
|
||||
)
|
||||
target_prompt.update(extra_items) # type: ignore
|
||||
|
||||
engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)
|
||||
|
||||
return engine_prompts
|
||||
|
||||
@@ -991,7 +1005,7 @@ class OpenAIServing:
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
|
||||
) -> tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]:
|
||||
) -> tuple[list[ConversationMessage], list[TokPrompt]]:
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
renderer = self.renderer
|
||||
@@ -1009,17 +1023,21 @@ class OpenAIServing:
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(default_template_kwargs)
|
||||
|
||||
conversation, prompt = await renderer.render_messages_async(
|
||||
conversation, in_prompt = await renderer.render_messages_async(
|
||||
messages, chat_params
|
||||
)
|
||||
engine_prompt = await renderer.tokenize_prompt_async(prompt, tok_params)
|
||||
target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore
|
||||
"encoder_prompt", in_prompt
|
||||
)
|
||||
|
||||
extra_items = {
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
}
|
||||
engine_prompt.update(extra_items) # type: ignore
|
||||
target_prompt.update(extra_items) # type: ignore
|
||||
|
||||
engine_prompt = await renderer.tokenize_prompt_async(target_prompt, tok_params)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
@@ -1040,6 +1058,15 @@ class OpenAIServing:
|
||||
|
||||
return conversation, [engine_prompt]
|
||||
|
||||
def _extract_prompt_components(self, prompt: object):
|
||||
return extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
def _extract_prompt_text(self, prompt: object):
|
||||
return self._extract_prompt_components(prompt).text
|
||||
|
||||
def _extract_prompt_len(self, prompt: object):
|
||||
return extract_prompt_len(self.model_config, prompt)
|
||||
|
||||
async def _render_next_turn(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
@@ -1067,7 +1094,7 @@ class OpenAIServing:
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: TokensPrompt | EmbedsPrompt,
|
||||
engine_prompt: TokPrompt,
|
||||
sampling_params: SamplingParams,
|
||||
tok_params: TokenizeParams,
|
||||
context: ConversationContext,
|
||||
@@ -1075,7 +1102,7 @@ class OpenAIServing:
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
):
|
||||
prompt_text = engine_prompt.get("prompt")
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
orig_priority = priority
|
||||
sub_request = 0
|
||||
@@ -1145,12 +1172,12 @@ class OpenAIServing:
|
||||
context.chat_template_content_format,
|
||||
)
|
||||
engine_prompt = engine_prompts[0]
|
||||
prompt_text = engine_prompt.get("prompt")
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
context.request,
|
||||
engine_prompt,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params, # type: ignore
|
||||
)
|
||||
|
||||
@@ -1161,20 +1188,20 @@ class OpenAIServing:
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: PromptType,
|
||||
inputs: PromptType | TokPrompt,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if self.request_logger is None:
|
||||
return
|
||||
|
||||
prompt, prompt_token_ids, prompt_embeds = get_prompt_components(inputs)
|
||||
components = self._extract_prompt_components(inputs)
|
||||
|
||||
self.request_logger.log_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
prompt_token_ids,
|
||||
prompt_embeds,
|
||||
components.text,
|
||||
components.token_ids,
|
||||
components.embeds,
|
||||
params=params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
@@ -116,13 +116,13 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
)
|
||||
from vllm.entrypoints.utils import get_max_tokens
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
|
||||
from vllm.inputs.parse import get_prompt_len
|
||||
from vllm.inputs.data import TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
@@ -292,10 +292,10 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
|
||||
def _validate_generator_input(
|
||||
self,
|
||||
engine_prompt: TokensPrompt | EmbedsPrompt,
|
||||
engine_prompt: TokPrompt,
|
||||
) -> ErrorResponse | None:
|
||||
"""Add validations to the input to the generator here."""
|
||||
prompt_len = get_prompt_len(engine_prompt)
|
||||
prompt_len = self._extract_prompt_len(engine_prompt)
|
||||
if self.max_model_len <= prompt_len:
|
||||
error_message = (
|
||||
f"The engine prompt length {prompt_len} "
|
||||
@@ -442,7 +442,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
request,
|
||||
engine_prompt,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import time
|
||||
import zlib
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from functools import cached_property
|
||||
from typing import Literal, TypeAlias, TypeVar, cast
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
from fastapi import Request
|
||||
@@ -37,12 +37,13 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
|
||||
TranslationStreamResponse,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import ExplicitEncoderDecoderPrompt, PromptType
|
||||
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import FlatLogprobs, Logprob
|
||||
from vllm.model_executor.models import SupportsTranscription, supports_transcription
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.inputs import EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
@@ -94,7 +95,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
)
|
||||
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
self.task_type = task_type
|
||||
self.task_type: Final = task_type
|
||||
|
||||
self.asr_config = self.model_cls.get_speech_to_text_config(
|
||||
self.model_config, task_type
|
||||
@@ -298,35 +299,26 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
to_language=to_language,
|
||||
)
|
||||
if request.response_format == "verbose_json":
|
||||
if not is_explicit_encoder_decoder_prompt(prompt):
|
||||
raise VLLMValidationError(
|
||||
"Expected prompt to be an encoder-decoder prompt",
|
||||
parameter="prompt",
|
||||
value=type(prompt).__name__,
|
||||
)
|
||||
|
||||
prompt = self._preprocess_verbose_prompt(prompt)
|
||||
prompt = self._preprocess_verbose_prompt(parse_enc_dec_prompt(prompt))
|
||||
|
||||
prompts.append(prompt)
|
||||
|
||||
return prompts, duration
|
||||
|
||||
def _repl_verbose_text(self, text: str):
|
||||
return text.replace("<|notimestamps|>", "<|0.00|>")
|
||||
|
||||
def _preprocess_verbose_prompt(self, prompt: ExplicitEncoderDecoderPrompt):
|
||||
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
|
||||
if isinstance(dec_prompt, str):
|
||||
prompt["decoder_prompt"] = self._repl_verbose_text(dec_prompt)
|
||||
elif isinstance(dec_prompt, dict) and "prompt" in dec_prompt:
|
||||
dec_prompt["prompt"] = self._repl_verbose_text(dec_prompt["prompt"])
|
||||
else:
|
||||
if not (isinstance(dec_prompt, dict) and "prompt" in dec_prompt):
|
||||
raise VLLMValidationError(
|
||||
"Expected decoder_prompt to contain text",
|
||||
parameter="decoder_prompt",
|
||||
value=type(dec_prompt).__name__,
|
||||
)
|
||||
|
||||
dec_prompt["prompt"] = dec_prompt["prompt"].replace(
|
||||
"<|notimestamps|>", "<|0.00|>"
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
def _get_verbose_segments(
|
||||
|
||||
Reference in New Issue
Block a user