[Chore] Remove redundant RequestPrompt (#30612)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,29 +5,61 @@ import json
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
|
||||
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fastapi import Request
|
||||
from openai.types.responses import (
|
||||
ToolChoiceFunction,
|
||||
)
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from starlette.datastructures import Headers
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
apply_hf_chat_template,
|
||||
apply_mistral_chat_template,
|
||||
parse_chat_messages_futures,
|
||||
resolve_chat_template_content_format,
|
||||
)
|
||||
from vllm.entrypoints.context import (
|
||||
ConversationContext,
|
||||
HarmonyContext,
|
||||
ParsableContext,
|
||||
StreamingHarmonyContext,
|
||||
)
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
DetokenizeRequest,
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
ResponseInputOutputItem,
|
||||
ResponsesRequest,
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranslationRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
|
||||
from vllm.entrypoints.pooling.classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
@@ -49,58 +81,13 @@ from vllm.entrypoints.pooling.score.protocol import (
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import TypedDict
|
||||
else:
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from openai.types.responses import (
|
||||
ToolChoiceFunction,
|
||||
)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
apply_hf_chat_template,
|
||||
apply_mistral_chat_template,
|
||||
parse_chat_messages_futures,
|
||||
resolve_chat_template_content_format,
|
||||
)
|
||||
from vllm.entrypoints.context import ConversationContext
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
DetokenizeRequest,
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
FunctionDefinition,
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranslationRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
|
||||
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
|
||||
from vllm.entrypoints.responses_utils import (
|
||||
construct_input_messages,
|
||||
)
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
|
||||
from vllm.inputs.data import PromptType, TokensPrompt
|
||||
from vllm.inputs.parse import (
|
||||
PromptComponents,
|
||||
get_prompt_components,
|
||||
@@ -109,10 +96,7 @@ from vllm.inputs.parse import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
|
||||
MultiModalDataDict,
|
||||
MultiModalUUIDDict,
|
||||
)
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -185,34 +169,6 @@ AnyResponse: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class TextTokensPrompt(TypedDict):
|
||||
prompt: str
|
||||
prompt_token_ids: list[int]
|
||||
|
||||
|
||||
class EmbedsPrompt(TypedDict):
|
||||
prompt_embeds: torch.Tensor
|
||||
|
||||
|
||||
RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
|
||||
|
||||
|
||||
def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
|
||||
return (
|
||||
isinstance(prompt, dict)
|
||||
and "prompt_token_ids" in prompt
|
||||
and "prompt_embeds" not in prompt
|
||||
)
|
||||
|
||||
|
||||
def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
|
||||
return (
|
||||
isinstance(prompt, dict)
|
||||
and "prompt_token_ids" not in prompt
|
||||
and "prompt_embeds" in prompt
|
||||
)
|
||||
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
|
||||
|
||||
@@ -223,8 +179,7 @@ class RequestProcessingMixin:
|
||||
handling prompt preparation and engine input.
|
||||
"""
|
||||
|
||||
request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list)
|
||||
engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list)
|
||||
engine_prompts: list[TokensPrompt] | None = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
@@ -425,7 +380,7 @@ class OpenAIServing:
|
||||
prompts_batch, lora_req_batch = zip(
|
||||
*[
|
||||
(
|
||||
EngineTokensPrompt(
|
||||
TokensPrompt(
|
||||
prompt_token_ids=beam.tokens,
|
||||
multi_modal_data=beam.multi_modal_data,
|
||||
mm_processor_kwargs=beam.mm_processor_kwargs,
|
||||
@@ -947,7 +902,7 @@ class OpenAIServing:
|
||||
prompt: str,
|
||||
tokenizer: TokenizerLike,
|
||||
add_special_tokens: bool,
|
||||
) -> TextTokensPrompt:
|
||||
) -> TokensPrompt:
|
||||
async_tokenizer = self._get_async_tokenizer(tokenizer)
|
||||
|
||||
if (
|
||||
@@ -988,7 +943,7 @@ class OpenAIServing:
|
||||
request: AnyRequest,
|
||||
prompt_ids: list[int],
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> TextTokensPrompt:
|
||||
) -> TokensPrompt:
|
||||
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
|
||||
|
||||
if truncate_prompt_tokens is None:
|
||||
@@ -1011,7 +966,7 @@ class OpenAIServing:
|
||||
request: AnyRequest,
|
||||
input_ids: list[int],
|
||||
input_text: str,
|
||||
) -> TextTokensPrompt:
|
||||
) -> TokensPrompt:
|
||||
token_num = len(input_ids)
|
||||
|
||||
# Note: EmbeddingRequest, ClassificationRequest,
|
||||
@@ -1042,7 +997,7 @@ class OpenAIServing:
|
||||
f"{token_num} tokens in the input for {operation}. "
|
||||
f"Please reduce the length of the input."
|
||||
)
|
||||
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
|
||||
# Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
|
||||
# and does not require model context length validation
|
||||
@@ -1050,7 +1005,7 @@ class OpenAIServing:
|
||||
request,
|
||||
(TokenizeCompletionRequest, TokenizeChatRequest, DetokenizeRequest),
|
||||
):
|
||||
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
|
||||
# chat completion endpoint supports max_completion_tokens
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
@@ -1078,7 +1033,7 @@ class OpenAIServing:
|
||||
f" - {token_num})."
|
||||
)
|
||||
|
||||
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
|
||||
|
||||
async def _tokenize_prompt_input_async(
|
||||
self,
|
||||
@@ -1086,7 +1041,7 @@ class OpenAIServing:
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_input: str | list[int],
|
||||
add_special_tokens: bool = True,
|
||||
) -> TextTokensPrompt:
|
||||
) -> TokensPrompt:
|
||||
"""
|
||||
A simpler implementation that tokenizes a single prompt input.
|
||||
"""
|
||||
@@ -1105,7 +1060,7 @@ class OpenAIServing:
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_inputs: Iterable[str | list[int]],
|
||||
add_special_tokens: bool = True,
|
||||
) -> AsyncGenerator[TextTokensPrompt, None]:
|
||||
) -> AsyncGenerator[TokensPrompt, None]:
|
||||
"""
|
||||
A simpler implementation that tokenizes multiple prompt inputs.
|
||||
"""
|
||||
@@ -1158,11 +1113,7 @@ class OpenAIServing:
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
|
||||
add_special_tokens: bool = False,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
Sequence[RequestPrompt],
|
||||
list[EngineTokensPrompt],
|
||||
]:
|
||||
) -> tuple[list[ConversationMessage], list[TokensPrompt]]:
|
||||
model_config = self.model_config
|
||||
|
||||
resolved_content_format = resolve_chat_template_content_format(
|
||||
@@ -1235,9 +1186,7 @@ class OpenAIServing:
|
||||
"Prompt has to be a string",
|
||||
"when the tokenizer is not initialised",
|
||||
)
|
||||
prompt_inputs = TextTokensPrompt(
|
||||
prompt=request_prompt, prompt_token_ids=[1]
|
||||
)
|
||||
prompt_inputs = TokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
|
||||
elif isinstance(request_prompt, str):
|
||||
prompt_inputs = await self._tokenize_prompt_input_async(
|
||||
request,
|
||||
@@ -1250,14 +1199,15 @@ class OpenAIServing:
|
||||
assert is_list_of(request_prompt, int), (
|
||||
"Prompt has to be either a string or a list of token ids"
|
||||
)
|
||||
prompt_inputs = TextTokensPrompt(
|
||||
prompt_inputs = TokensPrompt(
|
||||
prompt=tokenizer.decode(request_prompt),
|
||||
prompt_token_ids=request_prompt,
|
||||
)
|
||||
|
||||
engine_prompt = EngineTokensPrompt(
|
||||
prompt_token_ids=prompt_inputs["prompt_token_ids"]
|
||||
)
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
|
||||
if "prompt" in prompt_inputs:
|
||||
engine_prompt["prompt"] = prompt_inputs["prompt"]
|
||||
|
||||
if mm_data is not None:
|
||||
engine_prompt["multi_modal_data"] = mm_data
|
||||
|
||||
@@ -1270,7 +1220,7 @@ class OpenAIServing:
|
||||
if hasattr(request, "cache_salt") and request.cache_salt is not None:
|
||||
engine_prompt["cache_salt"] = request.cache_salt
|
||||
|
||||
return conversation, [request_prompt], [engine_prompt]
|
||||
return conversation, [engine_prompt]
|
||||
|
||||
async def _process_inputs(
|
||||
self,
|
||||
@@ -1302,7 +1252,7 @@ class OpenAIServing:
|
||||
async def _render_next_turn(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenizer: TokenizerLike | None,
|
||||
messages: list[ResponseInputOutputItem],
|
||||
tool_dicts: list[dict[str, Any]] | None,
|
||||
tool_parser,
|
||||
@@ -1313,7 +1263,7 @@ class OpenAIServing:
|
||||
request_input=messages,
|
||||
)
|
||||
|
||||
_, request_prompts, engine_prompts = await self._preprocess_chat(
|
||||
_, engine_prompts = await self._preprocess_chat(
|
||||
request,
|
||||
tokenizer,
|
||||
new_messages,
|
||||
@@ -1322,20 +1272,20 @@ class OpenAIServing:
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
)
|
||||
return request_prompts, engine_prompts
|
||||
return engine_prompts
|
||||
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
request_id: str,
|
||||
request_prompt: RequestPrompt,
|
||||
engine_prompt: EngineTokensPrompt,
|
||||
engine_prompt: TokensPrompt,
|
||||
sampling_params: SamplingParams,
|
||||
context: ConversationContext,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
prompt_text, _, _ = self._get_prompt_components(request_prompt)
|
||||
prompt_text, _, _ = self._get_prompt_components(engine_prompt)
|
||||
|
||||
orig_priority = priority
|
||||
sub_request = 0
|
||||
while True:
|
||||
@@ -1343,7 +1293,7 @@ class OpenAIServing:
|
||||
sub_request_id = f"{request_id}_{sub_request}"
|
||||
self._log_inputs(
|
||||
sub_request_id,
|
||||
request_prompt,
|
||||
engine_prompt,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
@@ -1388,10 +1338,9 @@ class OpenAIServing:
|
||||
# Render the next prompt token ids.
|
||||
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
||||
prompt_token_ids = context.render_for_completion()
|
||||
engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||
request_prompt = prompt_token_ids
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||
elif isinstance(context, ParsableContext):
|
||||
request_prompts, engine_prompts = await self._render_next_turn(
|
||||
engine_prompts = await self._render_next_turn(
|
||||
context.request,
|
||||
context.tokenizer,
|
||||
context.parser.response_messages,
|
||||
@@ -1401,8 +1350,7 @@ class OpenAIServing:
|
||||
context.chat_template_content_format,
|
||||
)
|
||||
engine_prompt = engine_prompts[0]
|
||||
request_prompt = request_prompts[0]
|
||||
prompt_text, _, _ = self._get_prompt_components(request_prompt)
|
||||
prompt_text, _, _ = self._get_prompt_components(engine_prompt)
|
||||
|
||||
# Update the sampling params.
|
||||
sampling_params.max_tokens = self.max_model_len - len(
|
||||
@@ -1412,19 +1360,13 @@ class OpenAIServing:
|
||||
priority = orig_priority - 1
|
||||
sub_request += 1
|
||||
|
||||
def _get_prompt_components(
|
||||
self,
|
||||
prompt: RequestPrompt | PromptType,
|
||||
) -> PromptComponents:
|
||||
if isinstance(prompt, list):
|
||||
return PromptComponents(token_ids=prompt)
|
||||
|
||||
return get_prompt_components(prompt) # type: ignore[arg-type]
|
||||
def _get_prompt_components(self, prompt: PromptType) -> PromptComponents:
|
||||
return get_prompt_components(prompt)
|
||||
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: RequestPrompt | PromptType,
|
||||
inputs: PromptType,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
@@ -1486,7 +1428,7 @@ class OpenAIServing:
|
||||
@staticmethod
|
||||
def _parse_tool_calls_from_content(
|
||||
request: ResponsesRequest | ChatCompletionRequest,
|
||||
tokenizer: TokenizerLike,
|
||||
tokenizer: TokenizerLike | None,
|
||||
enable_auto_tools: bool,
|
||||
tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
|
||||
content: str | None = None,
|
||||
@@ -1526,6 +1468,11 @@ class OpenAIServing:
|
||||
and enable_auto_tools
|
||||
and (request.tool_choice == "auto" or request.tool_choice is None)
|
||||
):
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
"Tokenizer not available when `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
# Automatic Tool Call Parsing
|
||||
try:
|
||||
tool_parser = tool_parser_cls(tokenizer)
|
||||
|
||||
Reference in New Issue
Block a user