[Chore] Remove redundant RequestPrompt (#30612)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-12-14 09:22:37 +00:00
committed by GitHub
parent f569c654e1
commit dcb31196da
12 changed files with 188 additions and 253 deletions
+74 -127
View File
@@ -5,29 +5,61 @@ import json
import sys
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
import numpy as np
import torch
from fastapi import Request
from openai.types.responses import (
ToolChoiceFunction,
)
from pydantic import ConfigDict, TypeAdapter
from starlette.datastructures import Headers
from typing_extensions import TypeIs
import vllm.envs as envs
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
ConversationMessage,
apply_hf_chat_template,
apply_mistral_chat_template,
parse_chat_messages_futures,
resolve_chat_template_content_format,
)
from vllm.entrypoints.context import (
ConversationContext,
HarmonyContext,
ParsableContext,
StreamingHarmonyContext,
)
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponse,
CompletionRequest,
CompletionResponse,
DetokenizeRequest,
ErrorInfo,
ErrorResponse,
FunctionCall,
FunctionDefinition,
ResponseInputOutputItem,
ResponsesRequest,
TokenizeChatRequest,
TokenizeCompletionRequest,
TokenizeResponse,
TranscriptionRequest,
TranscriptionResponse,
TranslationRequest,
)
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
from vllm.entrypoints.pooling.classify.protocol import (
ClassificationChatRequest,
ClassificationCompletionRequest,
@@ -49,58 +81,13 @@ from vllm.entrypoints.pooling.score.protocol import (
ScoreRequest,
ScoreResponse,
)
from vllm.transformers_utils.tokenizer import AnyTokenizer
if sys.version_info >= (3, 12):
from typing import TypedDict
else:
from typing_extensions import TypedDict
from openai.types.responses import (
ToolChoiceFunction,
)
import vllm.envs as envs
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
ConversationMessage,
apply_hf_chat_template,
apply_mistral_chat_template,
parse_chat_messages_futures,
resolve_chat_template_content_format,
)
from vllm.entrypoints.context import ConversationContext
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponse,
CompletionRequest,
CompletionResponse,
DetokenizeRequest,
ErrorInfo,
ErrorResponse,
FunctionDefinition,
TokenizeChatRequest,
TokenizeCompletionRequest,
TokenizeResponse,
TranscriptionRequest,
TranscriptionResponse,
TranslationRequest,
)
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
from vllm.entrypoints.responses_utils import (
construct_input_messages,
)
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
from vllm.entrypoints.utils import _validate_truncation_size
from vllm.inputs.data import PromptType
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
from vllm.inputs.data import PromptType, TokensPrompt
from vllm.inputs.parse import (
PromptComponents,
get_prompt_components,
@@ -109,10 +96,7 @@ from vllm.inputs.parse import (
from vllm.logger import init_logger
from vllm.logprobs import Logprob, PromptLogprobs
from vllm.lora.request import LoRARequest
from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
MultiModalDataDict,
MultiModalUUIDDict,
)
from vllm.multimodal import MultiModalDataDict
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
from vllm.pooling_params import PoolingParams
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -185,34 +169,6 @@ AnyResponse: TypeAlias = (
)
class TextTokensPrompt(TypedDict):
prompt: str
prompt_token_ids: list[int]
class EmbedsPrompt(TypedDict):
prompt_embeds: torch.Tensor
RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
return (
isinstance(prompt, dict)
and "prompt_token_ids" in prompt
and "prompt_embeds" not in prompt
)
def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
return (
isinstance(prompt, dict)
and "prompt_token_ids" not in prompt
and "prompt_embeds" in prompt
)
RequestT = TypeVar("RequestT", bound=AnyRequest)
@@ -223,8 +179,7 @@ class RequestProcessingMixin:
handling prompt preparation and engine input.
"""
request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list)
engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list)
engine_prompts: list[TokensPrompt] | None = field(default_factory=list)
@dataclass(kw_only=True)
@@ -425,7 +380,7 @@ class OpenAIServing:
prompts_batch, lora_req_batch = zip(
*[
(
EngineTokensPrompt(
TokensPrompt(
prompt_token_ids=beam.tokens,
multi_modal_data=beam.multi_modal_data,
mm_processor_kwargs=beam.mm_processor_kwargs,
@@ -947,7 +902,7 @@ class OpenAIServing:
prompt: str,
tokenizer: TokenizerLike,
add_special_tokens: bool,
) -> TextTokensPrompt:
) -> TokensPrompt:
async_tokenizer = self._get_async_tokenizer(tokenizer)
if (
@@ -988,7 +943,7 @@ class OpenAIServing:
request: AnyRequest,
prompt_ids: list[int],
tokenizer: TokenizerLike | None,
) -> TextTokensPrompt:
) -> TokensPrompt:
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
if truncate_prompt_tokens is None:
@@ -1011,7 +966,7 @@ class OpenAIServing:
request: AnyRequest,
input_ids: list[int],
input_text: str,
) -> TextTokensPrompt:
) -> TokensPrompt:
token_num = len(input_ids)
# Note: EmbeddingRequest, ClassificationRequest,
@@ -1042,7 +997,7 @@ class OpenAIServing:
f"{token_num} tokens in the input for {operation}. "
f"Please reduce the length of the input."
)
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
# Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
# and does not require model context length validation
@@ -1050,7 +1005,7 @@ class OpenAIServing:
request,
(TokenizeCompletionRequest, TokenizeChatRequest, DetokenizeRequest),
):
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
# chat completion endpoint supports max_completion_tokens
if isinstance(request, ChatCompletionRequest):
@@ -1078,7 +1033,7 @@ class OpenAIServing:
f" - {token_num})."
)
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
async def _tokenize_prompt_input_async(
self,
@@ -1086,7 +1041,7 @@ class OpenAIServing:
tokenizer: TokenizerLike,
prompt_input: str | list[int],
add_special_tokens: bool = True,
) -> TextTokensPrompt:
) -> TokensPrompt:
"""
A simpler implementation that tokenizes a single prompt input.
"""
@@ -1105,7 +1060,7 @@ class OpenAIServing:
tokenizer: TokenizerLike,
prompt_inputs: Iterable[str | list[int]],
add_special_tokens: bool = True,
) -> AsyncGenerator[TextTokensPrompt, None]:
) -> AsyncGenerator[TokensPrompt, None]:
"""
A simpler implementation that tokenizes multiple prompt inputs.
"""
@@ -1158,11 +1113,7 @@ class OpenAIServing:
chat_template_kwargs: dict[str, Any] | None = None,
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
add_special_tokens: bool = False,
) -> tuple[
list[ConversationMessage],
Sequence[RequestPrompt],
list[EngineTokensPrompt],
]:
) -> tuple[list[ConversationMessage], list[TokensPrompt]]:
model_config = self.model_config
resolved_content_format = resolve_chat_template_content_format(
@@ -1235,9 +1186,7 @@ class OpenAIServing:
"Prompt has to be a string",
"when the tokenizer is not initialised",
)
prompt_inputs = TextTokensPrompt(
prompt=request_prompt, prompt_token_ids=[1]
)
prompt_inputs = TokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
elif isinstance(request_prompt, str):
prompt_inputs = await self._tokenize_prompt_input_async(
request,
@@ -1250,14 +1199,15 @@ class OpenAIServing:
assert is_list_of(request_prompt, int), (
"Prompt has to be either a string or a list of token ids"
)
prompt_inputs = TextTokensPrompt(
prompt_inputs = TokensPrompt(
prompt=tokenizer.decode(request_prompt),
prompt_token_ids=request_prompt,
)
engine_prompt = EngineTokensPrompt(
prompt_token_ids=prompt_inputs["prompt_token_ids"]
)
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
if "prompt" in prompt_inputs:
engine_prompt["prompt"] = prompt_inputs["prompt"]
if mm_data is not None:
engine_prompt["multi_modal_data"] = mm_data
@@ -1270,7 +1220,7 @@ class OpenAIServing:
if hasattr(request, "cache_salt") and request.cache_salt is not None:
engine_prompt["cache_salt"] = request.cache_salt
return conversation, [request_prompt], [engine_prompt]
return conversation, [engine_prompt]
async def _process_inputs(
self,
@@ -1302,7 +1252,7 @@ class OpenAIServing:
async def _render_next_turn(
self,
request: ResponsesRequest,
tokenizer: AnyTokenizer,
tokenizer: TokenizerLike | None,
messages: list[ResponseInputOutputItem],
tool_dicts: list[dict[str, Any]] | None,
tool_parser,
@@ -1313,7 +1263,7 @@ class OpenAIServing:
request_input=messages,
)
_, request_prompts, engine_prompts = await self._preprocess_chat(
_, engine_prompts = await self._preprocess_chat(
request,
tokenizer,
new_messages,
@@ -1322,20 +1272,20 @@ class OpenAIServing:
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
)
return request_prompts, engine_prompts
return engine_prompts
async def _generate_with_builtin_tools(
self,
request_id: str,
request_prompt: RequestPrompt,
engine_prompt: EngineTokensPrompt,
engine_prompt: TokensPrompt,
sampling_params: SamplingParams,
context: ConversationContext,
lora_request: LoRARequest | None = None,
priority: int = 0,
**kwargs,
):
prompt_text, _, _ = self._get_prompt_components(request_prompt)
prompt_text, _, _ = self._get_prompt_components(engine_prompt)
orig_priority = priority
sub_request = 0
while True:
@@ -1343,7 +1293,7 @@ class OpenAIServing:
sub_request_id = f"{request_id}_{sub_request}"
self._log_inputs(
sub_request_id,
request_prompt,
engine_prompt,
params=sampling_params,
lora_request=lora_request,
)
@@ -1388,10 +1338,9 @@ class OpenAIServing:
# Render the next prompt token ids.
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
prompt_token_ids = context.render_for_completion()
engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
request_prompt = prompt_token_ids
engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
elif isinstance(context, ParsableContext):
request_prompts, engine_prompts = await self._render_next_turn(
engine_prompts = await self._render_next_turn(
context.request,
context.tokenizer,
context.parser.response_messages,
@@ -1401,8 +1350,7 @@ class OpenAIServing:
context.chat_template_content_format,
)
engine_prompt = engine_prompts[0]
request_prompt = request_prompts[0]
prompt_text, _, _ = self._get_prompt_components(request_prompt)
prompt_text, _, _ = self._get_prompt_components(engine_prompt)
# Update the sampling params.
sampling_params.max_tokens = self.max_model_len - len(
@@ -1412,19 +1360,13 @@ class OpenAIServing:
priority = orig_priority - 1
sub_request += 1
def _get_prompt_components(
self,
prompt: RequestPrompt | PromptType,
) -> PromptComponents:
if isinstance(prompt, list):
return PromptComponents(token_ids=prompt)
return get_prompt_components(prompt) # type: ignore[arg-type]
def _get_prompt_components(self, prompt: PromptType) -> PromptComponents:
return get_prompt_components(prompt)
def _log_inputs(
self,
request_id: str,
inputs: RequestPrompt | PromptType,
inputs: PromptType,
params: SamplingParams | PoolingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
@@ -1486,7 +1428,7 @@ class OpenAIServing:
@staticmethod
def _parse_tool_calls_from_content(
request: ResponsesRequest | ChatCompletionRequest,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
enable_auto_tools: bool,
tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
content: str | None = None,
@@ -1526,6 +1468,11 @@ class OpenAIServing:
and enable_auto_tools
and (request.tool_choice == "auto" or request.tool_choice is None)
):
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
)
# Automatic Tool Call Parsing
try:
tool_parser = tool_parser_cls(tokenizer)