[Frontend] Move engine/protocol.py out openai folder (#54492)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -16,7 +16,7 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,11 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicUsage,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
@@ -38,14 +43,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
@@ -38,7 +38,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
|
||||
@@ -29,11 +29,9 @@ from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
|
||||
@@ -71,6 +71,11 @@ from vllm.entrypoints.cohere.protocol import (
|
||||
ToolCallStartEvent,
|
||||
ToolPlanDeltaEvent,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
@@ -79,13 +84,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
@@ -1761,7 +1761,6 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
# so the router can translate the envelope uniformly. ``param``
|
||||
# is accepted for signature parity with the base class but is
|
||||
# not surfaced in the Cohere wire format.
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo
|
||||
|
||||
del param # unused; kept for signature compatibility
|
||||
return ErrorResponse(
|
||||
|
||||
+3
-110
@@ -4,9 +4,7 @@
|
||||
# Adapted from
|
||||
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
@@ -14,16 +12,15 @@ from pydantic import (
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_serializer,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import replace
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.exceptions import VLLMServerError, VLLMValidationError
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -33,102 +30,6 @@ StopParam: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class OpenAIBaseModel(BaseModel):
|
||||
# OpenAI API does allow extra fields
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Cache class field names
|
||||
field_names: ClassVar[set[str] | None] = None
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
def __log_extra_fields__(cls, data, handler):
|
||||
result = handler(data)
|
||||
if not isinstance(data, dict):
|
||||
return result
|
||||
field_names = cls.field_names
|
||||
if field_names is None:
|
||||
# Get all class field names and their potential aliases
|
||||
field_names = set()
|
||||
for field_name, field in cls.model_fields.items():
|
||||
field_names.add(field_name)
|
||||
if alias := getattr(field, "alias", None):
|
||||
field_names.add(alias)
|
||||
cls.field_names = field_names
|
||||
|
||||
# Compare against both field names and aliases
|
||||
if any(k not in field_names for k in data):
|
||||
logger.debug(
|
||||
"The following fields were present in the request but ignored: %s",
|
||||
data.keys() - field_names,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class ErrorInfo(OpenAIBaseModel):
|
||||
message: str
|
||||
type: str
|
||||
param: str | None = None
|
||||
code: int
|
||||
|
||||
|
||||
class ErrorResponse(OpenAIBaseModel):
|
||||
error: ErrorInfo
|
||||
|
||||
|
||||
class ModelPermission(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
|
||||
object: str = "model_permission"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
allow_create_engine: bool = False
|
||||
allow_sampling: bool = True
|
||||
allow_logprobs: bool = True
|
||||
allow_search_indices: bool = False
|
||||
allow_view: bool = True
|
||||
allow_fine_tuning: bool = False
|
||||
organization: str = "*"
|
||||
group: str | None = None
|
||||
is_blocking: bool = False
|
||||
|
||||
|
||||
class ModelCard(OpenAIBaseModel):
|
||||
id: str
|
||||
object: str = "model"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
owned_by: str = "vllm"
|
||||
root: str | None = None
|
||||
parent: str | None = None
|
||||
max_model_len: int | None = None
|
||||
permission: list[ModelPermission] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ModelList(OpenAIBaseModel):
|
||||
object: str = "list"
|
||||
data: list[ModelCard] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PromptTokenUsageInfo(OpenAIBaseModel):
|
||||
cached_tokens: int | None = None
|
||||
created_cache_tokens: int | None = None
|
||||
multimodal_tokens: dict[str, int] | None = None
|
||||
"""Prompt tokens contributed by each input modality, keyed by modality name
|
||||
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
|
||||
placeholder tokens already counted in `prompt_tokens`; `None` when the
|
||||
request has no multimodal input."""
|
||||
|
||||
|
||||
class CompletionTokenUsageInfo(OpenAIBaseModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class UsageInfo(OpenAIBaseModel):
|
||||
prompt_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
completion_tokens: int | None = 0
|
||||
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||
|
||||
|
||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||
"""Per-request speculative-decoding acceptance metrics.
|
||||
|
||||
@@ -437,11 +338,3 @@ class DeltaMessage(OpenAIBaseModel):
|
||||
if len(data.get("tool_calls", [])) == 0:
|
||||
data.pop("tool_calls", None)
|
||||
return data
|
||||
|
||||
|
||||
class GenerationError(VLLMServerError):
|
||||
"""raised when finish_reason indicates internal server error (500)"""
|
||||
|
||||
def __init__(self, message: str = "Internal server error"):
|
||||
super().__init__(message)
|
||||
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
@@ -5,27 +5,28 @@ import time
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
||||
from typing import ClassVar, Generic, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
from starlette.datastructures import Headers
|
||||
|
||||
from vllm import RequestOutput
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
@@ -38,9 +39,6 @@ from vllm.tracing import (
|
||||
)
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringResponse,
|
||||
ServingGenerativeScoring,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -18,12 +18,12 @@ from fastapi import Request
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
OpenAIBaseModel,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
|
||||
@@ -35,11 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
@@ -51,6 +46,11 @@ from vllm.entrypoints.pooling.scoring.protocol import (
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
|
||||
@@ -14,8 +14,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -9,6 +9,7 @@ from http import HTTPStatus
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.chat_utils import ConversationMessage
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
@@ -16,11 +17,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -23,21 +23,20 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
OpenAIBaseModel,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
structured_outputs_from_response_format,
|
||||
validate_structural_tag_response_format,
|
||||
validate_structured_outputs_structural_tag,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -16,9 +16,15 @@ from vllm.entrypoints.chat_utils import (
|
||||
ConversationMessage,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
PerRequestMetrics,
|
||||
RequestResponseMetadata,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
@@ -36,23 +42,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
CompletionTokenUsageInfo,
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
)
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -12,8 +12,8 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -10,17 +10,16 @@ from pydantic import Field, model_validator
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
OpenAIBaseModel,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
structured_outputs_from_response_format,
|
||||
validate_structural_tag_response_format,
|
||||
validate_structured_outputs_structural_tag,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -10,9 +10,12 @@ from typing import cast
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
@@ -26,17 +29,15 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponseStreamChoice,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -7,14 +7,14 @@ from http import HTTPStatus
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
ModelCard,
|
||||
ModelList,
|
||||
ModelPermission,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
|
||||
@@ -8,13 +8,13 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
StreamingResponsesResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -24,11 +24,9 @@ from vllm import envs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.mcp.tool import Tool
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
|
||||
@@ -60,7 +60,8 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
|
||||
from vllm.entrypoints.generate.base.protocol import StopParam
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
|
||||
@@ -31,16 +31,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
@@ -89,9 +85,10 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
|
||||
@@ -61,8 +61,8 @@ from openai.types.responses.response_reasoning_item import (
|
||||
from openai.types.responses.tool import Tool
|
||||
from openai_harmony import Message as HarmonyMessage
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, DeltaToolCall
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
extract_function_from_recipient,
|
||||
is_function_recipient,
|
||||
|
||||
@@ -31,11 +31,11 @@ from openai.types.responses.tool import Tool
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall, FunctionDefinition
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tool_parsers.utils import (
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.tasks import check_removed_pooling_task
|
||||
|
||||
@@ -16,7 +16,6 @@ from vllm import PoolingRequestOutput, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
@@ -30,6 +29,7 @@ from vllm.tracing import (
|
||||
)
|
||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from .io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Annotated, TypeAlias
|
||||
from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TypeAlias
|
||||
import numpy as np
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import ClassificationOutput
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -18,7 +18,7 @@ from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
|
||||
@@ -4,7 +4,7 @@ from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -7,7 +7,7 @@ from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
@@ -4,7 +4,7 @@ from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
|
||||
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
@@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
||||
|
||||
@@ -14,7 +14,7 @@ import torch
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
@@ -5,7 +5,7 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
|
||||
@@ -12,14 +12,11 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
@@ -17,6 +16,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
@@ -10,7 +10,6 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
@@ -19,6 +18,7 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
|
||||
@@ -11,6 +11,7 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
@@ -20,7 +21,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
@@ -12,6 +12,7 @@ from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
clamp_prompt_logprobs,
|
||||
@@ -21,16 +22,15 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionLogProbsContent,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput, TokensPrompt, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -9,10 +9,8 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||
import time
|
||||
from typing import ClassVar
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIBaseModel(BaseModel):
|
||||
# OpenAI API does allow extra fields
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Cache class field names
|
||||
field_names: ClassVar[set[str] | None] = None
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
def __log_extra_fields__(cls, data, handler):
|
||||
result = handler(data)
|
||||
if not isinstance(data, dict):
|
||||
return result
|
||||
field_names = cls.field_names
|
||||
if field_names is None:
|
||||
# Get all class field names and their potential aliases
|
||||
field_names = set()
|
||||
for field_name, field in cls.model_fields.items():
|
||||
field_names.add(field_name)
|
||||
if alias := getattr(field, "alias", None):
|
||||
field_names.add(alias)
|
||||
cls.field_names = field_names
|
||||
|
||||
# Compare against both field names and aliases
|
||||
if any(k not in field_names for k in data):
|
||||
logger.debug(
|
||||
"The following fields were present in the request but ignored: %s",
|
||||
data.keys() - field_names,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class ErrorInfo(OpenAIBaseModel):
|
||||
message: str
|
||||
type: str
|
||||
param: str | None = None
|
||||
code: int
|
||||
|
||||
|
||||
class ErrorResponse(OpenAIBaseModel):
|
||||
error: ErrorInfo
|
||||
|
||||
|
||||
class ModelPermission(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
|
||||
object: str = "model_permission"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
allow_create_engine: bool = False
|
||||
allow_sampling: bool = True
|
||||
allow_logprobs: bool = True
|
||||
allow_search_indices: bool = False
|
||||
allow_view: bool = True
|
||||
allow_fine_tuning: bool = False
|
||||
organization: str = "*"
|
||||
group: str | None = None
|
||||
is_blocking: bool = False
|
||||
|
||||
|
||||
class ModelCard(OpenAIBaseModel):
|
||||
id: str
|
||||
object: str = "model"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
owned_by: str = "vllm"
|
||||
root: str | None = None
|
||||
parent: str | None = None
|
||||
max_model_len: int | None = None
|
||||
permission: list[ModelPermission] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ModelList(OpenAIBaseModel):
|
||||
object: str = "list"
|
||||
data: list[ModelCard] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PromptTokenUsageInfo(OpenAIBaseModel):
|
||||
cached_tokens: int | None = None
|
||||
created_cache_tokens: int | None = None
|
||||
multimodal_tokens: dict[str, int] | None = None
|
||||
"""Prompt tokens contributed by each input modality, keyed by modality name
|
||||
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
|
||||
placeholder tokens already counted in `prompt_tokens`; `None` when the
|
||||
request has no multimodal input."""
|
||||
|
||||
|
||||
class CompletionTokenUsageInfo(OpenAIBaseModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class UsageInfo(OpenAIBaseModel):
|
||||
prompt_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
completion_tokens: int | None = 0
|
||||
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||
@@ -6,13 +6,13 @@ from fastapi import Request
|
||||
|
||||
from vllm import PromptType, SamplingParams, envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
|
||||
@@ -2,13 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..engine.protocol import ErrorInfo, ErrorResponse
|
||||
from .utils import sanitize_message
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -5,7 +5,7 @@ from http import HTTPStatus
|
||||
from fastapi import HTTPException, Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..utils import sanitize_message
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
|
||||
@@ -5,8 +5,7 @@ from fastapi import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.launchers.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.exceptions import VLLMError
|
||||
from vllm.exceptions import GenerationError, VLLMError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends, FastAPI, HTTPException,
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest
|
||||
|
||||
@@ -7,11 +7,9 @@ from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.api_router import models
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
|
||||
@@ -12,8 +12,8 @@ from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
DetokenizeRequest,
|
||||
DetokenizeResponse,
|
||||
|
||||
@@ -11,12 +11,8 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
|
||||
|
||||
@@ -6,11 +6,11 @@ from typing import Any, Final
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
DetokenizeRequest,
|
||||
|
||||
@@ -17,7 +17,7 @@ from starlette.background import BackgroundTask, BackgroundTasks
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import current_formatter_type, init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -16,14 +16,13 @@ from transformers import PreTrainedTokenizerBase
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
@@ -13,7 +13,7 @@ from fastapi import WebSocket
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -6,10 +6,7 @@ from typing import Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
OpenAIBaseModel,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
# Client -> Server Events
|
||||
|
||||
@@ -8,7 +8,6 @@ from typing import Annotated
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
with_cancellation,
|
||||
@@ -16,6 +15,7 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
||||
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from .protocol import TranscriptionRequest, TranscriptionResponseVariant
|
||||
from .serving import OpenAIServingTranscription
|
||||
|
||||
|
||||
@@ -13,11 +13,8 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import (
|
||||
|
||||
@@ -5,15 +5,13 @@ from collections.abc import AsyncGenerator
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranscriptionRequest,
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Annotated
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
with_cancellation,
|
||||
|
||||
@@ -12,11 +12,8 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import (
|
||||
|
||||
@@ -5,11 +5,9 @@ from collections.abc import AsyncGenerator
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
"""Custom exceptions for vLLM."""
|
||||
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -116,3 +117,11 @@ class VLLMUnprocessableEntityError(VLLMClientError):
|
||||
if self.value is not None:
|
||||
extras.append(f"value={self.value}")
|
||||
return f"{base} ({', '.join(extras)})" if extras else base
|
||||
|
||||
|
||||
class GenerationError(VLLMServerError):
|
||||
"""raised when finish_reason indicates internal server error (500)"""
|
||||
|
||||
def __init__(self, message: str = "Internal server error"):
|
||||
super().__init__(message)
|
||||
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
||||
@@ -15,16 +15,16 @@ from vllm.entrypoints.chat_utils import (
|
||||
get_tool_call_id_type,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
|
||||
@@ -19,13 +19,13 @@ from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
|
||||
@@ -17,7 +17,7 @@ import json
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
|
||||
@@ -27,15 +27,15 @@ from xgrammar.structural_tag import (
|
||||
)
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
FunctionCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
extract_function_from_recipient,
|
||||
get_streamable_parser_for_assistant,
|
||||
|
||||
@@ -29,9 +29,7 @@ import json
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ExtractedToolCallInformation,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
|
||||
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaFunctionCall, DeltaToolCall
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaFunctionCall, DeltaToolCall
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
)
|
||||
|
||||
@@ -28,10 +28,7 @@ from mistral_common.protocol.instruct.tool_calls import (
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokens
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
@@ -39,6 +36,9 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
@@ -19,10 +19,10 @@ from typing import TYPE_CHECKING
|
||||
from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
|
||||
@@ -15,8 +15,8 @@ from vllm.utils.import_utils import import_from_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from collections.abc import Iterable, Sequence
|
||||
from itertools import islice
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, NamedTuple, TypedDict, TypeGuard
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict, TypeGuard
|
||||
|
||||
import regex as re
|
||||
import xgrammar as xgr
|
||||
@@ -25,20 +25,20 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
CitationSource,
|
||||
CohereDeltaMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
REPLACEMENT_CHAR = "\ufffd"
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParse
|
||||
from .identity_reasoning_parser import IdentityReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -2,17 +2,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -4,17 +4,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TypedDict
|
||||
from typing import TYPE_CHECKING, TypedDict
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Must match the chat template's own default for ``reasoning_effort``.
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -28,7 +28,7 @@ from typing import TYPE_CHECKING
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -4,9 +4,7 @@
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.parser.engine.registered_adapters import MinimaxM2ParserReasoningAdapter
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -19,8 +19,8 @@ from collections.abc import Iterable, Sequence
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, ToolCall
|
||||
from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
@@ -21,12 +22,12 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponseStreamChoice,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, ToolCall, UsageInfo
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
GenerateStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
extract_instructions_from_messages,
|
||||
@@ -27,6 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
|
||||
@@ -15,14 +15,14 @@ from openai.types.responses import (
|
||||
from openai.types.responses.function_tool import FunctionTool
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user