[Frontend] Move engine/protocol.py out openai folder (#54492)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -342,7 +342,7 @@ steps:
|
|||||||
- vllm/entrypoints/chat_utils.py
|
- vllm/entrypoints/chat_utils.py
|
||||||
- vllm/entrypoints/mcp/
|
- vllm/entrypoints/mcp/
|
||||||
- vllm/entrypoints/openai/chat_completion/protocol.py
|
- vllm/entrypoints/openai/chat_completion/protocol.py
|
||||||
- vllm/entrypoints/openai/engine/protocol.py
|
- vllm/entrypoints/generate/base/protocol.py
|
||||||
- vllm/envs.py
|
- vllm/envs.py
|
||||||
- vllm/exceptions.py
|
- vllm/exceptions.py
|
||||||
- vllm/inputs/
|
- vllm/inputs/
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ pub struct JsonSchemaFormat {
|
|||||||
/// that we can support the vLLM-specific `structural_tag` variant.
|
/// that we can support the vLLM-specific `structural_tag` variant.
|
||||||
///
|
///
|
||||||
/// Original Python definitions:
|
/// Original Python definitions:
|
||||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/entrypoints/openai/engine/protocol.py#L116-L157>
|
/// <https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/generate/base/protocol.py#L71-L112>
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
#[serde(tag = "type", rename_all = "snake_case")]
|
||||||
pub enum ResponseFormat {
|
pub enum ResponseFormat {
|
||||||
|
|||||||
@@ -32,6 +32,11 @@ from vllm.entrypoints.anthropic.serving import (
|
|||||||
AnthropicServingMessages,
|
AnthropicServingMessages,
|
||||||
_build_anthropic_usage,
|
_build_anthropic_usage,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
DeltaFunctionCall,
|
||||||
|
DeltaMessage,
|
||||||
|
DeltaToolCall,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
ChatCompletionResponseChoice,
|
ChatCompletionResponseChoice,
|
||||||
@@ -39,13 +44,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.serve.engine.protocol import PromptTokenUsageInfo, UsageInfo
|
||||||
DeltaFunctionCall,
|
|
||||||
DeltaMessage,
|
|
||||||
DeltaToolCall,
|
|
||||||
PromptTokenUsageInfo,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||||
validation_exception_handler,
|
validation_exception_handler,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ from vllm.entrypoints.cohere.protocol import (
|
|||||||
CohereChatV2Response,
|
CohereChatV2Response,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||||
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
||||||
http_exception_handler,
|
http_exception_handler,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ from vllm.entrypoints.generate.generative_scoring.serving import (
|
|||||||
GenerativeScoringResponse,
|
GenerativeScoringResponse,
|
||||||
ServingGenerativeScoring,
|
ServingGenerativeScoring,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from vllm.entrypoints.launchers.run_batch import (
|
|||||||
make_transcription_wrapper,
|
make_transcription_wrapper,
|
||||||
upload_data,
|
upload_data,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.utils.mem_constants import MiB_bytes
|
from vllm.utils.mem_constants import MiB_bytes
|
||||||
|
|
||||||
|
|||||||
@@ -8,13 +8,13 @@ import httpx
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||||
OpenAIServingChatBatch,
|
OpenAIServingChatBatch,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
BatchChatCompletionRequest,
|
BatchChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import RequestResponseMetadata
|
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
|
|
||||||
# any model with a chat template defined in tokenizer_config should work here
|
# any model with a chat template defined in tokenizer_config should work here
|
||||||
|
|||||||
@@ -13,11 +13,10 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ from tests.entrypoints.openai.utils import (
|
|||||||
from tests.utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||||
from vllm.config import MultiModalConfig
|
from vllm.config import MultiModalConfig
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
|
RequestResponseMetadata,
|
||||||
|
)
|
||||||
from vllm.entrypoints.generate.base.serving import build_per_request_timing_metrics
|
from vllm.entrypoints.generate.base.serving import build_per_request_timing_metrics
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
@@ -31,17 +35,13 @@ from vllm.entrypoints.openai.chat_completion.serving import (
|
|||||||
_make_completion_tokens_details,
|
_make_completion_tokens_details,
|
||||||
_make_prompt_tokens_details,
|
_make_prompt_tokens_details,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
|
||||||
ErrorResponse,
|
|
||||||
RequestResponseMetadata,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import (
|
from vllm.entrypoints.openai.models.serving import (
|
||||||
BaseModelPath,
|
BaseModelPath,
|
||||||
OpenAIModelRegistry,
|
OpenAIModelRegistry,
|
||||||
OpenAIServingModels,
|
OpenAIServingModels,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
from vllm.multimodal.inputs import PlaceholderRange
|
from vllm.multimodal.inputs import PlaceholderRange
|
||||||
|
|||||||
@@ -8,16 +8,13 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
GenerationError,
|
|
||||||
RequestResponseMetadata,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ import pytest
|
|||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
|
|||||||
@@ -7,8 +7,9 @@ from unittest.mock import MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing, GenerationError
|
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||||
from vllm.envs import disable_envs_cache
|
from vllm.envs import disable_envs_cache
|
||||||
|
from vllm.exceptions import GenerationError
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ExtractedToolCallInformation,
|
ExtractedToolCallInformation,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
|
|||||||
@@ -25,14 +25,13 @@ from openai.types.responses.tool import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
ErrorResponse,
|
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext
|
from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
ResponseCreatedEvent,
|
ResponseCreatedEvent,
|
||||||
@@ -49,6 +48,7 @@ from vllm.entrypoints.openai.responses.serving import (
|
|||||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||||
StreamingState,
|
StreamingState,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.inputs import tokens_input
|
from vllm.inputs import tokens_input
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.parser.harmony import Segment
|
from vllm.parser.harmony import Segment
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
|
|||||||
@@ -26,9 +26,9 @@ from openai.types.shared import Reasoning
|
|||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.inputs import tokens_input
|
from vllm.inputs import tokens_input
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.renderers.params import ChatParams
|
from vllm.renderers.params import ChatParams
|
||||||
|
|||||||
@@ -5,6 +5,13 @@ import pytest
|
|||||||
from openai.types.chat.chat_completion import ChatCompletion as OpenAIChatCompletion
|
from openai.types.chat.chat_completion import ChatCompletion as OpenAIChatCompletion
|
||||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
DeltaFunctionCall,
|
||||||
|
DeltaMessage,
|
||||||
|
DeltaToolCall,
|
||||||
|
FunctionCall,
|
||||||
|
ToolCall,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
@@ -13,15 +20,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaFunctionCall,
|
|
||||||
DeltaMessage,
|
|
||||||
DeltaToolCall,
|
|
||||||
FunctionCall,
|
|
||||||
ToolCall,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
|
|
||||||
pytestmark = pytest.mark.skip_global_cleanup
|
pytestmark = pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
|
|
||||||
|
|
||||||
async def accumulate_streaming_response(
|
async def accumulate_streaming_response(
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
|||||||
GenerateResponseStreamChoice,
|
GenerateResponseStreamChoice,
|
||||||
GenerateStreamResponse,
|
GenerateStreamResponse,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||||
|
|
||||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||||
|
|
||||||
@@ -40,7 +41,6 @@ def _make_stream_chunk(
|
|||||||
usage: dict | None = None,
|
usage: dict | None = None,
|
||||||
) -> GenerateStreamResponse:
|
) -> GenerateStreamResponse:
|
||||||
"""Build a GenerateStreamResponse SSE chunk."""
|
"""Build a GenerateStreamResponse SSE chunk."""
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
|
||||||
|
|
||||||
return GenerateStreamResponse(
|
return GenerateStreamResponse(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
@@ -61,7 +61,6 @@ def _make_usage_chunk(
|
|||||||
request_id: str = "test-req",
|
request_id: str = "test-req",
|
||||||
) -> GenerateStreamResponse:
|
) -> GenerateStreamResponse:
|
||||||
"""Build a usage only final SSE chunk (empty choices)."""
|
"""Build a usage only final SSE chunk (empty choices)."""
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
|
||||||
|
|
||||||
return GenerateStreamResponse(
|
return GenerateStreamResponse(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
@@ -473,7 +472,6 @@ class TestServingDerenderStreamErrorHandling:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
||||||
async def test_completion_stream_bad_state_returns_400(self, exc):
|
async def test_completion_stream_bad_state_returns_400(self, exc):
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
DerenderCompletionStreamRequest,
|
DerenderCompletionStreamRequest,
|
||||||
)
|
)
|
||||||
@@ -492,7 +490,6 @@ class TestServingDerenderStreamErrorHandling:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
||||||
async def test_chat_stream_bad_state_returns_400(self, exc):
|
async def test_chat_stream_bad_state_returns_400(self, exc):
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
DerenderChatStreamRequest,
|
DerenderChatStreamRequest,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import GenerationError, StreamOptions
|
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
@@ -17,6 +17,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
|||||||
GenerateResponse,
|
GenerateResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||||
|
from vllm.exceptions import GenerationError
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers import renderer_from_config
|
from vllm.renderers import renderer_from_config
|
||||||
|
|||||||
@@ -9,13 +9,11 @@ import pytest
|
|||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.lora.protocol import (
|
from vllm.entrypoints.serve.lora.protocol import (
|
||||||
LoadLoRAAdapterRequest,
|
LoadLoRAAdapterRequest,
|
||||||
UnloadLoRAAdapterRequest,
|
UnloadLoRAAdapterRequest,
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from argparse import Namespace
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||||
from vllm.entrypoints.serve.utils import api_utils
|
from vllm.entrypoints.serve.utils import api_utils
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
get_max_tokens,
|
get_max_tokens,
|
||||||
|
|||||||
+2
-4
@@ -19,11 +19,9 @@ import pytest
|
|||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||||
ErrorResponse,
|
|
||||||
RequestResponseMetadata,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.speech_to_text.base.serving import (
|
from vllm.entrypoints.speech_to_text.base.serving import (
|
||||||
SpeechToTextBaseServing,
|
SpeechToTextBaseServing,
|
||||||
asr_inter_chunk_separator,
|
asr_inter_chunk_separator,
|
||||||
|
|||||||
@@ -13,10 +13,10 @@ import json
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
|
|
||||||
|
|
||||||
def _build_token_id_map(parser) -> dict[str, int]:
|
def _build_token_id_map(parser) -> dict[str, int]:
|
||||||
|
|||||||
@@ -14,10 +14,10 @@ from tests.parser.engine.streaming_helpers import (
|
|||||||
collect_tool_arguments,
|
collect_tool_arguments,
|
||||||
simulate_tool_streaming,
|
simulate_tool_streaming,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.parser.gemma4 import Gemma4Parser
|
from vllm.parser.gemma4 import Gemma4Parser
|
||||||
|
|
||||||
# ── Special token IDs (arbitrary but consistent) ─────────────────────
|
# ── Special token IDs (arbitrary but consistent) ─────────────────────
|
||||||
|
|||||||
@@ -15,15 +15,15 @@ import pytest
|
|||||||
import regex as re
|
import regex as re
|
||||||
|
|
||||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
ChatCompletionRequest,
|
|
||||||
ChatCompletionToolsParam,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
from vllm.parser.engine.adapters import make_adapters
|
from vllm.parser.engine.adapters import make_adapters
|
||||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||||
|
|||||||
@@ -25,16 +25,16 @@ from mistral_common.protocol.instruct.tool_calls import (
|
|||||||
)
|
)
|
||||||
from partial_json_parser.core.options import Allow
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
ChatCompletionRequest,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
ExtractedToolCallInformation,
|
ExtractedToolCallInformation,
|
||||||
StructuralTagResponseFormat,
|
StructuralTagResponseFormat,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
from vllm.parser.engine.events import EventType
|
from vllm.parser.engine.events import EventType
|
||||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||||
from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
|
from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ from openai_harmony import (
|
|||||||
from transformers import AutoTokenizer, GenerationConfig
|
from transformers import AutoTokenizer, GenerationConfig
|
||||||
|
|
||||||
from vllm.config import StructuredOutputsConfig, VllmConfig
|
from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
get_encoding,
|
get_encoding,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ import json
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
|
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from vllm.config import PoolerConfig, VllmConfig
|
from vllm.config import PoolerConfig, VllmConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.inputs import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.plugins.io_processors.interface import IOProcessor
|
from vllm.plugins.io_processors.interface import IOProcessor
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ from typing import Literal, get_args
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
|
||||||
from vllm.entrypoints.pooling.base.protocol import (
|
from vllm.entrypoints.pooling.base.protocol import (
|
||||||
CompletionRequestMixin,
|
CompletionRequestMixin,
|
||||||
EmbedRequestMixin,
|
EmbedRequestMixin,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
|
|
||||||
EmbedTask = Literal[
|
EmbedTask = Literal[
|
||||||
"sparse",
|
"sparse",
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ from collections.abc import Iterator, Sequence
|
|||||||
from typing import cast
|
from typing import cast
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.inputs import PromptType, TokensPrompt
|
from vllm.inputs import PromptType, TokensPrompt
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.plugins.io_processors.interface import IOProcessor
|
from vllm.plugins.io_processors.interface import IOProcessor
|
||||||
|
|||||||
@@ -5,8 +5,8 @@ from typing import Literal, get_args
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
|
||||||
from vllm.entrypoints.pooling.base.protocol import CompletionRequestMixin
|
from vllm.entrypoints.pooling.base.protocol import CompletionRequestMixin
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
|
|
||||||
InputType = Literal["query", "document"]
|
InputType = Literal["query", "document"]
|
||||||
INPUT_TYPES: tuple[InputType, ...] = get_args(InputType)
|
INPUT_TYPES: tuple[InputType, ...] = get_args(InputType)
|
||||||
|
|||||||
@@ -12,14 +12,14 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
ChatCompletionRequest,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
JsonSchemaResponseFormat,
|
JsonSchemaResponseFormat,
|
||||||
ResponseFormat,
|
ResponseFormat,
|
||||||
StructuralTagResponseFormat,
|
StructuralTagResponseFormat,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.reasoning.cohere_command_reasoning_parser import (
|
from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||||
CohereCommand3ReasoningParser,
|
CohereCommand3ReasoningParser,
|
||||||
|
|||||||
@@ -4,8 +4,8 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.reasoning import ReasoningParserManager
|
from vllm.reasoning import ReasoningParserManager
|
||||||
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
||||||
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
|
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
|
||||||
|
|||||||
@@ -5,8 +5,8 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser
|
from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|
||||||
|
|||||||
@@ -3,10 +3,10 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||||
from vllm.parser.parser_manager import ParserManager
|
from vllm.parser.parser_manager import ParserManager
|
||||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
from vllm.utils.mistral import is_mistral_tokenizer
|
from vllm.utils.mistral import is_mistral_tokenizer
|
||||||
|
|
||||||
|
|||||||
@@ -7,11 +7,11 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.tool_parsers import ToolParserManager
|
from vllm.tool_parsers import ToolParserManager
|
||||||
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
|
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -7,12 +7,12 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
|
from vllm.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,12 +9,12 @@ from unittest.mock import Mock
|
|||||||
import pytest
|
import pytest
|
||||||
from openai.types.responses import ResponseFunctionToolCall
|
from openai.types.responses import ResponseFunctionToolCall
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
|
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
|
||||||
|
|
||||||
MODEL = "ibm-granite/granite-4.0-h-tiny"
|
MODEL = "ibm-granite/granite-4.0-h-tiny"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,12 +8,12 @@ from unittest.mock import Mock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser
|
from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import partial_json_parser
|
|||||||
import pytest
|
import pytest
|
||||||
from partial_json_parser.core.options import Allow
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall, ToolCall
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall, ToolCall
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
|
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation
|
from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation
|
||||||
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
||||||
|
|
||||||
LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,11 +9,11 @@ from unittest.mock import MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
from vllm.tool_parsers.minicpm5xml_tool_parser import MiniCPM5XMLToolParser
|
from vllm.tool_parsers.minicpm5xml_tool_parser import MiniCPM5XMLToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -6,11 +6,11 @@ from typing import Any
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.tool_parsers import ToolParserManager
|
from vllm.tool_parsers import ToolParserManager
|
||||||
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
|
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ import pytest
|
|||||||
from openai.types.responses.function_tool import FunctionTool
|
from openai.types.responses.function_tool import FunctionTool
|
||||||
from xgrammar import StructuralTag
|
from xgrammar import StructuralTag
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
|
FunctionCall,
|
||||||
|
ToolCall,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedFunction,
|
ChatCompletionNamedFunction,
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
@@ -16,11 +21,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
|
||||||
FunctionCall,
|
|
||||||
ToolCall,
|
|
||||||
)
|
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
|
|||||||
@@ -6,15 +6,15 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
ChatCompletionRequest,
|
|
||||||
ChatCompletionToolsParam,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tool_parsers.step3p5_tool_parser import Step3p5ToolParser
|
from vllm.tool_parsers.step3p5_tool_parser import Step3p5ToolParser
|
||||||
|
|||||||
@@ -6,12 +6,12 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tool_parsers.xlam_tool_parser import xLAMToolParser
|
from vllm.tool_parsers.xlam_tool_parser import xLAMToolParser
|
||||||
|
|||||||
@@ -3,13 +3,13 @@
|
|||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ExtractedToolCallInformation,
|
ExtractedToolCallInformation,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser
|
from vllm.tool_parsers import ToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -5,10 +5,10 @@ import json
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm.entrypoints.anthropic.protocol import (
|
|||||||
AnthropicMessagesResponse,
|
AnthropicMessagesResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||||
create_error_response,
|
create_error_response,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -30,6 +30,11 @@ from vllm.entrypoints.anthropic.protocol import (
|
|||||||
AnthropicUsage,
|
AnthropicUsage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
JsonSchemaResponseFormat,
|
||||||
|
ResponseFormat,
|
||||||
|
StreamOptions,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
@@ -38,14 +43,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
JsonSchemaResponseFormat,
|
|
||||||
ResponseFormat,
|
|
||||||
StreamOptions,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
|
|||||||
from starlette.middleware.base import BaseHTTPMiddleware
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
|
|||||||
@@ -29,11 +29,9 @@ from typing import Any, Literal
|
|||||||
|
|
||||||
from pydantic import Field, model_serializer
|
from pydantic import Field, model_serializer
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||||
DeltaMessage,
|
|
||||||
OpenAIBaseModel,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class CitationSource(OpenAIBaseModel):
|
class CitationSource(OpenAIBaseModel):
|
||||||
|
|||||||
@@ -71,6 +71,11 @@ from vllm.entrypoints.cohere.protocol import (
|
|||||||
ToolCallStartEvent,
|
ToolCallStartEvent,
|
||||||
ToolPlanDeltaEvent,
|
ToolPlanDeltaEvent,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
JsonSchemaResponseFormat,
|
||||||
|
ResponseFormat,
|
||||||
|
StreamOptions,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
@@ -79,13 +84,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
JsonSchemaResponseFormat,
|
|
||||||
ResponseFormat,
|
|
||||||
StreamOptions,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.parser.abstract_parser import Parser
|
from vllm.parser.abstract_parser import Parser
|
||||||
@@ -1761,7 +1761,6 @@ class CohereServingChatV2(OpenAIServingChat):
|
|||||||
# so the router can translate the envelope uniformly. ``param``
|
# so the router can translate the envelope uniformly. ``param``
|
||||||
# is accepted for signature parity with the base class but is
|
# is accepted for signature parity with the base class but is
|
||||||
# not surfaced in the Cohere wire format.
|
# not surfaced in the Cohere wire format.
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo
|
|
||||||
|
|
||||||
del param # unused; kept for signature compatibility
|
del param # unused; kept for signature compatibility
|
||||||
return ErrorResponse(
|
return ErrorResponse(
|
||||||
|
|||||||
+3
-110
@@ -4,9 +4,7 @@
|
|||||||
# Adapted from
|
# Adapted from
|
||||||
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||||
import json
|
import json
|
||||||
import time
|
from typing import Annotated, Any, Literal, TypeAlias
|
||||||
from http import HTTPStatus
|
|
||||||
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
|
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
from pydantic import (
|
from pydantic import (
|
||||||
@@ -14,16 +12,15 @@ from pydantic import (
|
|||||||
ConfigDict,
|
ConfigDict,
|
||||||
Field,
|
Field,
|
||||||
model_serializer,
|
model_serializer,
|
||||||
model_validator,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.config.utils import replace
|
from vllm.config.utils import replace
|
||||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||||
from vllm.exceptions import VLLMServerError, VLLMValidationError
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.sampling_params import StructuredOutputsParams
|
from vllm.sampling_params import StructuredOutputsParams
|
||||||
from vllm.utils import random_uuid
|
|
||||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -33,102 +30,6 @@ StopParam: TypeAlias = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class OpenAIBaseModel(BaseModel):
|
|
||||||
# OpenAI API does allow extra fields
|
|
||||||
model_config = ConfigDict(extra="allow")
|
|
||||||
|
|
||||||
# Cache class field names
|
|
||||||
field_names: ClassVar[set[str] | None] = None
|
|
||||||
|
|
||||||
@model_validator(mode="wrap")
|
|
||||||
@classmethod
|
|
||||||
def __log_extra_fields__(cls, data, handler):
|
|
||||||
result = handler(data)
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
return result
|
|
||||||
field_names = cls.field_names
|
|
||||||
if field_names is None:
|
|
||||||
# Get all class field names and their potential aliases
|
|
||||||
field_names = set()
|
|
||||||
for field_name, field in cls.model_fields.items():
|
|
||||||
field_names.add(field_name)
|
|
||||||
if alias := getattr(field, "alias", None):
|
|
||||||
field_names.add(alias)
|
|
||||||
cls.field_names = field_names
|
|
||||||
|
|
||||||
# Compare against both field names and aliases
|
|
||||||
if any(k not in field_names for k in data):
|
|
||||||
logger.debug(
|
|
||||||
"The following fields were present in the request but ignored: %s",
|
|
||||||
data.keys() - field_names,
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
class ErrorInfo(OpenAIBaseModel):
|
|
||||||
message: str
|
|
||||||
type: str
|
|
||||||
param: str | None = None
|
|
||||||
code: int
|
|
||||||
|
|
||||||
|
|
||||||
class ErrorResponse(OpenAIBaseModel):
|
|
||||||
error: ErrorInfo
|
|
||||||
|
|
||||||
|
|
||||||
class ModelPermission(OpenAIBaseModel):
|
|
||||||
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
|
|
||||||
object: str = "model_permission"
|
|
||||||
created: int = Field(default_factory=lambda: int(time.time()))
|
|
||||||
allow_create_engine: bool = False
|
|
||||||
allow_sampling: bool = True
|
|
||||||
allow_logprobs: bool = True
|
|
||||||
allow_search_indices: bool = False
|
|
||||||
allow_view: bool = True
|
|
||||||
allow_fine_tuning: bool = False
|
|
||||||
organization: str = "*"
|
|
||||||
group: str | None = None
|
|
||||||
is_blocking: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
class ModelCard(OpenAIBaseModel):
|
|
||||||
id: str
|
|
||||||
object: str = "model"
|
|
||||||
created: int = Field(default_factory=lambda: int(time.time()))
|
|
||||||
owned_by: str = "vllm"
|
|
||||||
root: str | None = None
|
|
||||||
parent: str | None = None
|
|
||||||
max_model_len: int | None = None
|
|
||||||
permission: list[ModelPermission] = Field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
class ModelList(OpenAIBaseModel):
|
|
||||||
object: str = "list"
|
|
||||||
data: list[ModelCard] = Field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
class PromptTokenUsageInfo(OpenAIBaseModel):
|
|
||||||
cached_tokens: int | None = None
|
|
||||||
created_cache_tokens: int | None = None
|
|
||||||
multimodal_tokens: dict[str, int] | None = None
|
|
||||||
"""Prompt tokens contributed by each input modality, keyed by modality name
|
|
||||||
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
|
|
||||||
placeholder tokens already counted in `prompt_tokens`; `None` when the
|
|
||||||
request has no multimodal input."""
|
|
||||||
|
|
||||||
|
|
||||||
class CompletionTokenUsageInfo(OpenAIBaseModel):
|
|
||||||
reasoning_tokens: int = 0
|
|
||||||
|
|
||||||
|
|
||||||
class UsageInfo(OpenAIBaseModel):
|
|
||||||
prompt_tokens: int = 0
|
|
||||||
total_tokens: int = 0
|
|
||||||
completion_tokens: int | None = 0
|
|
||||||
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
|
||||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||||
"""Per-request speculative-decoding acceptance metrics.
|
"""Per-request speculative-decoding acceptance metrics.
|
||||||
|
|
||||||
@@ -437,11 +338,3 @@ class DeltaMessage(OpenAIBaseModel):
|
|||||||
if len(data.get("tool_calls", [])) == 0:
|
if len(data.get("tool_calls", [])) == 0:
|
||||||
data.pop("tool_calls", None)
|
data.pop("tool_calls", None)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
class GenerationError(VLLMServerError):
|
|
||||||
"""raised when finish_reason indicates internal server error (500)"""
|
|
||||||
|
|
||||||
def __init__(self, message: str = "Internal server error"):
|
|
||||||
super().__init__(message)
|
|
||||||
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
|
||||||
@@ -5,27 +5,28 @@ import time
|
|||||||
from collections.abc import Awaitable, Mapping
|
from collections.abc import Awaitable, Mapping
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
from typing import ClassVar, Generic, TypeVar
|
||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from pydantic import ConfigDict
|
from pydantic import ConfigDict
|
||||||
from starlette.datastructures import Headers
|
from starlette.datastructures import Headers
|
||||||
|
|
||||||
|
from vllm import RequestOutput
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
GenerationError,
|
|
||||||
PerRequestMetrics,
|
PerRequestMetrics,
|
||||||
SpeculativeDecodingMetrics,
|
SpeculativeDecodingMetrics,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
from vllm.exceptions import GenerationError
|
||||||
from vllm.inputs import EngineInput
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob, PromptLogprobs
|
from vllm.logprobs import Logprob, PromptLogprobs
|
||||||
@@ -38,9 +39,6 @@ from vllm.tracing import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.metrics.stats import RequestStateStats
|
from vllm.v1.metrics.stats import RequestStateStats
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from vllm.outputs import RequestOutput
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from vllm.entrypoints.generate.generative_scoring.serving import (
|
|||||||
GenerativeScoringResponse,
|
GenerativeScoringResponse,
|
||||||
ServingGenerativeScoring,
|
ServingGenerativeScoring,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -18,12 +18,12 @@ from fastapi import Request
|
|||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
OpenAIBaseModel,
|
OpenAIBaseModel,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|
||||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.inputs import EngineInput, tokens_input
|
from vllm.inputs import EngineInput, tokens_input
|
||||||
|
|||||||
@@ -35,11 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorInfo,
|
|
||||||
ErrorResponse,
|
|
||||||
OpenAIBaseModel,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.pooling.embed.protocol import (
|
from vllm.entrypoints.pooling.embed.protocol import (
|
||||||
EmbeddingRequest,
|
EmbeddingRequest,
|
||||||
EmbeddingResponse,
|
EmbeddingResponse,
|
||||||
@@ -51,6 +46,11 @@ from vllm.entrypoints.pooling.scoring.protocol import (
|
|||||||
ScoreResponse,
|
ScoreResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve import create_error_response
|
from vllm.entrypoints.serve import create_error_response
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import (
|
||||||
|
ErrorInfo,
|
||||||
|
ErrorResponse,
|
||||||
|
OpenAIBaseModel,
|
||||||
|
)
|
||||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||||
TranscriptionRequest,
|
TranscriptionRequest,
|
||||||
TranscriptionResponse,
|
TranscriptionResponse,
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from http import HTTPStatus
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
|
|
||||||
from vllm.entrypoints.chat_utils import ConversationMessage
|
from vllm.entrypoints.chat_utils import ConversationMessage
|
||||||
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
BatchChatCompletionRequest,
|
BatchChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
@@ -16,11 +17,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||||
ErrorResponse,
|
|
||||||
RequestResponseMetadata,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||||
from vllm.inputs import EngineInput
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|||||||
@@ -23,21 +23,20 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
AnyResponseFormat,
|
AnyResponseFormat,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
OpenAIBaseModel,
|
|
||||||
PerRequestMetrics,
|
PerRequestMetrics,
|
||||||
StopParam,
|
StopParam,
|
||||||
StreamOptions,
|
StreamOptions,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
UsageInfo,
|
|
||||||
structured_outputs_from_response_format,
|
structured_outputs_from_response_format,
|
||||||
validate_structural_tag_response_format,
|
validate_structural_tag_response_format,
|
||||||
validate_structured_outputs_structural_tag,
|
validate_structured_outputs_structural_tag,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
|
|||||||
@@ -16,9 +16,15 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
make_tool_call_id,
|
make_tool_call_id,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
DeltaMessage,
|
||||||
|
FunctionCall,
|
||||||
|
PerRequestMetrics,
|
||||||
|
RequestResponseMetadata,
|
||||||
|
ToolCall,
|
||||||
|
)
|
||||||
from vllm.entrypoints.generate.base.serving import (
|
from vllm.entrypoints.generate.base.serving import (
|
||||||
GenerateBaseServing,
|
GenerateBaseServing,
|
||||||
GenerationError,
|
|
||||||
build_per_request_timing_metrics,
|
build_per_request_timing_metrics,
|
||||||
build_spec_decoding_metrics,
|
build_spec_decoding_metrics,
|
||||||
clamp_prompt_logprobs,
|
clamp_prompt_logprobs,
|
||||||
@@ -36,23 +42,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import (
|
||||||
CompletionTokenUsageInfo,
|
CompletionTokenUsageInfo,
|
||||||
DeltaMessage,
|
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
FunctionCall,
|
|
||||||
PerRequestMetrics,
|
|
||||||
PromptTokenUsageInfo,
|
PromptTokenUsageInfo,
|
||||||
RequestResponseMetadata,
|
|
||||||
ToolCall,
|
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||||
maybe_filter_parallel_tool_calls,
|
maybe_filter_parallel_tool_calls,
|
||||||
)
|
)
|
||||||
|
from vllm.exceptions import GenerationError
|
||||||
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
from vllm.entrypoints.openai.sse_keep_alive import with_sse_keep_alive
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -10,17 +10,16 @@ from pydantic import Field, model_validator
|
|||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
AnyResponseFormat,
|
AnyResponseFormat,
|
||||||
OpenAIBaseModel,
|
|
||||||
PerRequestMetrics,
|
PerRequestMetrics,
|
||||||
StopParam,
|
StopParam,
|
||||||
StreamOptions,
|
StreamOptions,
|
||||||
UsageInfo,
|
|
||||||
structured_outputs_from_response_format,
|
structured_outputs_from_response_format,
|
||||||
validate_structural_tag_response_format,
|
validate_structural_tag_response_format,
|
||||||
validate_structured_outputs_structural_tag,
|
validate_structured_outputs_structural_tag,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
|
|||||||
@@ -10,9 +10,12 @@ from typing import cast
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
PerRequestMetrics,
|
||||||
|
RequestResponseMetadata,
|
||||||
|
)
|
||||||
from vllm.entrypoints.generate.base.serving import (
|
from vllm.entrypoints.generate.base.serving import (
|
||||||
GenerateBaseServing,
|
GenerateBaseServing,
|
||||||
GenerationError,
|
|
||||||
build_per_request_timing_metrics,
|
build_per_request_timing_metrics,
|
||||||
build_spec_decoding_metrics,
|
build_spec_decoding_metrics,
|
||||||
clamp_prompt_logprobs,
|
clamp_prompt_logprobs,
|
||||||
@@ -26,17 +29,15 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
CompletionResponseStreamChoice,
|
CompletionResponseStreamChoice,
|
||||||
CompletionStreamResponse,
|
CompletionStreamResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
PerRequestMetrics,
|
|
||||||
PromptTokenUsageInfo,
|
PromptTokenUsageInfo,
|
||||||
RequestResponseMetadata,
|
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.inputs import EngineInput
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
@@ -7,14 +7,14 @@ from http import HTTPStatus
|
|||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||||
|
from vllm.entrypoints.serve import create_error_response
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
ModelCard,
|
ModelCard,
|
||||||
ModelList,
|
ModelList,
|
||||||
ModelPermission,
|
ModelPermission,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
|
||||||
from vllm.entrypoints.serve import create_error_response
|
|
||||||
from vllm.entrypoints.serve.lora.protocol import (
|
from vllm.entrypoints.serve.lora.protocol import (
|
||||||
LoadLoRAAdapterRequest,
|
LoadLoRAAdapterRequest,
|
||||||
UnloadLoRAAdapterRequest,
|
UnloadLoRAAdapterRequest,
|
||||||
|
|||||||
@@ -8,13 +8,13 @@ from http import HTTPStatus
|
|||||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
ResponsesResponse,
|
ResponsesResponse,
|
||||||
StreamingResponsesResponse,
|
StreamingResponsesResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -24,11 +24,9 @@ from vllm import envs
|
|||||||
from vllm.entrypoints.chat_utils import (
|
from vllm.entrypoints.chat_utils import (
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.entrypoints.mcp.tool import Tool
|
from vllm.entrypoints.mcp.tool import Tool
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
FunctionCall,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
|
from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
ResponseInputOutputItem,
|
ResponseInputOutputItem,
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
|
from vllm.entrypoints.generate.base.protocol import StopParam
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||||
|
|||||||
@@ -31,16 +31,12 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.generate.base.serving import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
GenerateBaseServing,
|
|
||||||
GenerationError,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ErrorResponse,
|
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||||
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
build_harmony_preamble,
|
build_harmony_preamble,
|
||||||
@@ -89,9 +85,10 @@ from vllm.entrypoints.openai.responses.utils import (
|
|||||||
extract_function_tool_names,
|
extract_function_tool_names,
|
||||||
extract_tool_types,
|
extract_tool_types,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.inputs import EngineInput, tokens_input
|
from vllm.inputs import EngineInput, tokens_input
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob as SampleLogprob
|
from vllm.logprobs import Logprob as SampleLogprob
|
||||||
|
|||||||
@@ -61,8 +61,8 @@ from openai.types.responses.response_reasoning_item import (
|
|||||||
from openai.types.responses.tool import Tool
|
from openai.types.responses.tool import Tool
|
||||||
from openai_harmony import Message as HarmonyMessage
|
from openai_harmony import Message as HarmonyMessage
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage, DeltaToolCall
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
|
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
extract_function_from_recipient,
|
extract_function_from_recipient,
|
||||||
is_function_recipient,
|
is_function_recipient,
|
||||||
|
|||||||
@@ -31,11 +31,11 @@ from openai.types.responses.tool import Tool
|
|||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall, FunctionDefinition
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
|
from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.tool_parsers.utils import (
|
from vllm.tool_parsers.utils import (
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||||
from vllm.tasks import check_removed_pooling_task
|
from vllm.tasks import check_removed_pooling_task
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ from vllm import PoolingRequestOutput, envs
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||||
@@ -30,6 +29,7 @@ from vllm.tracing import (
|
|||||||
)
|
)
|
||||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||||
|
|
||||||
|
from ...serve.engine.protocol import ErrorResponse
|
||||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||||
from .io_processor import PoolingIOProcessor
|
from .io_processor import PoolingIOProcessor
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from typing import Annotated, TypeAlias
|
|||||||
from pydantic import BeforeValidator, Field
|
from pydantic import BeforeValidator, Field
|
||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from typing import TypeAlias
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import ClassificationOutput
|
from vllm.outputs import ClassificationOutput
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from http import HTTPStatus
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
from ..base.protocol import (
|
from ..base.protocol import (
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from http import HTTPStatus
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from pydantic import BeforeValidator, Field
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.renderers import TokenizeParams
|
from vllm.renderers import TokenizeParams
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from http import HTTPStatus
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, model_validator
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.renderers import TokenizeParams
|
from vllm.renderers import TokenizeParams
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse, Response
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||||
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import torch
|
|||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.outputs import PoolingRequestOutput
|
from vllm.outputs import PoolingRequestOutput
|
||||||
from vllm.tasks import SupportedTask
|
from vllm.tasks import SupportedTask
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from http import HTTPStatus
|
|||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,11 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
CompletionStreamResponse,
|
CompletionStreamResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import (
|
from vllm.entrypoints.openai.models.serving import (
|
||||||
OpenAIModelRegistry,
|
OpenAIModelRegistry,
|
||||||
OpenAIServingModels,
|
OpenAIServingModels,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.inputs import (
|
from vllm.inputs import (
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse
|
|||||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user