[Refactor] [6/N] to simplify the vLLM openai chat_completion serving architecture (#32240)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -6,8 +6,8 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.serving_engine import GenerationError, OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import json
|
|||||||
import pytest
|
import pytest
|
||||||
from openai.types.responses import ResponseFunctionToolCall
|
from openai.types.responses import ResponseFunctionToolCall
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
|
||||||
|
|
||||||
|
|
||||||
def test_function_call_dict_converted_to_object():
|
def test_function_call_dict_converted_to_object():
|
||||||
@@ -253,7 +253,7 @@ def test_function_call_validation_failure_logs_debug(caplog):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Mock the logger to verify debug was called
|
# Mock the logger to verify debug was called
|
||||||
with patch("vllm.entrypoints.openai.protocol.logger") as mock_logger:
|
with patch("vllm.entrypoints.openai.engine.protocol.logger") as mock_logger:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
ResponsesRequest(**request_data)
|
ResponsesRequest(**request_data)
|
||||||
|
|
||||||
|
|||||||
@@ -9,8 +9,9 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ErrorResponse
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import pytest
|
|||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
|
from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
|
|
||||||
from ...models.registry import HF_EXAMPLE_MODELS
|
from ...models.registry import HF_EXAMPLE_MODELS
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.protocol import CompletionRequest, ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
|
||||||
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from unittest.mock import Mock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
StructuredOutputsParams,
|
StructuredOutputsParams,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.tool_server import ToolServer
|
from vllm.entrypoints.tool_server import ToolServer
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.protocol import CompletionRequest, ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
|
||||||
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
|
|||||||
@@ -4,7 +4,10 @@ from openai_harmony import (
|
|||||||
Message,
|
Message,
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import serialize_message, serialize_messages
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
serialize_message,
|
||||||
|
serialize_messages,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_serialize_message() -> None:
|
def test_serialize_message() -> None:
|
||||||
|
|||||||
@@ -11,14 +11,16 @@ import pytest_asyncio
|
|||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
@@ -1517,12 +1519,12 @@ class TestCreateRemainingArgsDelta:
|
|||||||
|
|
||||||
def test_preserves_id_type_name(self):
|
def test_preserves_id_type_name(self):
|
||||||
"""Test that id, type, and name are preserved from original delta."""
|
"""Test that id, type, and name are preserved from original delta."""
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
|
|
||||||
original_delta = DeltaMessage(
|
original_delta = DeltaMessage(
|
||||||
tool_calls=[
|
tool_calls=[
|
||||||
@@ -1552,12 +1554,12 @@ class TestCreateRemainingArgsDelta:
|
|||||||
|
|
||||||
def test_matches_by_index(self):
|
def test_matches_by_index(self):
|
||||||
"""Test that the correct tool call is matched by index."""
|
"""Test that the correct tool call is matched by index."""
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
|
|
||||||
original_delta = DeltaMessage(
|
original_delta = DeltaMessage(
|
||||||
tool_calls=[
|
tool_calls=[
|
||||||
@@ -1588,12 +1590,12 @@ class TestCreateRemainingArgsDelta:
|
|||||||
|
|
||||||
def test_no_matching_tool_call(self):
|
def test_no_matching_tool_call(self):
|
||||||
"""Test graceful handling when no matching tool call is found."""
|
"""Test graceful handling when no matching tool call is found."""
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
|
|
||||||
original_delta = DeltaMessage(
|
original_delta = DeltaMessage(
|
||||||
tool_calls=[
|
tool_calls=[
|
||||||
@@ -1620,8 +1622,8 @@ class TestCreateRemainingArgsDelta:
|
|||||||
|
|
||||||
def test_function_is_none(self):
|
def test_function_is_none(self):
|
||||||
"""Test handling when original tool call has no function."""
|
"""Test handling when original tool call has no function."""
|
||||||
from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
|
||||||
|
|
||||||
original_delta = DeltaMessage(
|
original_delta = DeltaMessage(
|
||||||
tool_calls=[
|
tool_calls=[
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.serving_chat_stream_harmony import (
|
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
|
||||||
extract_harmony_streaming_delta,
|
extract_harmony_streaming_delta,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -82,7 +82,7 @@ class TestExtractHarmonyStreamingDelta:
|
|||||||
assert tools_streamed is False
|
assert tools_streamed is False
|
||||||
|
|
||||||
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
|
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
|
||||||
@patch("vllm.entrypoints.openai.serving_chat_stream_harmony.make_tool_call_id")
|
@patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
|
||||||
def test_new_tool_call(self, mock_make_tool_call_id, channel):
|
def test_new_tool_call(self, mock_make_tool_call_id, channel):
|
||||||
"""Test new tool call creation when recipient changes."""
|
"""Test new tool call creation when recipient changes."""
|
||||||
mock_make_tool_call_id.return_value = "call_test123"
|
mock_make_tool_call_id.return_value = "call_test123"
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from unittest.mock import Mock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.tokenizers.mistral import MistralTokenizer
|
from vllm.tokenizers.mistral import MistralTokenizer
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import pytest
|
|||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
LoadLoRAAdapterRequest,
|
LoadLoRAAdapterRequest,
|
||||||
UnloadLoRAAdapterRequest,
|
UnloadLoRAAdapterRequest,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from openai.types.responses.tool import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.context import ConversationContext
|
from vllm.entrypoints.context import ConversationContext
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, ResponsesRequest
|
||||||
from vllm.entrypoints.openai.serving_responses import (
|
from vllm.entrypoints.openai.serving_responses import (
|
||||||
OpenAIServingResponses,
|
OpenAIServingResponses,
|
||||||
_extract_allowed_tools_from_mcp_requests,
|
_extract_allowed_tools_from_mcp_requests,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import json
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
|
from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
|
|||||||
run_tool_extraction,
|
run_tool_extraction,
|
||||||
run_tool_extraction_streaming,
|
run_tool_extraction_streaming,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ExtractedToolCallInformation,
|
ExtractedToolCallInformation,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ import json
|
|||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
ChatCompletionResponseChoice,
|
ChatCompletionResponseChoice,
|
||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import pytest
|
|||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from tests.reasoning.utils import run_reasoning_extraction
|
from tests.reasoning.utils import run_reasoning_extraction
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
||||||
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
|
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
|
||||||
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
|
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
from vllm.tokenizers.mistral import MistralTokenizer
|
from vllm.tokenizers.mistral import MistralTokenizer
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
|
from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import json
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.tool_parsers.glm4_moe_tool_parser import (
|
from vllm.tool_parsers.glm4_moe_tool_parser import (
|
||||||
Glm4MoeModelToolParser,
|
Glm4MoeModelToolParser,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import partial_json_parser
|
|||||||
import pytest
|
import pytest
|
||||||
from partial_json_parser.core.options import Allow
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall, ToolCall
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
|
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import json
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,10 @@ from typing import Any
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from mistral_common.protocol.instruct.request import InstructRequest
|
|||||||
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
|
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
|
||||||
from partial_json_parser.core.options import Allow
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||||
from vllm.tokenizers.mistral import MistralTokenizer
|
from vllm.tokenizers.mistral import MistralTokenizer
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from openai_harmony import (
|
|||||||
load_harmony_encoding,
|
load_harmony_encoding,
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
|
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
|
||||||
from vllm.tokenizers import get_tokenizer
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
|
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
|
||||||
|
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
|
|||||||
@@ -7,9 +7,11 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ from collections.abc import Generator
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
ToolCall,
|
ToolCall,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
|
||||||
def test_chat_completion_request_with_no_tools():
|
def test_chat_completion_request_with_no_tools():
|
||||||
|
|||||||
@@ -8,10 +8,10 @@ import pytest
|
|||||||
import regex as re
|
import regex as re
|
||||||
from pydantic import TypeAdapter
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.tool_parsers.utils import get_json_schema_from_tools
|
from vllm.tool_parsers.utils import get_json_schema_from_tools
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ from vllm import SamplingParams
|
|||||||
from vllm.assets.image import ImageAsset
|
from vllm.assets.image import ImageAsset
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.inputs import PromptType
|
from vllm.inputs import PromptType
|
||||||
from vllm.outputs import RequestOutput
|
from vllm.outputs import RequestOutput
|
||||||
|
|||||||
@@ -25,16 +25,18 @@ from vllm.entrypoints.anthropic.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
StreamOptions,
|
StreamOptions,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -22,6 +22,12 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.constants import MCP_PREFIX
|
from vllm.entrypoints.constants import MCP_PREFIX
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
FunctionCall,
|
||||||
|
ResponseInputOutputItem,
|
||||||
|
ResponseRawMessageAndToken,
|
||||||
|
ResponsesRequest,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
get_encoding,
|
get_encoding,
|
||||||
get_streamable_parser_for_assistant,
|
get_streamable_parser_for_assistant,
|
||||||
@@ -30,12 +36,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
|||||||
from vllm.entrypoints.openai.parser.responses_parser import (
|
from vllm.entrypoints.openai.parser.responses_parser import (
|
||||||
get_responses_parser_for_simple_context,
|
get_responses_parser_for_simple_context,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
FunctionCall,
|
|
||||||
ResponseInputOutputItem,
|
|
||||||
ResponseRawMessageAndToken,
|
|
||||||
ResponsesRequest,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.responses_utils import construct_tool_dicts
|
from vllm.entrypoints.responses_utils import construct_tool_dicts
|
||||||
from vllm.entrypoints.tool import Tool
|
from vllm.entrypoints.tool import Tool
|
||||||
from vllm.entrypoints.tool_server import ToolServer
|
from vllm.entrypoints.tool_server import ToolServer
|
||||||
|
|||||||
@@ -42,11 +42,9 @@ from vllm.entrypoints.anthropic.protocol import (
|
|||||||
from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
|
from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
|
||||||
from vllm.entrypoints.launcher import serve_http
|
from vllm.entrypoints.launcher import serve_http
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||||
from vllm.entrypoints.openai.orca_metrics import metrics_header
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
ChatCompletionRequest,
|
|
||||||
ChatCompletionResponse,
|
|
||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
ErrorInfo,
|
ErrorInfo,
|
||||||
@@ -59,9 +57,9 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
TranslationRequest,
|
TranslationRequest,
|
||||||
TranslationResponseVariant,
|
TranslationResponseVariant,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
|
from vllm.entrypoints.openai.orca_metrics import metrics_header
|
||||||
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
|
||||||
from vllm.entrypoints.openai.serving_models import (
|
from vllm.entrypoints.openai.serving_models import (
|
||||||
BaseModelPath,
|
BaseModelPath,
|
||||||
OpenAIServingModels,
|
OpenAIServingModels,
|
||||||
@@ -475,47 +473,6 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques
|
|||||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
|
||||||
"/v1/chat/completions",
|
|
||||||
dependencies=[Depends(validate_json_request)],
|
|
||||||
responses={
|
|
||||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
|
||||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
@with_cancellation
|
|
||||||
@load_aware_call
|
|
||||||
async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
|
|
||||||
metrics_header_format = raw_request.headers.get(
|
|
||||||
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
|
|
||||||
)
|
|
||||||
handler = chat(raw_request)
|
|
||||||
if handler is None:
|
|
||||||
return base(raw_request).create_error_response(
|
|
||||||
message="The model does not support Chat Completions API"
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
generator = await handler.create_chat_completion(request, raw_request)
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
|
|
||||||
) from e
|
|
||||||
if isinstance(generator, ErrorResponse):
|
|
||||||
return JSONResponse(
|
|
||||||
content=generator.model_dump(), status_code=generator.error.code
|
|
||||||
)
|
|
||||||
|
|
||||||
elif isinstance(generator, ChatCompletionResponse):
|
|
||||||
return JSONResponse(
|
|
||||||
content=generator.model_dump(),
|
|
||||||
headers=metrics_header(metrics_header_format),
|
|
||||||
)
|
|
||||||
|
|
||||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/v1/completions",
|
"/v1/completions",
|
||||||
dependencies=[Depends(validate_json_request)],
|
dependencies=[Depends(validate_json_request)],
|
||||||
@@ -735,8 +692,10 @@ class XRequestIdMiddleware:
|
|||||||
def _extract_content_from_chunk(chunk_data: dict) -> str:
|
def _extract_content_from_chunk(chunk_data: dict) -> str:
|
||||||
"""Extract content from a streaming response chunk."""
|
"""Extract content from a streaming response chunk."""
|
||||||
try:
|
try:
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
CompletionStreamResponse,
|
CompletionStreamResponse,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -880,7 +839,11 @@ def build_app(args: Namespace) -> FastAPI:
|
|||||||
from vllm.entrypoints.serve import register_vllm_serve_api_routers
|
from vllm.entrypoints.serve import register_vllm_serve_api_routers
|
||||||
|
|
||||||
register_vllm_serve_api_routers(app)
|
register_vllm_serve_api_routers(app)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||||
|
attach_router as register_chat_api_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_chat_api_router(app)
|
||||||
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
|
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
|
||||||
|
|
||||||
register_sagemaker_routes(router)
|
register_sagemaker_routes(router)
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
|
||||||
|
from http import HTTPStatus
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||||
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
ChatCompletionResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
|
from vllm.entrypoints.openai.orca_metrics import metrics_header
|
||||||
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
|
from vllm.entrypoints.utils import (
|
||||||
|
load_aware_call,
|
||||||
|
with_cancellation,
|
||||||
|
)
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
|
||||||
|
|
||||||
|
|
||||||
|
def chat(request: Request) -> OpenAIServingChat | None:
|
||||||
|
return request.app.state.openai_serving_chat
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/chat/completions",
|
||||||
|
dependencies=[Depends(validate_json_request)],
|
||||||
|
responses={
|
||||||
|
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||||
|
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
@with_cancellation
|
||||||
|
@load_aware_call
|
||||||
|
async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
|
||||||
|
metrics_header_format = raw_request.headers.get(
|
||||||
|
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
|
||||||
|
)
|
||||||
|
handler = chat(raw_request)
|
||||||
|
if handler is None:
|
||||||
|
base_server = raw_request.app.state.openai_serving_tokenization
|
||||||
|
return base_server.create_error_response(
|
||||||
|
message="The model does not support Chat Completions API"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
generator = await handler.create_chat_completion(request, raw_request)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
|
||||||
|
) from e
|
||||||
|
if isinstance(generator, ErrorResponse):
|
||||||
|
return JSONResponse(
|
||||||
|
content=generator.model_dump(), status_code=generator.error.code
|
||||||
|
)
|
||||||
|
|
||||||
|
elif isinstance(generator, ChatCompletionResponse):
|
||||||
|
return JSONResponse(
|
||||||
|
content=generator.model_dump(),
|
||||||
|
headers=metrics_header(metrics_header_format),
|
||||||
|
)
|
||||||
|
|
||||||
|
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||||
|
|
||||||
|
|
||||||
|
def attach_router(app: FastAPI):
|
||||||
|
app.include_router(router)
|
||||||
@@ -0,0 +1,654 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
# Adapted from
|
||||||
|
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from typing import Annotated, Any, ClassVar, Literal
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from openai.types.chat.chat_completion_audio import (
|
||||||
|
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||||
|
)
|
||||||
|
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
||||||
|
from pydantic import (
|
||||||
|
Field,
|
||||||
|
model_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
AnyResponseFormat,
|
||||||
|
DeltaMessage,
|
||||||
|
FunctionCall,
|
||||||
|
FunctionDefinition,
|
||||||
|
LegacyStructuralTagResponseFormat,
|
||||||
|
LogitsProcessors,
|
||||||
|
OpenAIBaseModel,
|
||||||
|
StreamOptions,
|
||||||
|
StructuralTagResponseFormat,
|
||||||
|
ToolCall,
|
||||||
|
UsageInfo,
|
||||||
|
get_logits_processors,
|
||||||
|
)
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.logprobs import Logprob
|
||||||
|
from vllm.sampling_params import (
|
||||||
|
BeamSearchParams,
|
||||||
|
RequestOutputKind,
|
||||||
|
SamplingParams,
|
||||||
|
StructuredOutputsParams,
|
||||||
|
)
|
||||||
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_LONG_INFO = torch.iinfo(torch.long)
|
||||||
|
|
||||||
|
|
||||||
|
class ChatMessage(OpenAIBaseModel):
|
||||||
|
role: str
|
||||||
|
content: str | None = None
|
||||||
|
refusal: str | None = None
|
||||||
|
annotations: OpenAIAnnotation | None = None
|
||||||
|
audio: OpenAIChatCompletionAudio | None = None
|
||||||
|
function_call: FunctionCall | None = None
|
||||||
|
tool_calls: list[ToolCall] = Field(default_factory=list)
|
||||||
|
|
||||||
|
# vLLM-specific fields that are not in OpenAI spec
|
||||||
|
reasoning: str | None = None
|
||||||
|
reasoning_content: str | None = None
|
||||||
|
"""Deprecated: use `reasoning` instead."""
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def handle_deprecated_reasoning_content(self):
|
||||||
|
"""Copy reasoning to reasoning_content for backward compatibility."""
|
||||||
|
self.reasoning_content = self.reasoning
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionLogProb(OpenAIBaseModel):
|
||||||
|
token: str
|
||||||
|
logprob: float = -9999.0
|
||||||
|
bytes: list[int] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionLogProbsContent(ChatCompletionLogProb):
|
||||||
|
# Workaround: redefine fields name cache so that it's not
|
||||||
|
# shared with the super class.
|
||||||
|
field_names: ClassVar[set[str] | None] = None
|
||||||
|
top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||||
|
content: list[ChatCompletionLogProbsContent] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||||
|
index: int
|
||||||
|
message: ChatMessage
|
||||||
|
logprobs: ChatCompletionLogProbs | None = None
|
||||||
|
# per OpenAI spec this is the default
|
||||||
|
finish_reason: str | None = "stop"
|
||||||
|
# not part of the OpenAI spec but included in vLLM for legacy reasons
|
||||||
|
stop_reason: int | str | None = None
|
||||||
|
# not part of the OpenAI spec but is useful for tracing the tokens
|
||||||
|
# in agent scenarios
|
||||||
|
token_ids: list[int] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionResponse(OpenAIBaseModel):
|
||||||
|
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
||||||
|
object: Literal["chat.completion"] = "chat.completion"
|
||||||
|
created: int = Field(default_factory=lambda: int(time.time()))
|
||||||
|
model: str
|
||||||
|
choices: list[ChatCompletionResponseChoice]
|
||||||
|
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||||
|
system_fingerprint: str | None = None
|
||||||
|
usage: UsageInfo
|
||||||
|
|
||||||
|
# vLLM-specific fields that are not in OpenAI spec
|
||||||
|
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||||
|
prompt_token_ids: list[int] | None = None
|
||||||
|
kv_transfer_params: dict[str, Any] | None = Field(
|
||||||
|
default=None, description="KVTransfer parameters."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||||
|
index: int
|
||||||
|
delta: DeltaMessage
|
||||||
|
logprobs: ChatCompletionLogProbs | None = None
|
||||||
|
finish_reason: str | None = None
|
||||||
|
stop_reason: int | str | None = None
|
||||||
|
# not part of the OpenAI spec but for tracing the tokens
|
||||||
|
token_ids: list[int] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||||
|
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
||||||
|
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
|
||||||
|
created: int = Field(default_factory=lambda: int(time.time()))
|
||||||
|
model: str
|
||||||
|
choices: list[ChatCompletionResponseStreamChoice]
|
||||||
|
usage: UsageInfo | None = Field(default=None)
|
||||||
|
# not part of the OpenAI spec but for tracing the tokens
|
||||||
|
prompt_token_ids: list[int] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionToolsParam(OpenAIBaseModel):
|
||||||
|
type: Literal["function"] = "function"
|
||||||
|
function: FunctionDefinition
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionNamedFunction(OpenAIBaseModel):
|
||||||
|
name: str
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
|
||||||
|
function: ChatCompletionNamedFunction
|
||||||
|
type: Literal["function"] = "function"
|
||||||
|
|
||||||
|
|
||||||
|
class ChatCompletionRequest(OpenAIBaseModel):
|
||||||
|
# Ordered by official OpenAI API documentation
|
||||||
|
# https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
messages: list[ChatCompletionMessageParam]
|
||||||
|
model: str | None = None
|
||||||
|
frequency_penalty: float | None = 0.0
|
||||||
|
logit_bias: dict[str, float] | None = None
|
||||||
|
logprobs: bool | None = False
|
||||||
|
top_logprobs: int | None = 0
|
||||||
|
max_tokens: int | None = Field(
|
||||||
|
default=None,
|
||||||
|
deprecated="max_tokens is deprecated in favor of "
|
||||||
|
"the max_completion_tokens field",
|
||||||
|
)
|
||||||
|
max_completion_tokens: int | None = None
|
||||||
|
n: int | None = 1
|
||||||
|
presence_penalty: float | None = 0.0
|
||||||
|
response_format: AnyResponseFormat | None = None
|
||||||
|
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
|
||||||
|
stop: str | list[str] | None = []
|
||||||
|
stream: bool | None = False
|
||||||
|
stream_options: StreamOptions | None = None
|
||||||
|
temperature: float | None = None
|
||||||
|
top_p: float | None = None
|
||||||
|
tools: list[ChatCompletionToolsParam] | None = None
|
||||||
|
tool_choice: (
|
||||||
|
Literal["none"]
|
||||||
|
| Literal["auto"]
|
||||||
|
| Literal["required"]
|
||||||
|
| ChatCompletionNamedToolChoiceParam
|
||||||
|
| None
|
||||||
|
) = "none"
|
||||||
|
reasoning_effort: Literal["low", "medium", "high"] | None = None
|
||||||
|
include_reasoning: bool = True
|
||||||
|
parallel_tool_calls: bool | None = True
|
||||||
|
|
||||||
|
# NOTE this will be ignored by vLLM
|
||||||
|
user: str | None = None
|
||||||
|
|
||||||
|
# --8<-- [start:chat-completion-sampling-params]
|
||||||
|
use_beam_search: bool = False
|
||||||
|
top_k: int | None = None
|
||||||
|
min_p: float | None = None
|
||||||
|
repetition_penalty: float | None = None
|
||||||
|
length_penalty: float = 1.0
|
||||||
|
stop_token_ids: list[int] | None = []
|
||||||
|
include_stop_str_in_output: bool = False
|
||||||
|
ignore_eos: bool = False
|
||||||
|
min_tokens: int = 0
|
||||||
|
skip_special_tokens: bool = True
|
||||||
|
spaces_between_special_tokens: bool = True
|
||||||
|
truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
|
||||||
|
None
|
||||||
|
)
|
||||||
|
prompt_logprobs: int | None = None
|
||||||
|
allowed_token_ids: list[int] | None = None
|
||||||
|
bad_words: list[str] = Field(default_factory=list)
|
||||||
|
# --8<-- [end:chat-completion-sampling-params]
|
||||||
|
|
||||||
|
# --8<-- [start:chat-completion-extra-params]
|
||||||
|
echo: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If true, the new message will be prepended with the last message "
|
||||||
|
"if they belong to the same role."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
add_generation_prompt: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If true, the generation prompt will be added to the chat template. "
|
||||||
|
"This is a parameter used by chat template in tokenizer config of the "
|
||||||
|
"model."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
continue_final_message: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If this is set, the chat will be formatted so that the final "
|
||||||
|
"message in the chat is open-ended, without any EOS tokens. The "
|
||||||
|
"model will continue this message rather than starting a new one. "
|
||||||
|
'This allows you to "prefill" part of the model\'s response for it. '
|
||||||
|
"Cannot be used at the same time as `add_generation_prompt`."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
add_special_tokens: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
||||||
|
"on top of what is added by the chat template. "
|
||||||
|
"For most models, the chat template takes care of adding the "
|
||||||
|
"special tokens so this should be set to false (as is the "
|
||||||
|
"default)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
documents: list[dict[str, str]] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"A list of dicts representing documents that will be accessible to "
|
||||||
|
"the model if it is performing RAG (retrieval-augmented generation)."
|
||||||
|
" If the template does not support RAG, this argument will have no "
|
||||||
|
"effect. We recommend that each document should be a dict containing "
|
||||||
|
'"title" and "text" keys.'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chat_template: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"A Jinja template to use for this conversion. "
|
||||||
|
"As of transformers v4.44, default chat template is no longer "
|
||||||
|
"allowed, so you must provide a chat template if the tokenizer "
|
||||||
|
"does not define one."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"Additional keyword args to pass to the template renderer. "
|
||||||
|
"Will be accessible by the chat template."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=("Additional kwargs to pass to the HF processor."),
|
||||||
|
)
|
||||||
|
structured_outputs: StructuredOutputsParams | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="Additional kwargs for structured outputs",
|
||||||
|
)
|
||||||
|
priority: int = Field(
|
||||||
|
default=0,
|
||||||
|
description=(
|
||||||
|
"The priority of the request (lower means earlier handling; "
|
||||||
|
"default: 0). Any priority other than 0 will raise an error "
|
||||||
|
"if the served model does not use priority scheduling."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
request_id: str = Field(
|
||||||
|
default_factory=random_uuid,
|
||||||
|
description=(
|
||||||
|
"The request_id related to this request. If the caller does "
|
||||||
|
"not set it, a random_uuid will be generated. This id is used "
|
||||||
|
"through out the inference process and return in response."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
logits_processors: LogitsProcessors | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"A list of either qualified names of logits processors, or "
|
||||||
|
"constructor objects, to apply when sampling. A constructor is "
|
||||||
|
"a JSON object with a required 'qualname' field specifying the "
|
||||||
|
"qualified name of the processor class/factory, and optional "
|
||||||
|
"'args' and 'kwargs' fields containing positional and keyword "
|
||||||
|
"arguments. For example: {'qualname': "
|
||||||
|
"'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
|
||||||
|
"{'param': 'value'}}."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return_tokens_as_token_ids: bool | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"If specified with 'logprobs', tokens are represented "
|
||||||
|
" as strings of the form 'token_id:{token_id}' so that tokens "
|
||||||
|
"that are not JSON-encodable can be identified."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return_token_ids: bool | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"If specified, the result will include token IDs alongside the "
|
||||||
|
"generated text. In streaming mode, prompt_token_ids is included "
|
||||||
|
"only in the first chunk, and token_ids contains the delta tokens "
|
||||||
|
"for each chunk. This is useful for debugging or when you "
|
||||||
|
"need to map generated text back to input tokens."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
cache_salt: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"If specified, the prefix cache will be salted with the provided "
|
||||||
|
"string to prevent an attacker to guess prompts in multi-user "
|
||||||
|
"environments. The salt should be random, protected from "
|
||||||
|
"access by 3rd parties, and long enough to be "
|
||||||
|
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||||
|
"to 256 bit)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
kv_transfer_params: dict[str, Any] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="KVTransfer parameters used for disaggregated serving.",
|
||||||
|
)
|
||||||
|
|
||||||
|
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"Additional request parameters with (list of) string or "
|
||||||
|
"numeric values, used by custom extensions."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# --8<-- [end:chat-completion-extra-params]
|
||||||
|
|
||||||
|
# Default sampling parameters for chat completion requests
|
||||||
|
_DEFAULT_SAMPLING_PARAMS: dict = {
|
||||||
|
"repetition_penalty": 1.0,
|
||||||
|
"temperature": 1.0,
|
||||||
|
"top_p": 1.0,
|
||||||
|
"top_k": 0,
|
||||||
|
"min_p": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
def to_beam_search_params(
|
||||||
|
self, max_tokens: int, default_sampling_params: dict
|
||||||
|
) -> BeamSearchParams:
|
||||||
|
n = self.n if self.n is not None else 1
|
||||||
|
if (temperature := self.temperature) is None:
|
||||||
|
temperature = default_sampling_params.get(
|
||||||
|
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
||||||
|
)
|
||||||
|
|
||||||
|
return BeamSearchParams(
|
||||||
|
beam_width=n,
|
||||||
|
max_tokens=max_tokens,
|
||||||
|
ignore_eos=self.ignore_eos,
|
||||||
|
temperature=temperature,
|
||||||
|
length_penalty=self.length_penalty,
|
||||||
|
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_sampling_params(
|
||||||
|
self,
|
||||||
|
max_tokens: int,
|
||||||
|
logits_processor_pattern: str | None,
|
||||||
|
default_sampling_params: dict,
|
||||||
|
) -> SamplingParams:
|
||||||
|
# Default parameters
|
||||||
|
if (repetition_penalty := self.repetition_penalty) is None:
|
||||||
|
repetition_penalty = default_sampling_params.get(
|
||||||
|
"repetition_penalty",
|
||||||
|
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
|
||||||
|
)
|
||||||
|
if (temperature := self.temperature) is None:
|
||||||
|
temperature = default_sampling_params.get(
|
||||||
|
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
||||||
|
)
|
||||||
|
if (top_p := self.top_p) is None:
|
||||||
|
top_p = default_sampling_params.get(
|
||||||
|
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
|
||||||
|
)
|
||||||
|
if (top_k := self.top_k) is None:
|
||||||
|
top_k = default_sampling_params.get(
|
||||||
|
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
|
||||||
|
)
|
||||||
|
if (min_p := self.min_p) is None:
|
||||||
|
min_p = default_sampling_params.get(
|
||||||
|
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt_logprobs = self.prompt_logprobs
|
||||||
|
if prompt_logprobs is None and self.echo:
|
||||||
|
prompt_logprobs = self.top_logprobs
|
||||||
|
|
||||||
|
response_format = self.response_format
|
||||||
|
if response_format is not None:
|
||||||
|
# If structured outputs wasn't already enabled,
|
||||||
|
# we must enable it for these features to work
|
||||||
|
if self.structured_outputs is None:
|
||||||
|
self.structured_outputs = StructuredOutputsParams()
|
||||||
|
|
||||||
|
# Set structured output params for response format
|
||||||
|
if response_format.type == "json_object":
|
||||||
|
self.structured_outputs.json_object = True
|
||||||
|
elif response_format.type == "json_schema":
|
||||||
|
json_schema = response_format.json_schema
|
||||||
|
assert json_schema is not None
|
||||||
|
self.structured_outputs.json = json_schema.json_schema
|
||||||
|
elif response_format.type == "structural_tag":
|
||||||
|
structural_tag = response_format
|
||||||
|
assert structural_tag is not None and isinstance(
|
||||||
|
structural_tag,
|
||||||
|
(
|
||||||
|
LegacyStructuralTagResponseFormat,
|
||||||
|
StructuralTagResponseFormat,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
s_tag_obj = structural_tag.model_dump(by_alias=True)
|
||||||
|
self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
|
||||||
|
|
||||||
|
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
|
||||||
|
if self.kv_transfer_params:
|
||||||
|
# Pass in kv_transfer_params via extra_args
|
||||||
|
extra_args["kv_transfer_params"] = self.kv_transfer_params
|
||||||
|
return SamplingParams.from_optional(
|
||||||
|
n=self.n,
|
||||||
|
presence_penalty=self.presence_penalty,
|
||||||
|
frequency_penalty=self.frequency_penalty,
|
||||||
|
repetition_penalty=repetition_penalty,
|
||||||
|
temperature=temperature,
|
||||||
|
top_p=top_p,
|
||||||
|
top_k=top_k,
|
||||||
|
min_p=min_p,
|
||||||
|
seed=self.seed,
|
||||||
|
stop=self.stop,
|
||||||
|
stop_token_ids=self.stop_token_ids,
|
||||||
|
logprobs=self.top_logprobs if self.logprobs else None,
|
||||||
|
prompt_logprobs=prompt_logprobs,
|
||||||
|
ignore_eos=self.ignore_eos,
|
||||||
|
max_tokens=max_tokens,
|
||||||
|
min_tokens=self.min_tokens,
|
||||||
|
skip_special_tokens=self.skip_special_tokens,
|
||||||
|
spaces_between_special_tokens=self.spaces_between_special_tokens,
|
||||||
|
logits_processors=get_logits_processors(
|
||||||
|
self.logits_processors, logits_processor_pattern
|
||||||
|
),
|
||||||
|
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||||
|
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||||
|
output_kind=RequestOutputKind.DELTA
|
||||||
|
if self.stream
|
||||||
|
else RequestOutputKind.FINAL_ONLY,
|
||||||
|
structured_outputs=self.structured_outputs,
|
||||||
|
logit_bias=self.logit_bias,
|
||||||
|
bad_words=self.bad_words,
|
||||||
|
allowed_token_ids=self.allowed_token_ids,
|
||||||
|
extra_args=extra_args or None,
|
||||||
|
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||||
|
)
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def validate_stream_options(cls, data):
|
||||||
|
if data.get("stream_options") and not data.get("stream"):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Stream options can only be defined when `stream=True`.",
|
||||||
|
parameter="stream_options",
|
||||||
|
)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_logprobs(cls, data):
|
||||||
|
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
|
||||||
|
if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"`prompt_logprobs` are not available when `stream=True`.",
|
||||||
|
parameter="prompt_logprobs",
|
||||||
|
)
|
||||||
|
|
||||||
|
if prompt_logprobs < 0 and prompt_logprobs != -1:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"`prompt_logprobs` must be a positive value or -1.",
|
||||||
|
parameter="prompt_logprobs",
|
||||||
|
value=prompt_logprobs,
|
||||||
|
)
|
||||||
|
if (top_logprobs := data.get("top_logprobs")) is not None:
|
||||||
|
if top_logprobs < 0 and top_logprobs != -1:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"`top_logprobs` must be a positive value or -1.",
|
||||||
|
parameter="top_logprobs",
|
||||||
|
value=top_logprobs,
|
||||||
|
)
|
||||||
|
|
||||||
|
if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"when using `top_logprobs`, `logprobs` must be set to true.",
|
||||||
|
parameter="top_logprobs",
|
||||||
|
)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_structured_outputs_count(cls, data):
|
||||||
|
if isinstance(data, ValueError):
|
||||||
|
raise data
|
||||||
|
|
||||||
|
if data.get("structured_outputs", None) is None:
|
||||||
|
return data
|
||||||
|
|
||||||
|
structured_outputs_kwargs = data["structured_outputs"]
|
||||||
|
count = sum(
|
||||||
|
structured_outputs_kwargs.get(k) is not None
|
||||||
|
for k in ("json", "regex", "choice")
|
||||||
|
)
|
||||||
|
# you can only use one kind of constraints for structured outputs
|
||||||
|
if count > 1:
|
||||||
|
raise ValueError(
|
||||||
|
"You can only use one kind of constraints for structured "
|
||||||
|
"outputs ('json', 'regex' or 'choice')."
|
||||||
|
)
|
||||||
|
# you can only either use structured outputs or tools, not both
|
||||||
|
if count > 1 and data.get("tool_choice", "none") not in (
|
||||||
|
"none",
|
||||||
|
"auto",
|
||||||
|
"required",
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"You can only either use constraints for structured outputs "
|
||||||
|
"or tools, not both."
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_tool_usage(cls, data):
|
||||||
|
# if "tool_choice" is not specified but tools are provided,
|
||||||
|
# default to "auto" tool_choice
|
||||||
|
if "tool_choice" not in data and data.get("tools"):
|
||||||
|
data["tool_choice"] = "auto"
|
||||||
|
|
||||||
|
# if "tool_choice" is "none" -- no validation is needed for tools
|
||||||
|
if "tool_choice" in data and data["tool_choice"] == "none":
|
||||||
|
return data
|
||||||
|
|
||||||
|
# if "tool_choice" is specified -- validation
|
||||||
|
if "tool_choice" in data and data["tool_choice"] is not None:
|
||||||
|
# ensure that if "tool choice" is specified, tools are present
|
||||||
|
if "tools" not in data or data["tools"] is None:
|
||||||
|
raise ValueError("When using `tool_choice`, `tools` must be set.")
|
||||||
|
|
||||||
|
# make sure that tool choice is either a named tool
|
||||||
|
# OR that it's set to "auto" or "required"
|
||||||
|
if data["tool_choice"] not in ["auto", "required"] and not isinstance(
|
||||||
|
data["tool_choice"], dict
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid value for `tool_choice`: {data['tool_choice']}! "
|
||||||
|
'Only named tools, "none", "auto" or "required" '
|
||||||
|
"are supported."
|
||||||
|
)
|
||||||
|
|
||||||
|
# if tool_choice is "required" but the "tools" list is empty,
|
||||||
|
# override the data to behave like "none" to align with
|
||||||
|
# OpenAI’s behavior.
|
||||||
|
if (
|
||||||
|
data["tool_choice"] == "required"
|
||||||
|
and isinstance(data["tools"], list)
|
||||||
|
and len(data["tools"]) == 0
|
||||||
|
):
|
||||||
|
data["tool_choice"] = "none"
|
||||||
|
del data["tools"]
|
||||||
|
return data
|
||||||
|
|
||||||
|
# ensure that if "tool_choice" is specified as an object,
|
||||||
|
# it matches a valid tool
|
||||||
|
correct_usage_message = (
|
||||||
|
'Correct usage: `{"type": "function",'
|
||||||
|
' "function": {"name": "my_function"}}`'
|
||||||
|
)
|
||||||
|
if isinstance(data["tool_choice"], dict):
|
||||||
|
valid_tool = False
|
||||||
|
function = data["tool_choice"].get("function")
|
||||||
|
if not isinstance(function, dict):
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid value for `function`: `{function}` in "
|
||||||
|
f"`tool_choice`! {correct_usage_message}"
|
||||||
|
)
|
||||||
|
if "name" not in function:
|
||||||
|
raise ValueError(
|
||||||
|
f"Expected field `name` in `function` in "
|
||||||
|
f"`tool_choice`! {correct_usage_message}"
|
||||||
|
)
|
||||||
|
function_name = function["name"]
|
||||||
|
if not isinstance(function_name, str) or len(function_name) == 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid `name` in `function`: `{function_name}`"
|
||||||
|
f" in `tool_choice`! {correct_usage_message}"
|
||||||
|
)
|
||||||
|
for tool in data["tools"]:
|
||||||
|
if tool["function"]["name"] == function_name:
|
||||||
|
valid_tool = True
|
||||||
|
break
|
||||||
|
if not valid_tool:
|
||||||
|
raise ValueError(
|
||||||
|
"The tool specified in `tool_choice` does not match any"
|
||||||
|
" of the specified `tools`"
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_generation_prompt(cls, data):
|
||||||
|
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
||||||
|
raise ValueError(
|
||||||
|
"Cannot set both `continue_final_message` and "
|
||||||
|
"`add_generation_prompt` to True."
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_cache_salt_support(cls, data):
|
||||||
|
if data.get("cache_salt") is not None and (
|
||||||
|
not isinstance(data["cache_salt"], str) or not data["cache_salt"]
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Parameter 'cache_salt' must be a non-empty string if provided."
|
||||||
|
)
|
||||||
|
return data
|
||||||
+16
-14
@@ -23,16 +23,7 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
make_tool_call_id,
|
make_tool_call_id,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
get_developer_message,
|
|
||||||
get_stop_tokens_for_assistant_actions,
|
|
||||||
get_streamable_parser_for_assistant,
|
|
||||||
get_system_message,
|
|
||||||
parse_chat_inputs_to_harmony_messages,
|
|
||||||
parse_chat_output,
|
|
||||||
render_for_completion,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
ChatCompletionLogProb,
|
ChatCompletionLogProb,
|
||||||
ChatCompletionLogProbs,
|
ChatCompletionLogProbs,
|
||||||
ChatCompletionLogProbsContent,
|
ChatCompletionLogProbsContent,
|
||||||
@@ -43,6 +34,11 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
ChatCompletionResponseStreamChoice,
|
ChatCompletionResponseStreamChoice,
|
||||||
ChatCompletionStreamResponse,
|
ChatCompletionStreamResponse,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
|
||||||
|
extract_harmony_streaming_delta,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
@@ -52,14 +48,20 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
ToolCall,
|
ToolCall,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat_stream_harmony import (
|
from vllm.entrypoints.openai.engine.serving import (
|
||||||
extract_harmony_streaming_delta,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.serving_engine import (
|
|
||||||
GenerationError,
|
GenerationError,
|
||||||
OpenAIServing,
|
OpenAIServing,
|
||||||
clamp_prompt_logprobs,
|
clamp_prompt_logprobs,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
|
get_developer_message,
|
||||||
|
get_stop_tokens_for_assistant_actions,
|
||||||
|
get_streamable_parser_for_assistant,
|
||||||
|
get_system_message,
|
||||||
|
parse_chat_inputs_to_harmony_messages,
|
||||||
|
parse_chat_output,
|
||||||
|
render_for_completion,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
||||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||||
+1
-1
@@ -10,7 +10,7 @@ harmony parser state during streaming chat completions.
|
|||||||
from openai_harmony import StreamableParser
|
from openai_harmony import StreamableParser
|
||||||
|
|
||||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
@@ -11,10 +11,6 @@ from typing import Annotated, Any, ClassVar, Literal, TypeAlias
|
|||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
from fastapi import HTTPException, UploadFile
|
from fastapi import HTTPException, UploadFile
|
||||||
from openai.types.chat.chat_completion_audio import (
|
|
||||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
|
||||||
)
|
|
||||||
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
ResponseCodeInterpreterCallCodeDeltaEvent,
|
ResponseCodeInterpreterCallCodeDeltaEvent,
|
||||||
ResponseCodeInterpreterCallCodeDoneEvent,
|
ResponseCodeInterpreterCallCodeDoneEvent,
|
||||||
@@ -234,20 +230,6 @@ class FunctionDefinition(OpenAIBaseModel):
|
|||||||
parameters: dict[str, Any] | None = None
|
parameters: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionToolsParam(OpenAIBaseModel):
|
|
||||||
type: Literal["function"] = "function"
|
|
||||||
function: FunctionDefinition
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionNamedFunction(OpenAIBaseModel):
|
|
||||||
name: str
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
|
|
||||||
function: ChatCompletionNamedFunction
|
|
||||||
type: Literal["function"] = "function"
|
|
||||||
|
|
||||||
|
|
||||||
# extra="forbid" is a workaround to have kwargs as a field,
|
# extra="forbid" is a workaround to have kwargs as a field,
|
||||||
# see https://github.com/pydantic/pydantic/issues/3125
|
# see https://github.com/pydantic/pydantic/issues/3125
|
||||||
class LogitsProcessorConstructor(BaseModel):
|
class LogitsProcessorConstructor(BaseModel):
|
||||||
@@ -526,507 +508,6 @@ class ResponsesRequest(OpenAIBaseModel):
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionRequest(OpenAIBaseModel):
|
|
||||||
# Ordered by official OpenAI API documentation
|
|
||||||
# https://platform.openai.com/docs/api-reference/chat/create
|
|
||||||
messages: list[ChatCompletionMessageParam]
|
|
||||||
model: str | None = None
|
|
||||||
frequency_penalty: float | None = 0.0
|
|
||||||
logit_bias: dict[str, float] | None = None
|
|
||||||
logprobs: bool | None = False
|
|
||||||
top_logprobs: int | None = 0
|
|
||||||
max_tokens: int | None = Field(
|
|
||||||
default=None,
|
|
||||||
deprecated="max_tokens is deprecated in favor of "
|
|
||||||
"the max_completion_tokens field",
|
|
||||||
)
|
|
||||||
max_completion_tokens: int | None = None
|
|
||||||
n: int | None = 1
|
|
||||||
presence_penalty: float | None = 0.0
|
|
||||||
response_format: AnyResponseFormat | None = None
|
|
||||||
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
|
|
||||||
stop: str | list[str] | None = []
|
|
||||||
stream: bool | None = False
|
|
||||||
stream_options: StreamOptions | None = None
|
|
||||||
temperature: float | None = None
|
|
||||||
top_p: float | None = None
|
|
||||||
tools: list[ChatCompletionToolsParam] | None = None
|
|
||||||
tool_choice: (
|
|
||||||
Literal["none"]
|
|
||||||
| Literal["auto"]
|
|
||||||
| Literal["required"]
|
|
||||||
| ChatCompletionNamedToolChoiceParam
|
|
||||||
| None
|
|
||||||
) = "none"
|
|
||||||
reasoning_effort: Literal["low", "medium", "high"] | None = None
|
|
||||||
include_reasoning: bool = True
|
|
||||||
parallel_tool_calls: bool | None = True
|
|
||||||
|
|
||||||
# NOTE this will be ignored by vLLM
|
|
||||||
user: str | None = None
|
|
||||||
|
|
||||||
# --8<-- [start:chat-completion-sampling-params]
|
|
||||||
use_beam_search: bool = False
|
|
||||||
top_k: int | None = None
|
|
||||||
min_p: float | None = None
|
|
||||||
repetition_penalty: float | None = None
|
|
||||||
length_penalty: float = 1.0
|
|
||||||
stop_token_ids: list[int] | None = []
|
|
||||||
include_stop_str_in_output: bool = False
|
|
||||||
ignore_eos: bool = False
|
|
||||||
min_tokens: int = 0
|
|
||||||
skip_special_tokens: bool = True
|
|
||||||
spaces_between_special_tokens: bool = True
|
|
||||||
truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
|
|
||||||
None
|
|
||||||
)
|
|
||||||
prompt_logprobs: int | None = None
|
|
||||||
allowed_token_ids: list[int] | None = None
|
|
||||||
bad_words: list[str] = Field(default_factory=list)
|
|
||||||
# --8<-- [end:chat-completion-sampling-params]
|
|
||||||
|
|
||||||
# --8<-- [start:chat-completion-extra-params]
|
|
||||||
echo: bool = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If true, the new message will be prepended with the last message "
|
|
||||||
"if they belong to the same role."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
add_generation_prompt: bool = Field(
|
|
||||||
default=True,
|
|
||||||
description=(
|
|
||||||
"If true, the generation prompt will be added to the chat template. "
|
|
||||||
"This is a parameter used by chat template in tokenizer config of the "
|
|
||||||
"model."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
continue_final_message: bool = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If this is set, the chat will be formatted so that the final "
|
|
||||||
"message in the chat is open-ended, without any EOS tokens. The "
|
|
||||||
"model will continue this message rather than starting a new one. "
|
|
||||||
'This allows you to "prefill" part of the model\'s response for it. '
|
|
||||||
"Cannot be used at the same time as `add_generation_prompt`."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
add_special_tokens: bool = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
|
||||||
"on top of what is added by the chat template. "
|
|
||||||
"For most models, the chat template takes care of adding the "
|
|
||||||
"special tokens so this should be set to false (as is the "
|
|
||||||
"default)."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
documents: list[dict[str, str]] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"A list of dicts representing documents that will be accessible to "
|
|
||||||
"the model if it is performing RAG (retrieval-augmented generation)."
|
|
||||||
" If the template does not support RAG, this argument will have no "
|
|
||||||
"effect. We recommend that each document should be a dict containing "
|
|
||||||
'"title" and "text" keys.'
|
|
||||||
),
|
|
||||||
)
|
|
||||||
chat_template: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"A Jinja template to use for this conversion. "
|
|
||||||
"As of transformers v4.44, default chat template is no longer "
|
|
||||||
"allowed, so you must provide a chat template if the tokenizer "
|
|
||||||
"does not define one."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"Additional keyword args to pass to the template renderer. "
|
|
||||||
"Will be accessible by the chat template."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=("Additional kwargs to pass to the HF processor."),
|
|
||||||
)
|
|
||||||
structured_outputs: StructuredOutputsParams | None = Field(
|
|
||||||
default=None,
|
|
||||||
description="Additional kwargs for structured outputs",
|
|
||||||
)
|
|
||||||
priority: int = Field(
|
|
||||||
default=0,
|
|
||||||
description=(
|
|
||||||
"The priority of the request (lower means earlier handling; "
|
|
||||||
"default: 0). Any priority other than 0 will raise an error "
|
|
||||||
"if the served model does not use priority scheduling."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
request_id: str = Field(
|
|
||||||
default_factory=random_uuid,
|
|
||||||
description=(
|
|
||||||
"The request_id related to this request. If the caller does "
|
|
||||||
"not set it, a random_uuid will be generated. This id is used "
|
|
||||||
"through out the inference process and return in response."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
logits_processors: LogitsProcessors | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"A list of either qualified names of logits processors, or "
|
|
||||||
"constructor objects, to apply when sampling. A constructor is "
|
|
||||||
"a JSON object with a required 'qualname' field specifying the "
|
|
||||||
"qualified name of the processor class/factory, and optional "
|
|
||||||
"'args' and 'kwargs' fields containing positional and keyword "
|
|
||||||
"arguments. For example: {'qualname': "
|
|
||||||
"'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
|
|
||||||
"{'param': 'value'}}."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return_tokens_as_token_ids: bool | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"If specified with 'logprobs', tokens are represented "
|
|
||||||
" as strings of the form 'token_id:{token_id}' so that tokens "
|
|
||||||
"that are not JSON-encodable can be identified."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return_token_ids: bool | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"If specified, the result will include token IDs alongside the "
|
|
||||||
"generated text. In streaming mode, prompt_token_ids is included "
|
|
||||||
"only in the first chunk, and token_ids contains the delta tokens "
|
|
||||||
"for each chunk. This is useful for debugging or when you "
|
|
||||||
"need to map generated text back to input tokens."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
cache_salt: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"If specified, the prefix cache will be salted with the provided "
|
|
||||||
"string to prevent an attacker to guess prompts in multi-user "
|
|
||||||
"environments. The salt should be random, protected from "
|
|
||||||
"access by 3rd parties, and long enough to be "
|
|
||||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
|
||||||
"to 256 bit)."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
kv_transfer_params: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description="KVTransfer parameters used for disaggregated serving.",
|
|
||||||
)
|
|
||||||
|
|
||||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"Additional request parameters with (list of) string or "
|
|
||||||
"numeric values, used by custom extensions."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# --8<-- [end:chat-completion-extra-params]
|
|
||||||
|
|
||||||
# Default sampling parameters for chat completion requests
|
|
||||||
_DEFAULT_SAMPLING_PARAMS: dict = {
|
|
||||||
"repetition_penalty": 1.0,
|
|
||||||
"temperature": 1.0,
|
|
||||||
"top_p": 1.0,
|
|
||||||
"top_k": 0,
|
|
||||||
"min_p": 0.0,
|
|
||||||
}
|
|
||||||
|
|
||||||
def to_beam_search_params(
|
|
||||||
self, max_tokens: int, default_sampling_params: dict
|
|
||||||
) -> BeamSearchParams:
|
|
||||||
n = self.n if self.n is not None else 1
|
|
||||||
if (temperature := self.temperature) is None:
|
|
||||||
temperature = default_sampling_params.get(
|
|
||||||
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
|
||||||
)
|
|
||||||
|
|
||||||
return BeamSearchParams(
|
|
||||||
beam_width=n,
|
|
||||||
max_tokens=max_tokens,
|
|
||||||
ignore_eos=self.ignore_eos,
|
|
||||||
temperature=temperature,
|
|
||||||
length_penalty=self.length_penalty,
|
|
||||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
|
||||||
)
|
|
||||||
|
|
||||||
def to_sampling_params(
|
|
||||||
self,
|
|
||||||
max_tokens: int,
|
|
||||||
logits_processor_pattern: str | None,
|
|
||||||
default_sampling_params: dict,
|
|
||||||
) -> SamplingParams:
|
|
||||||
# Default parameters
|
|
||||||
if (repetition_penalty := self.repetition_penalty) is None:
|
|
||||||
repetition_penalty = default_sampling_params.get(
|
|
||||||
"repetition_penalty",
|
|
||||||
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
|
|
||||||
)
|
|
||||||
if (temperature := self.temperature) is None:
|
|
||||||
temperature = default_sampling_params.get(
|
|
||||||
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
|
||||||
)
|
|
||||||
if (top_p := self.top_p) is None:
|
|
||||||
top_p = default_sampling_params.get(
|
|
||||||
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
|
|
||||||
)
|
|
||||||
if (top_k := self.top_k) is None:
|
|
||||||
top_k = default_sampling_params.get(
|
|
||||||
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
|
|
||||||
)
|
|
||||||
if (min_p := self.min_p) is None:
|
|
||||||
min_p = default_sampling_params.get(
|
|
||||||
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt_logprobs = self.prompt_logprobs
|
|
||||||
if prompt_logprobs is None and self.echo:
|
|
||||||
prompt_logprobs = self.top_logprobs
|
|
||||||
|
|
||||||
response_format = self.response_format
|
|
||||||
if response_format is not None:
|
|
||||||
# If structured outputs wasn't already enabled,
|
|
||||||
# we must enable it for these features to work
|
|
||||||
if self.structured_outputs is None:
|
|
||||||
self.structured_outputs = StructuredOutputsParams()
|
|
||||||
|
|
||||||
# Set structured output params for response format
|
|
||||||
if response_format.type == "json_object":
|
|
||||||
self.structured_outputs.json_object = True
|
|
||||||
elif response_format.type == "json_schema":
|
|
||||||
json_schema = response_format.json_schema
|
|
||||||
assert json_schema is not None
|
|
||||||
self.structured_outputs.json = json_schema.json_schema
|
|
||||||
elif response_format.type == "structural_tag":
|
|
||||||
structural_tag = response_format
|
|
||||||
assert structural_tag is not None and isinstance(
|
|
||||||
structural_tag,
|
|
||||||
(
|
|
||||||
LegacyStructuralTagResponseFormat,
|
|
||||||
StructuralTagResponseFormat,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
s_tag_obj = structural_tag.model_dump(by_alias=True)
|
|
||||||
self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
|
|
||||||
|
|
||||||
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
|
|
||||||
if self.kv_transfer_params:
|
|
||||||
# Pass in kv_transfer_params via extra_args
|
|
||||||
extra_args["kv_transfer_params"] = self.kv_transfer_params
|
|
||||||
return SamplingParams.from_optional(
|
|
||||||
n=self.n,
|
|
||||||
presence_penalty=self.presence_penalty,
|
|
||||||
frequency_penalty=self.frequency_penalty,
|
|
||||||
repetition_penalty=repetition_penalty,
|
|
||||||
temperature=temperature,
|
|
||||||
top_p=top_p,
|
|
||||||
top_k=top_k,
|
|
||||||
min_p=min_p,
|
|
||||||
seed=self.seed,
|
|
||||||
stop=self.stop,
|
|
||||||
stop_token_ids=self.stop_token_ids,
|
|
||||||
logprobs=self.top_logprobs if self.logprobs else None,
|
|
||||||
prompt_logprobs=prompt_logprobs,
|
|
||||||
ignore_eos=self.ignore_eos,
|
|
||||||
max_tokens=max_tokens,
|
|
||||||
min_tokens=self.min_tokens,
|
|
||||||
skip_special_tokens=self.skip_special_tokens,
|
|
||||||
spaces_between_special_tokens=self.spaces_between_special_tokens,
|
|
||||||
logits_processors=get_logits_processors(
|
|
||||||
self.logits_processors, logits_processor_pattern
|
|
||||||
),
|
|
||||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
|
||||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
|
||||||
output_kind=RequestOutputKind.DELTA
|
|
||||||
if self.stream
|
|
||||||
else RequestOutputKind.FINAL_ONLY,
|
|
||||||
structured_outputs=self.structured_outputs,
|
|
||||||
logit_bias=self.logit_bias,
|
|
||||||
bad_words=self.bad_words,
|
|
||||||
allowed_token_ids=self.allowed_token_ids,
|
|
||||||
extra_args=extra_args or None,
|
|
||||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
|
||||||
)
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def validate_stream_options(cls, data):
|
|
||||||
if data.get("stream_options") and not data.get("stream"):
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"Stream options can only be defined when `stream=True`.",
|
|
||||||
parameter="stream_options",
|
|
||||||
)
|
|
||||||
|
|
||||||
return data
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_logprobs(cls, data):
|
|
||||||
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
|
|
||||||
if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"`prompt_logprobs` are not available when `stream=True`.",
|
|
||||||
parameter="prompt_logprobs",
|
|
||||||
)
|
|
||||||
|
|
||||||
if prompt_logprobs < 0 and prompt_logprobs != -1:
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"`prompt_logprobs` must be a positive value or -1.",
|
|
||||||
parameter="prompt_logprobs",
|
|
||||||
value=prompt_logprobs,
|
|
||||||
)
|
|
||||||
if (top_logprobs := data.get("top_logprobs")) is not None:
|
|
||||||
if top_logprobs < 0 and top_logprobs != -1:
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"`top_logprobs` must be a positive value or -1.",
|
|
||||||
parameter="top_logprobs",
|
|
||||||
value=top_logprobs,
|
|
||||||
)
|
|
||||||
|
|
||||||
if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"when using `top_logprobs`, `logprobs` must be set to true.",
|
|
||||||
parameter="top_logprobs",
|
|
||||||
)
|
|
||||||
|
|
||||||
return data
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_structured_outputs_count(cls, data):
|
|
||||||
if isinstance(data, ValueError):
|
|
||||||
raise data
|
|
||||||
|
|
||||||
if data.get("structured_outputs", None) is None:
|
|
||||||
return data
|
|
||||||
|
|
||||||
structured_outputs_kwargs = data["structured_outputs"]
|
|
||||||
count = sum(
|
|
||||||
structured_outputs_kwargs.get(k) is not None
|
|
||||||
for k in ("json", "regex", "choice")
|
|
||||||
)
|
|
||||||
# you can only use one kind of constraints for structured outputs
|
|
||||||
if count > 1:
|
|
||||||
raise ValueError(
|
|
||||||
"You can only use one kind of constraints for structured "
|
|
||||||
"outputs ('json', 'regex' or 'choice')."
|
|
||||||
)
|
|
||||||
# you can only either use structured outputs or tools, not both
|
|
||||||
if count > 1 and data.get("tool_choice", "none") not in (
|
|
||||||
"none",
|
|
||||||
"auto",
|
|
||||||
"required",
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
"You can only either use constraints for structured outputs "
|
|
||||||
"or tools, not both."
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_tool_usage(cls, data):
|
|
||||||
# if "tool_choice" is not specified but tools are provided,
|
|
||||||
# default to "auto" tool_choice
|
|
||||||
if "tool_choice" not in data and data.get("tools"):
|
|
||||||
data["tool_choice"] = "auto"
|
|
||||||
|
|
||||||
# if "tool_choice" is "none" -- no validation is needed for tools
|
|
||||||
if "tool_choice" in data and data["tool_choice"] == "none":
|
|
||||||
return data
|
|
||||||
|
|
||||||
# if "tool_choice" is specified -- validation
|
|
||||||
if "tool_choice" in data and data["tool_choice"] is not None:
|
|
||||||
# ensure that if "tool choice" is specified, tools are present
|
|
||||||
if "tools" not in data or data["tools"] is None:
|
|
||||||
raise ValueError("When using `tool_choice`, `tools` must be set.")
|
|
||||||
|
|
||||||
# make sure that tool choice is either a named tool
|
|
||||||
# OR that it's set to "auto" or "required"
|
|
||||||
if data["tool_choice"] not in ["auto", "required"] and not isinstance(
|
|
||||||
data["tool_choice"], dict
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid value for `tool_choice`: {data['tool_choice']}! "
|
|
||||||
'Only named tools, "none", "auto" or "required" '
|
|
||||||
"are supported."
|
|
||||||
)
|
|
||||||
|
|
||||||
# if tool_choice is "required" but the "tools" list is empty,
|
|
||||||
# override the data to behave like "none" to align with
|
|
||||||
# OpenAI’s behavior.
|
|
||||||
if (
|
|
||||||
data["tool_choice"] == "required"
|
|
||||||
and isinstance(data["tools"], list)
|
|
||||||
and len(data["tools"]) == 0
|
|
||||||
):
|
|
||||||
data["tool_choice"] = "none"
|
|
||||||
del data["tools"]
|
|
||||||
return data
|
|
||||||
|
|
||||||
# ensure that if "tool_choice" is specified as an object,
|
|
||||||
# it matches a valid tool
|
|
||||||
correct_usage_message = (
|
|
||||||
'Correct usage: `{"type": "function",'
|
|
||||||
' "function": {"name": "my_function"}}`'
|
|
||||||
)
|
|
||||||
if isinstance(data["tool_choice"], dict):
|
|
||||||
valid_tool = False
|
|
||||||
function = data["tool_choice"].get("function")
|
|
||||||
if not isinstance(function, dict):
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid value for `function`: `{function}` in "
|
|
||||||
f"`tool_choice`! {correct_usage_message}"
|
|
||||||
)
|
|
||||||
if "name" not in function:
|
|
||||||
raise ValueError(
|
|
||||||
f"Expected field `name` in `function` in "
|
|
||||||
f"`tool_choice`! {correct_usage_message}"
|
|
||||||
)
|
|
||||||
function_name = function["name"]
|
|
||||||
if not isinstance(function_name, str) or len(function_name) == 0:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid `name` in `function`: `{function_name}`"
|
|
||||||
f" in `tool_choice`! {correct_usage_message}"
|
|
||||||
)
|
|
||||||
for tool in data["tools"]:
|
|
||||||
if tool["function"]["name"] == function_name:
|
|
||||||
valid_tool = True
|
|
||||||
break
|
|
||||||
if not valid_tool:
|
|
||||||
raise ValueError(
|
|
||||||
"The tool specified in `tool_choice` does not match any"
|
|
||||||
" of the specified `tools`"
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_generation_prompt(cls, data):
|
|
||||||
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
|
||||||
raise ValueError(
|
|
||||||
"Cannot set both `continue_final_message` and "
|
|
||||||
"`add_generation_prompt` to True."
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_cache_salt_support(cls, data):
|
|
||||||
if data.get("cache_salt") is not None and (
|
|
||||||
not isinstance(data["cache_salt"], str) or not data["cache_salt"]
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
"Parameter 'cache_salt' must be a non-empty string if provided."
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
class CompletionRequest(OpenAIBaseModel):
|
class CompletionRequest(OpenAIBaseModel):
|
||||||
# Ordered by official OpenAI API documentation
|
# Ordered by official OpenAI API documentation
|
||||||
# https://platform.openai.com/docs/api-reference/completions/create
|
# https://platform.openai.com/docs/api-reference/completions/create
|
||||||
@@ -1486,75 +967,6 @@ class ExtractedToolCallInformation(BaseModel):
|
|||||||
content: str | None = None
|
content: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class ChatMessage(OpenAIBaseModel):
|
|
||||||
role: str
|
|
||||||
content: str | None = None
|
|
||||||
refusal: str | None = None
|
|
||||||
annotations: OpenAIAnnotation | None = None
|
|
||||||
audio: OpenAIChatCompletionAudio | None = None
|
|
||||||
function_call: FunctionCall | None = None
|
|
||||||
tool_calls: list[ToolCall] = Field(default_factory=list)
|
|
||||||
|
|
||||||
# vLLM-specific fields that are not in OpenAI spec
|
|
||||||
reasoning: str | None = None
|
|
||||||
reasoning_content: str | None = None
|
|
||||||
"""Deprecated: use `reasoning` instead."""
|
|
||||||
|
|
||||||
@model_validator(mode="after")
|
|
||||||
def handle_deprecated_reasoning_content(self):
|
|
||||||
"""Copy reasoning to reasoning_content for backward compatibility."""
|
|
||||||
self.reasoning_content = self.reasoning
|
|
||||||
return self
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionLogProb(OpenAIBaseModel):
|
|
||||||
token: str
|
|
||||||
logprob: float = -9999.0
|
|
||||||
bytes: list[int] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionLogProbsContent(ChatCompletionLogProb):
|
|
||||||
# Workaround: redefine fields name cache so that it's not
|
|
||||||
# shared with the super class.
|
|
||||||
field_names: ClassVar[set[str] | None] = None
|
|
||||||
top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionLogProbs(OpenAIBaseModel):
|
|
||||||
content: list[ChatCompletionLogProbsContent] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
|
||||||
index: int
|
|
||||||
message: ChatMessage
|
|
||||||
logprobs: ChatCompletionLogProbs | None = None
|
|
||||||
# per OpenAI spec this is the default
|
|
||||||
finish_reason: str | None = "stop"
|
|
||||||
# not part of the OpenAI spec but included in vLLM for legacy reasons
|
|
||||||
stop_reason: int | str | None = None
|
|
||||||
# not part of the OpenAI spec but is useful for tracing the tokens
|
|
||||||
# in agent scenarios
|
|
||||||
token_ids: list[int] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionResponse(OpenAIBaseModel):
|
|
||||||
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
|
||||||
object: Literal["chat.completion"] = "chat.completion"
|
|
||||||
created: int = Field(default_factory=lambda: int(time.time()))
|
|
||||||
model: str
|
|
||||||
choices: list[ChatCompletionResponseChoice]
|
|
||||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
|
||||||
system_fingerprint: str | None = None
|
|
||||||
usage: UsageInfo
|
|
||||||
|
|
||||||
# vLLM-specific fields that are not in OpenAI spec
|
|
||||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
|
||||||
prompt_token_ids: list[int] | None = None
|
|
||||||
kv_transfer_params: dict[str, Any] | None = Field(
|
|
||||||
default=None, description="KVTransfer parameters."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DeltaMessage(OpenAIBaseModel):
|
class DeltaMessage(OpenAIBaseModel):
|
||||||
role: str | None = None
|
role: str | None = None
|
||||||
content: str | None = None
|
content: str | None = None
|
||||||
@@ -1570,27 +982,6 @@ class DeltaMessage(OpenAIBaseModel):
|
|||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
|
||||||
index: int
|
|
||||||
delta: DeltaMessage
|
|
||||||
logprobs: ChatCompletionLogProbs | None = None
|
|
||||||
finish_reason: str | None = None
|
|
||||||
stop_reason: int | str | None = None
|
|
||||||
# not part of the OpenAI spec but for tracing the tokens
|
|
||||||
token_ids: list[int] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ChatCompletionStreamResponse(OpenAIBaseModel):
|
|
||||||
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
|
||||||
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
|
|
||||||
created: int = Field(default_factory=lambda: int(time.time()))
|
|
||||||
model: str
|
|
||||||
choices: list[ChatCompletionResponseStreamChoice]
|
|
||||||
usage: UsageInfo | None = Field(default=None)
|
|
||||||
# not part of the OpenAI spec but for tracing the tokens
|
|
||||||
prompt_token_ids: list[int] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
|
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
|
||||||
delta: DeltaMessage
|
delta: DeltaMessage
|
||||||
finish_reason: str | None = None
|
finish_reason: str | None = None
|
||||||
@@ -1856,128 +1247,6 @@ StreamingResponsesResponse: TypeAlias = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TokenizeCompletionRequest(OpenAIBaseModel):
|
|
||||||
model: str | None = None
|
|
||||||
prompt: str
|
|
||||||
|
|
||||||
add_special_tokens: bool = Field(
|
|
||||||
default=True,
|
|
||||||
description=(
|
|
||||||
"If true (the default), special tokens (e.g. BOS) will be added to "
|
|
||||||
"the prompt."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return_token_strs: bool | None = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If true, also return the token strings corresponding to the token ids."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TokenizeChatRequest(OpenAIBaseModel):
|
|
||||||
model: str | None = None
|
|
||||||
messages: list[ChatCompletionMessageParam]
|
|
||||||
|
|
||||||
add_generation_prompt: bool = Field(
|
|
||||||
default=True,
|
|
||||||
description=(
|
|
||||||
"If true, the generation prompt will be added to the chat template. "
|
|
||||||
"This is a parameter used by chat template in tokenizer config of the "
|
|
||||||
"model."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return_token_strs: bool | None = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If true, also return the token strings corresponding to the token ids."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
continue_final_message: bool = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If this is set, the chat will be formatted so that the final "
|
|
||||||
"message in the chat is open-ended, without any EOS tokens. The "
|
|
||||||
"model will continue this message rather than starting a new one. "
|
|
||||||
'This allows you to "prefill" part of the model\'s response for it. '
|
|
||||||
"Cannot be used at the same time as `add_generation_prompt`."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
add_special_tokens: bool = Field(
|
|
||||||
default=False,
|
|
||||||
description=(
|
|
||||||
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
|
||||||
"on top of what is added by the chat template. "
|
|
||||||
"For most models, the chat template takes care of adding the "
|
|
||||||
"special tokens so this should be set to false (as is the "
|
|
||||||
"default)."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
chat_template: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"A Jinja template to use for this conversion. "
|
|
||||||
"As of transformers v4.44, default chat template is no longer "
|
|
||||||
"allowed, so you must provide a chat template if the tokenizer "
|
|
||||||
"does not define one."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"Additional keyword args to pass to the template renderer. "
|
|
||||||
"Will be accessible by the chat template."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=("Additional kwargs to pass to the HF processor."),
|
|
||||||
)
|
|
||||||
tools: list[ChatCompletionToolsParam] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=("A list of tools the model may call."),
|
|
||||||
)
|
|
||||||
|
|
||||||
@model_validator(mode="before")
|
|
||||||
@classmethod
|
|
||||||
def check_generation_prompt(cls, data):
|
|
||||||
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
|
||||||
raise ValueError(
|
|
||||||
"Cannot set both `continue_final_message` and "
|
|
||||||
"`add_generation_prompt` to True."
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
|
|
||||||
|
|
||||||
|
|
||||||
class TokenizeResponse(OpenAIBaseModel):
|
|
||||||
count: int
|
|
||||||
max_model_len: int
|
|
||||||
tokens: list[int]
|
|
||||||
token_strs: list[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class DetokenizeRequest(OpenAIBaseModel):
|
|
||||||
model: str | None = None
|
|
||||||
tokens: list[int]
|
|
||||||
|
|
||||||
|
|
||||||
class DetokenizeResponse(OpenAIBaseModel):
|
|
||||||
prompt: str
|
|
||||||
|
|
||||||
|
|
||||||
class TokenizerInfoResponse(OpenAIBaseModel):
|
|
||||||
"""
|
|
||||||
Response containing tokenizer configuration
|
|
||||||
equivalent to tokenizer_config.json
|
|
||||||
"""
|
|
||||||
|
|
||||||
model_config = ConfigDict(extra="allow")
|
|
||||||
tokenizer_class: str
|
|
||||||
|
|
||||||
|
|
||||||
class LoadLoRAAdapterRequest(BaseModel):
|
class LoadLoRAAdapterRequest(BaseModel):
|
||||||
lora_name: str
|
lora_name: str
|
||||||
lora_path: str
|
lora_path: str
|
||||||
@@ -2537,30 +1806,3 @@ class GenerateRequest(BaseModel):
|
|||||||
default=None,
|
default=None,
|
||||||
description="KVTransfer parameters used for disaggregated serving.",
|
description="KVTransfer parameters used for disaggregated serving.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class GenerateResponseChoice(BaseModel):
|
|
||||||
index: int
|
|
||||||
logprobs: ChatCompletionLogProbs | None = None
|
|
||||||
# per OpenAI spec this is the default
|
|
||||||
finish_reason: str | None = "stop"
|
|
||||||
token_ids: list[int] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class GenerateResponse(BaseModel):
|
|
||||||
request_id: str = Field(
|
|
||||||
default_factory=random_uuid,
|
|
||||||
description=(
|
|
||||||
"The request_id related to this request. If the caller does "
|
|
||||||
"not set it, a random_uuid will be generated. This id is used "
|
|
||||||
"through out the inference process and return in response."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
choices: list[GenerateResponseChoice]
|
|
||||||
|
|
||||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
|
||||||
|
|
||||||
kv_transfer_params: dict[str, Any] | None = Field(
|
|
||||||
default=None,
|
|
||||||
description="KVTransfer parameters used for disaggregated serving.",
|
|
||||||
)
|
|
||||||
+9
-5
@@ -38,22 +38,20 @@ from vllm.entrypoints.context import (
|
|||||||
StreamingHarmonyContext,
|
StreamingHarmonyContext,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
DetokenizeRequest,
|
|
||||||
ErrorInfo,
|
ErrorInfo,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
FunctionCall,
|
FunctionCall,
|
||||||
FunctionDefinition,
|
FunctionDefinition,
|
||||||
ResponseInputOutputItem,
|
ResponseInputOutputItem,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
TokenizeChatRequest,
|
|
||||||
TokenizeCompletionRequest,
|
|
||||||
TokenizeResponse,
|
|
||||||
TranscriptionRequest,
|
TranscriptionRequest,
|
||||||
TranscriptionResponse,
|
TranscriptionResponse,
|
||||||
TranslationRequest,
|
TranslationRequest,
|
||||||
@@ -86,6 +84,12 @@ from vllm.entrypoints.responses_utils import (
|
|||||||
construct_input_messages,
|
construct_input_messages,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
|
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
|
||||||
|
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||||
|
DetokenizeRequest,
|
||||||
|
TokenizeChatRequest,
|
||||||
|
TokenizeCompletionRequest,
|
||||||
|
TokenizeResponse,
|
||||||
|
)
|
||||||
from vllm.entrypoints.utils import _validate_truncation_size
|
from vllm.entrypoints.utils import _validate_truncation_size
|
||||||
from vllm.inputs.data import PromptType, TokensPrompt
|
from vllm.inputs.data import PromptType, TokensPrompt
|
||||||
from vllm.inputs.parse import (
|
from vllm.inputs.parse import (
|
||||||
@@ -43,8 +43,8 @@ from openai_harmony import Message as OpenAIHarmonyMessage
|
|||||||
from openai_harmony import Role as OpenAIHarmonyRole
|
from openai_harmony import Role as OpenAIHarmonyRole
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
|
||||||
ChatCompletionToolsParam,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ResponseInputOutputItem,
|
ResponseInputOutputItem,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,7 +16,10 @@ from openai.types.responses.response_reasoning_item import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.constants import MCP_PREFIX
|
from vllm.entrypoints.constants import MCP_PREFIX
|
||||||
from vllm.entrypoints.openai.protocol import ResponseInputOutputItem, ResponsesRequest
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
ResponseInputOutputItem,
|
||||||
|
ResponsesRequest,
|
||||||
|
)
|
||||||
from vllm.outputs import CompletionOutput
|
from vllm.outputs import CompletionOutput
|
||||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|||||||
@@ -19,13 +19,15 @@ from tqdm import tqdm
|
|||||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
OpenAIBaseModel,
|
OpenAIBaseModel,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
||||||
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
|
||||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest, EmbeddingResponse
|
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest, EmbeddingResponse
|
||||||
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
|
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from fastapi import Request
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
CompletionLogProbs,
|
CompletionLogProbs,
|
||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
UsageInfo,
|
UsageInfo,
|
||||||
VLLMValidationError,
|
VLLMValidationError,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import (
|
from vllm.entrypoints.openai.engine.serving import (
|
||||||
GenerationError,
|
GenerationError,
|
||||||
OpenAIServing,
|
OpenAIServing,
|
||||||
clamp_prompt_logprobs,
|
clamp_prompt_logprobs,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from dataclasses import dataclass
|
|||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorInfo,
|
ErrorInfo,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
LoadLoRAAdapterRequest,
|
LoadLoRAAdapterRequest,
|
||||||
|
|||||||
@@ -72,19 +72,7 @@ from vllm.entrypoints.context import (
|
|||||||
StreamingHarmonyContext,
|
StreamingHarmonyContext,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
construct_harmony_previous_input_messages,
|
|
||||||
get_developer_message,
|
|
||||||
get_stop_tokens_for_assistant_actions,
|
|
||||||
get_system_message,
|
|
||||||
get_user_message,
|
|
||||||
has_custom_tools,
|
|
||||||
parse_output_message,
|
|
||||||
parse_remaining_state,
|
|
||||||
parse_response_input,
|
|
||||||
render_for_completion,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
InputTokensDetails,
|
InputTokensDetails,
|
||||||
@@ -102,10 +90,22 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
StreamingResponsesResponse,
|
StreamingResponsesResponse,
|
||||||
VLLMValidationError,
|
VLLMValidationError,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import (
|
from vllm.entrypoints.openai.engine.serving import (
|
||||||
GenerationError,
|
GenerationError,
|
||||||
OpenAIServing,
|
OpenAIServing,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
|
construct_harmony_previous_input_messages,
|
||||||
|
get_developer_message,
|
||||||
|
get_stop_tokens_for_assistant_actions,
|
||||||
|
get_system_message,
|
||||||
|
get_user_message,
|
||||||
|
has_custom_tools,
|
||||||
|
parse_output_message,
|
||||||
|
parse_remaining_state,
|
||||||
|
parse_response_input,
|
||||||
|
render_for_completion,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.entrypoints.responses_utils import (
|
from vllm.entrypoints.responses_utils import (
|
||||||
construct_input_messages,
|
construct_input_messages,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from fastapi import Request
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
TranscriptionRequest,
|
TranscriptionRequest,
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from transformers import PreTrainedTokenizerBase
|
|||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.protocol import (
|
|||||||
UsageInfo,
|
UsageInfo,
|
||||||
VLLMValidationError,
|
VLLMValidationError,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing, SpeechToTextRequest
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.inputs.data import PromptType
|
from vllm.inputs.data import PromptType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from typing import TypeVar
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponseChoice,
|
ChatCompletionResponseChoice,
|
||||||
ChatCompletionResponseStreamChoice,
|
ChatCompletionResponseStreamChoice,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
|
|||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.classify.protocol import (
|
from vllm.entrypoints.pooling.classify.protocol import (
|
||||||
ClassificationRequest,
|
ClassificationRequest,
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from pydantic import (
|
|||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config.pooler import get_use_activation
|
from vllm.config.pooler import get_use_activation
|
||||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ from fastapi import Request
|
|||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import (
|
from vllm.entrypoints.openai.engine.serving import (
|
||||||
ClassificationServeContext,
|
ClassificationServeContext,
|
||||||
OpenAIServing,
|
OpenAIServing,
|
||||||
ServeContext,
|
ServeContext,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
|
|||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.embed.protocol import (
|
from vllm.entrypoints.pooling.embed.protocol import (
|
||||||
EmbeddingBytesResponse,
|
EmbeddingBytesResponse,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from pydantic import (
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
|
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
|
||||||
|
|
||||||
|
|||||||
@@ -12,11 +12,11 @@ from typing_extensions import assert_never, override
|
|||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import (
|
from vllm.entrypoints.openai.engine.serving import (
|
||||||
EmbeddingServeContext,
|
EmbeddingServeContext,
|
||||||
OpenAIServing,
|
OpenAIServing,
|
||||||
ServeContext,
|
ServeContext,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
|
|||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.pooling.protocol import (
|
from vllm.entrypoints.pooling.pooling.protocol import (
|
||||||
IOProcessorResponse,
|
IOProcessorResponse,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from pydantic import (
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config.pooler import get_use_activation
|
from vllm.config.pooler import get_use_activation
|
||||||
from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.entrypoints.pooling.embed.protocol import (
|
from vllm.entrypoints.pooling.embed.protocol import (
|
||||||
EmbeddingChatRequest,
|
EmbeddingChatRequest,
|
||||||
EmbeddingCompletionRequest,
|
EmbeddingCompletionRequest,
|
||||||
|
|||||||
@@ -14,11 +14,11 @@ from typing_extensions import assert_never
|
|||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.entrypoints.pooling.pooling.protocol import (
|
from vllm.entrypoints.pooling.pooling.protocol import (
|
||||||
IOProcessorRequest,
|
IOProcessorRequest,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
|
|||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.score.protocol import (
|
from vllm.entrypoints.pooling.score.protocol import (
|
||||||
RerankRequest,
|
RerankRequest,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from pydantic import (
|
|||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.config.pooler import get_use_activation
|
from vllm.config.pooler import get_use_activation
|
||||||
from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
|
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||||
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
|
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
|
|||||||
@@ -9,11 +9,11 @@ from fastapi import Request
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.entrypoints.pooling.score.protocol import (
|
from vllm.entrypoints.pooling.score.protocol import (
|
||||||
RerankDocument,
|
RerankDocument,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from openai.types.responses.tool import Tool
|
|||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.entrypoints.constants import MCP_PREFIX
|
from vllm.entrypoints.constants import MCP_PREFIX
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ResponseInputOutputItem,
|
ResponseInputOutputItem,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -14,16 +14,20 @@ from vllm.entrypoints.openai.api_server import (
|
|||||||
base,
|
base,
|
||||||
chat,
|
chat,
|
||||||
completion,
|
completion,
|
||||||
create_chat_completion,
|
|
||||||
create_completion,
|
create_completion,
|
||||||
validate_json_request,
|
validate_json_request,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||||
|
create_chat_completion,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.pooling.classify.api_router import classify, create_classify
|
from vllm.entrypoints.pooling.classify.api_router import classify, create_classify
|
||||||
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
|
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
|
||||||
from vllm.entrypoints.pooling.embed.api_router import create_embedding, embedding
|
from vllm.entrypoints.pooling.embed.api_router import create_embedding, embedding
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.api_server import validate_json_request
|
from vllm.entrypoints.openai.api_server import validate_json_request
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.serve.disagg.protocol import (
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ from typing import Any
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
|
||||||
ChatCompletionLogProbs,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
Logprob,
|
Logprob,
|
||||||
SamplingParams,
|
SamplingParams,
|
||||||
StreamOptions,
|
StreamOptions,
|
||||||
|
|||||||
@@ -11,16 +11,18 @@ from fastapi import Request
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionLogProb,
|
ChatCompletionLogProb,
|
||||||
ChatCompletionLogProbs,
|
ChatCompletionLogProbs,
|
||||||
ChatCompletionLogProbsContent,
|
ChatCompletionLogProbsContent,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
PromptTokenUsageInfo,
|
PromptTokenUsageInfo,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
UsageInfo,
|
UsageInfo,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.serve.disagg.protocol import (
|
||||||
GenerateRequest,
|
GenerateRequest,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.api_server import validate_json_request
|
from vllm.entrypoints.openai.api_server import validate_json_request
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse, Response
|
|||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.entrypoints.openai.api_server import models, validate_json_request
|
from vllm.entrypoints.openai.api_server import models, validate_json_request
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
LoadLoRAAdapterRequest,
|
LoadLoRAAdapterRequest,
|
||||||
UnloadLoRAAdapterRequest,
|
UnloadLoRAAdapterRequest,
|
||||||
|
|||||||
@@ -10,10 +10,12 @@ from fastapi.responses import JSONResponse
|
|||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.entrypoints.openai.api_server import validate_json_request
|
from vllm.entrypoints.openai.api_server import validate_json_request
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
ErrorResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||||
DetokenizeRequest,
|
DetokenizeRequest,
|
||||||
DetokenizeResponse,
|
DetokenizeResponse,
|
||||||
ErrorResponse,
|
|
||||||
TokenizeRequest,
|
TokenizeRequest,
|
||||||
TokenizeResponse,
|
TokenizeResponse,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
|
||||||
|
from typing import Any, TypeAlias
|
||||||
|
|
||||||
|
from pydantic import ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
from vllm.entrypoints.chat_utils import (
|
||||||
|
ChatCompletionMessageParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionToolsParam,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
OpenAIBaseModel,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenizeCompletionRequest(OpenAIBaseModel):
|
||||||
|
model: str | None = None
|
||||||
|
prompt: str
|
||||||
|
|
||||||
|
add_special_tokens: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If true (the default), special tokens (e.g. BOS) will be added to "
|
||||||
|
"the prompt."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return_token_strs: bool | None = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If true, also return the token strings corresponding to the token ids."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenizeChatRequest(OpenAIBaseModel):
|
||||||
|
model: str | None = None
|
||||||
|
messages: list[ChatCompletionMessageParam]
|
||||||
|
|
||||||
|
add_generation_prompt: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If true, the generation prompt will be added to the chat template. "
|
||||||
|
"This is a parameter used by chat template in tokenizer config of the "
|
||||||
|
"model."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return_token_strs: bool | None = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If true, also return the token strings corresponding to the token ids."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
continue_final_message: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If this is set, the chat will be formatted so that the final "
|
||||||
|
"message in the chat is open-ended, without any EOS tokens. The "
|
||||||
|
"model will continue this message rather than starting a new one. "
|
||||||
|
'This allows you to "prefill" part of the model\'s response for it. '
|
||||||
|
"Cannot be used at the same time as `add_generation_prompt`."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
add_special_tokens: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
||||||
|
"on top of what is added by the chat template. "
|
||||||
|
"For most models, the chat template takes care of adding the "
|
||||||
|
"special tokens so this should be set to false (as is the "
|
||||||
|
"default)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chat_template: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"A Jinja template to use for this conversion. "
|
||||||
|
"As of transformers v4.44, default chat template is no longer "
|
||||||
|
"allowed, so you must provide a chat template if the tokenizer "
|
||||||
|
"does not define one."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"Additional keyword args to pass to the template renderer. "
|
||||||
|
"Will be accessible by the chat template."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=("Additional kwargs to pass to the HF processor."),
|
||||||
|
)
|
||||||
|
tools: list[ChatCompletionToolsParam] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=("A list of tools the model may call."),
|
||||||
|
)
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_generation_prompt(cls, data):
|
||||||
|
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
||||||
|
raise ValueError(
|
||||||
|
"Cannot set both `continue_final_message` and "
|
||||||
|
"`add_generation_prompt` to True."
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
|
||||||
|
|
||||||
|
|
||||||
|
class TokenizeResponse(OpenAIBaseModel):
|
||||||
|
count: int
|
||||||
|
max_model_len: int
|
||||||
|
tokens: list[int]
|
||||||
|
token_strs: list[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DetokenizeRequest(OpenAIBaseModel):
|
||||||
|
model: str | None = None
|
||||||
|
tokens: list[int]
|
||||||
|
|
||||||
|
|
||||||
|
class DetokenizeResponse(OpenAIBaseModel):
|
||||||
|
prompt: str
|
||||||
|
|
||||||
|
|
||||||
|
class TokenizerInfoResponse(OpenAIBaseModel):
|
||||||
|
"""
|
||||||
|
Response containing tokenizer configuration
|
||||||
|
equivalent to tokenizer_config.json
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="allow")
|
||||||
|
tokenizer_class: str
|
||||||
@@ -9,18 +9,20 @@ from fastapi import Request
|
|||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
from vllm.entrypoints.logger import RequestLogger
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
ErrorResponse,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
|
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.renderer import RenderConfig
|
||||||
|
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||||
DetokenizeRequest,
|
DetokenizeRequest,
|
||||||
DetokenizeResponse,
|
DetokenizeResponse,
|
||||||
ErrorResponse,
|
|
||||||
TokenizeChatRequest,
|
TokenizeChatRequest,
|
||||||
TokenizeRequest,
|
TokenizeRequest,
|
||||||
TokenizeResponse,
|
TokenizeResponse,
|
||||||
TokenizerInfoResponse,
|
TokenizerInfoResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
|
||||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
|
||||||
from vllm.entrypoints.renderer import RenderConfig
|
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|||||||
@@ -22,9 +22,11 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
resolve_hf_chat_template,
|
resolve_hf_chat_template,
|
||||||
resolve_mistral_chat_template,
|
resolve_mistral_chat_template,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
StreamOptions,
|
StreamOptions,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -14,8 +14,10 @@ from vllm.utils.collection_utils import is_list_of
|
|||||||
from vllm.utils.import_utils import import_from_path
|
from vllm.utils.import_utils import import_from_path
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -5,13 +5,15 @@ from abc import abstractmethod
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import DeltaMessage
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import DeltaMessage
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|
||||||
|
|||||||
@@ -5,8 +5,11 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
|
from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
|
||||||
from vllm.entrypoints.tool_server import ToolServer
|
from vllm.entrypoints.tool_server import ToolServer
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ from collections.abc import Sequence
|
|||||||
import regex as re
|
import regex as re
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,10 @@
|
|||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import (
|
from vllm.reasoning import (
|
||||||
ReasoningParser,
|
ReasoningParser,
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ from collections.abc import Sequence
|
|||||||
import regex as re
|
import regex as re
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,10 @@
|
|||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,8 +3,10 @@
|
|||||||
|
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|||||||
@@ -10,9 +10,10 @@ import regex as re
|
|||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
from vllm.entrypoints.openai.protocol import (
|
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ResponsesRequest
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
|
||||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ from collections.abc import Sequence
|
|||||||
import regex as re
|
import regex as re
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.reasoning import ReasoningParser
|
from vllm.reasoning import ReasoningParser
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.sentencepiece import (
|
|||||||
from mistral_common.tokens.tokenizers.tekken import Tekkenizer
|
from mistral_common.tokens.tokenizers.tekken import Tekkenizer
|
||||||
|
|
||||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||||
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
from .protocol import TokenizerLike
|
from .protocol import TokenizerLike
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ from openai.types.responses.response_format_text_json_schema_config import (
|
|||||||
ResponseFormatTextJSONSchemaConfig,
|
ResponseFormatTextJSONSchemaConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
ExtractedToolCallInformation,
|
ExtractedToolCallInformation,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ from collections.abc import Sequence
|
|||||||
import regex as re
|
import regex as re
|
||||||
|
|
||||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||||
from vllm.entrypoints.openai.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
DeltaToolCall,
|
DeltaToolCall,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user