[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from tests.v1.utils import check_request_balancing
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
DP_SIZE = os.getenv("DP_SIZE", "1")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--api-server-count",
|
||||
"4",
|
||||
"--data_parallel_size",
|
||||
DP_SIZE,
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_completion(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
async def make_request():
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
|
||||
choice = completion.choices[0]
|
||||
# The exact number of tokens can vary slightly with temperature=1.0,
|
||||
# so we check for a reasonable minimum length.
|
||||
assert len(choice.text) >= 1
|
||||
# Finish reason might not always be 'length' if the model finishes early
|
||||
# or due to other reasons, especially with high temperature.
|
||||
# So, we'll accept 'length' or 'stop'.
|
||||
assert choice.finish_reason in ("length", "stop")
|
||||
|
||||
# Token counts can also vary, so we check they are positive.
|
||||
assert completion.usage.completion_tokens > 0
|
||||
assert completion.usage.prompt_tokens > 0
|
||||
assert completion.usage.total_tokens > 0
|
||||
return completion
|
||||
|
||||
# Test single request
|
||||
result = await make_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_streaming(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request():
|
||||
# Perform a non-streaming request to get the expected full output
|
||||
single_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
# Perform the streaming request
|
||||
stream = await client.completions.create(
|
||||
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
last_chunk = None
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
last_chunk = chunk # Keep track of the last chunk
|
||||
|
||||
# finish reason should only return in the last block for OpenAI API
|
||||
assert finish_reason_count == 1, "Finish reason should appear exactly once."
|
||||
assert last_chunk is not None, "Stream should have yielded at least one chunk."
|
||||
assert last_chunk.choices[0].finish_reason == "length", (
|
||||
"Finish reason should be 'length'."
|
||||
)
|
||||
# Check that the combined text matches the non-streamed version.
|
||||
assert "".join(chunks) == single_output, (
|
||||
"Streamed output should match non-streamed output."
|
||||
)
|
||||
return True # Indicate success for this request
|
||||
|
||||
# Test single request
|
||||
result = await make_streaming_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def vision_llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
"microsoft/Phi-3.5-vision-instruct",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
seed=0,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
|
||||
)
|
||||
def test_chat_multi_image(vision_llm, image_urls: list[str]):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*(
|
||||
{"type": "image_url", "image_url": {"url": image_url}}
|
||||
for image_url in image_urls
|
||||
),
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
outputs = vision_llm.chat(messages)
|
||||
assert len(outputs) >= 0
|
||||
@@ -1,643 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
MODEL_NAME_SHORT = "gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
|
||||
BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
task = "generate"
|
||||
runner_type = "generate"
|
||||
model = MODEL_NAME
|
||||
tokenizer = MODEL_NAME
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
revision = None
|
||||
code_revision = None
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = MockHFConfig()
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
def _build_renderer(model_config: MockModelConfig):
|
||||
return HfRenderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_chat.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_chat
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_non_stream():
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
with pytest.raises(GenerationError):
|
||||
await serving_chat.create_chat_completion(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chat_keeps_mm_cache_for_engine_execution():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
result = await serving_chat.render_chat_request(request)
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert (
|
||||
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream(include_usage: bool):
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output_1 = CompletionOutput(
|
||||
index=0,
|
||||
text="Hello",
|
||||
token_ids=[100],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=None,
|
||||
)
|
||||
|
||||
request_output_1 = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output_1],
|
||||
finished=False,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
completion_output_2 = CompletionOutput(
|
||||
index=0,
|
||||
text="Hello",
|
||||
token_ids=[100],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output_2 = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output_2],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output_1
|
||||
yield request_output_2
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream_first_chunk(include_usage: bool):
|
||||
"""An error before the first token must reach the streaming client."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_content",
|
||||
[
|
||||
[{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}],
|
||||
[{"image_url": {"url": "https://example.com/image.jpg"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_image(image_content):
|
||||
"""Test that system messages with image content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": image_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "image_url" in call_args
|
||||
|
||||
|
||||
def test_system_message_accepts_text():
|
||||
"""Test that system messages can contain text content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "system"
|
||||
|
||||
|
||||
def test_system_message_accepts_text_array():
|
||||
"""Test that system messages can contain an array with text content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "You are a helpful assistant."}],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "system"
|
||||
|
||||
|
||||
def test_user_message_accepts_image():
|
||||
"""Test that user messages can still contain image content."""
|
||||
# Should not raise an exception
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.jpg"},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert request.messages[0]["role"] == "user"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"audio_content",
|
||||
[
|
||||
[
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "base64data", "format": "wav"},
|
||||
}
|
||||
],
|
||||
[{"input_audio": {"data": "base64data", "format": "wav"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_audio(audio_content):
|
||||
"""Test that system messages with audio content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": audio_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "input_audio" in call_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"video_content",
|
||||
[
|
||||
[{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}],
|
||||
[{"video_url": {"url": "https://example.com/video.mp4"}}],
|
||||
],
|
||||
)
|
||||
def test_system_message_warns_on_video(video_content):
|
||||
"""Test that system messages with video content trigger a warning."""
|
||||
with patch(
|
||||
"vllm.entrypoints.openai.chat_completion.protocol.logger"
|
||||
) as mock_logger:
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": video_content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called()
|
||||
call_args = str(mock_logger.warning_once.call_args)
|
||||
assert "System messages should only contain text" in call_args
|
||||
assert "video_url" in call_args
|
||||
|
||||
|
||||
def test_json_schema_response_format_missing_schema():
|
||||
"""When response_format type is 'json_schema' but the json_schema field
|
||||
is not provided, request construction should raise a validation error
|
||||
so the API returns 400 instead of 500."""
|
||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "json_schema"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
mm_processor_kwargs={
|
||||
"patch_size": 1,
|
||||
"vision_min_num_patches": 3_000_000_000,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request mm_processor_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_media_io_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
media_io_kwargs={"video": {"num_frames": 1_000_000}},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request media_io_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs={"use_audio_in_video": True},
|
||||
media_io_kwargs={"video": {"num_frames": 4}},
|
||||
trust_request_mm_kwargs=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_structural_tag_response_format_invalid(format_value):
|
||||
"""Malformed structural tags should be rejected during request validation."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_batch_structural_tag_response_format_invalid(format_value):
|
||||
"""Batch chat should reject malformed structural tags at request parsing."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
BatchChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[[{"role": "user", "content": "hello"}]],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"structural_tag",
|
||||
[
|
||||
"not json",
|
||||
"",
|
||||
# json.loads raises a plain ValueError, not JSONDecodeError
|
||||
'{"type": "structural_tag", "x": ' + "1" * 5000 + "}",
|
||||
],
|
||||
)
|
||||
def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
"""Malformed direct structured_outputs structural tags should be rejected."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Invalid structured_outputs structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
structured_outputs={"structural_tag": structural_tag},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["response_format", "structured_outputs"])
|
||||
def test_deeply_nested_structural_tag_rejected(field):
|
||||
"""Structural tags are compiled during request parsing, so a deeply nested
|
||||
one must be rejected there instead of blocking the server while converting,
|
||||
and reported as too deep rather than malformed."""
|
||||
schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200
|
||||
tag = (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
value = json.loads(tag) if field == "response_format" else {"structural_tag": tag}
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
**{field: value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
||||
def test_non_numeric_logprobs_rejected(field_name):
|
||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
**{field_name: "2"},
|
||||
)
|
||||
@@ -1,304 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
|
||||
# downloading lora to test lora requests
|
||||
from openai import BadRequestError
|
||||
from transformers import AutoConfig
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
LORA_SERVING_MODEL_NAME = "opt125m-lora"
|
||||
|
||||
CONFIG = AutoConfig.from_pretrained(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=["use-lora"])
|
||||
def default_server_args(
|
||||
request: pytest.FixtureRequest, opt125_lora_files: str
|
||||
) -> list[str]:
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
if request.param == "use-lora":
|
||||
lora_module_1 = {
|
||||
"name": LORA_SERVING_MODEL_NAME,
|
||||
"path": opt125_lora_files,
|
||||
"base_model_name": MODEL_NAME,
|
||||
}
|
||||
|
||||
args.extend(
|
||||
[
|
||||
"--enable-lora",
|
||||
"--lora-module",
|
||||
json.dumps(lora_module_1),
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
EXAMPLE_PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"What is an LLM?",
|
||||
]
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor):
|
||||
buffer = io.BytesIO()
|
||||
torch.save(embeds, buffer)
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def example_prompt_embeds(hf_runner):
|
||||
"""Create example embeddings and return them as base64 encoded string."""
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
example_embeddings = hf_model.get_prompt_embeddings(EXAMPLE_PROMPTS)
|
||||
|
||||
return [_encode_embeds(item) for item in example_embeddings]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_with_prompt_embeds(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client_with_prompt_embeds(server_with_prompt_embeds):
|
||||
async with server_with_prompt_embeds.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_with_prompt_embeds(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
):
|
||||
encoded_embeds, encoded_embeds2 = example_prompt_embeds
|
||||
|
||||
# Test case: Single prompt embeds input
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
assert len(completion.choices[0].text) >= 1
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
# Test case: batch completion with prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
assert len(completion.choices) == 2
|
||||
assert len(completion.choices[0].text) >= 1
|
||||
assert len(completion.choices[1].text) >= 1
|
||||
|
||||
# Test case: streaming with prompt_embeds
|
||||
single_completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
stream = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
chunks = []
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
assert finish_reason_count == 1
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
assert chunk.choices[0].text
|
||||
assert "".join(chunks) == single_output
|
||||
|
||||
# Test case: batch streaming with prompt_embeds
|
||||
stream = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
chunks_stream_embeds: list[list[str]] = [[], []]
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
chunks_stream_embeds[chunk.choices[0].index].append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
assert finish_reason_count == 2
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
assert chunk.choices[0].text
|
||||
assert len(chunks_stream_embeds[0]) > 0
|
||||
assert len(chunks_stream_embeds[1]) > 0
|
||||
|
||||
# Test case: mixed text and prompt_embeds
|
||||
completion_mixed = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt="This is a prompt",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
assert len(completion.choices) == 2
|
||||
completion_text_only = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt="This is a prompt",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
completion_embeds_only = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
# Embeddings responses should be handled first
|
||||
assert completion_mixed.choices[0].text == completion_embeds_only.choices[0].text
|
||||
assert completion_mixed.choices[1].text == completion_text_only.choices[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_errors_with_prompt_embeds(
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
# Test error case: invalid prompt_embeds
|
||||
with pytest.raises(BadRequestError):
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
prompt=None,
|
||||
model=model_name,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": "invalid_base64"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logprobs_arg", [1, 0])
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME, LORA_SERVING_MODEL_NAME])
|
||||
async def test_completions_with_logprobs_and_prompt_embeds(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
logprobs_arg: int,
|
||||
model_name: str,
|
||||
):
|
||||
encoded_embeds, encoded_embeds2 = example_prompt_embeds
|
||||
|
||||
# Test case: Logprobs using prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
echo=False,
|
||||
logprobs=logprobs_arg,
|
||||
extra_body={"prompt_embeds": encoded_embeds},
|
||||
)
|
||||
|
||||
logprobs = completion.choices[0].logprobs
|
||||
assert logprobs is not None
|
||||
assert len(logprobs.text_offset) == 5
|
||||
assert len(logprobs.token_logprobs) == 5
|
||||
assert len(logprobs.top_logprobs) == 5
|
||||
for top_logprobs in logprobs.top_logprobs[1:]:
|
||||
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
|
||||
assert len(logprobs.tokens) == 5
|
||||
|
||||
# Test case: Log probs with batch completion and prompt_embeds
|
||||
completion = await client_with_prompt_embeds.completions.create(
|
||||
model=model_name,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
echo=False,
|
||||
logprobs=logprobs_arg,
|
||||
extra_body={"prompt_embeds": [encoded_embeds, encoded_embeds2]},
|
||||
)
|
||||
|
||||
assert len(completion.choices) == 2
|
||||
for choice in completion.choices:
|
||||
logprobs = choice.logprobs
|
||||
assert logprobs is not None
|
||||
assert len(logprobs.text_offset) == 5
|
||||
assert len(logprobs.token_logprobs) == 5
|
||||
assert len(logprobs.top_logprobs) == 5
|
||||
for top_logprobs in logprobs.top_logprobs[1:]:
|
||||
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
|
||||
assert len(logprobs.tokens) == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_logprobs_raises_error(
|
||||
example_prompt_embeds,
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
):
|
||||
encoded_embeds, _ = example_prompt_embeds
|
||||
|
||||
with pytest.raises(BadRequestError, match="not compatible"):
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt=None,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": encoded_embeds, "prompt_logprobs": True},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_prompt_embeds(
|
||||
client_with_prompt_embeds: openai.AsyncOpenAI,
|
||||
) -> None:
|
||||
await client_with_prompt_embeds.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body={"prompt_embeds": []},
|
||||
)
|
||||
@@ -1,256 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
|
||||
|
||||
MOCK_RESOLVER_NAME = "mock_test_resolver"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
"""Minimal mock ModelConfig for testing."""
|
||||
|
||||
model: str = MODEL_NAME
|
||||
runner_type = "generate"
|
||||
tokenizer: str = MODEL_NAME
|
||||
trust_remote_code: bool = False
|
||||
tokenizer_mode: str = "auto"
|
||||
max_model_len: int = 100
|
||||
tokenizer_revision: str | None = None
|
||||
multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)
|
||||
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
class MockLoRAResolver(LoRAResolver):
|
||||
async def resolve_lora(
|
||||
self, base_model_name: str, lora_name: str
|
||||
) -> LoRARequest | None:
|
||||
if lora_name == "test-lora":
|
||||
return LoRARequest(
|
||||
lora_name="test-lora",
|
||||
lora_int_id=1,
|
||||
lora_path="/fake/path/test-lora",
|
||||
)
|
||||
elif lora_name == "invalid-lora":
|
||||
return LoRARequest(
|
||||
lora_name="invalid-lora",
|
||||
lora_int_id=2,
|
||||
lora_path="/fake/path/invalid-lora",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def register_mock_resolver():
|
||||
"""Fixture to register and unregister the mock LoRA resolver."""
|
||||
resolver = MockLoRAResolver()
|
||||
LoRAResolverRegistry.register_resolver(MOCK_RESOLVER_NAME, resolver)
|
||||
yield
|
||||
# Cleanup: remove the resolver after the test runs
|
||||
if MOCK_RESOLVER_NAME in LoRAResolverRegistry.resolvers:
|
||||
del LoRAResolverRegistry.resolvers[MOCK_RESOLVER_NAME]
|
||||
|
||||
|
||||
def _build_renderer(model_config: MockModelConfig):
|
||||
return HfRenderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_serving_setup():
|
||||
"""Provides a mocked engine and serving completion instance."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
|
||||
async def mock_add_lora_side_effect(lora_request: LoRARequest):
|
||||
"""Simulate engine behavior when adding LoRAs."""
|
||||
if lora_request.lora_name == "test-lora":
|
||||
# Simulate successful addition
|
||||
return True
|
||||
if lora_request.lora_name == "invalid-lora":
|
||||
# Simulate failure during addition (e.g. invalid format)
|
||||
raise ValueError(f"Simulated failure adding LoRA: {lora_request.lora_name}")
|
||||
return True
|
||||
|
||||
mock_engine.add_lora = AsyncMock(side_effect=mock_add_lora_side_effect)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
for _ in []:
|
||||
yield _
|
||||
|
||||
mock_engine.generate = MagicMock(spec=AsyncLLM.generate, side_effect=mock_generate)
|
||||
|
||||
mock_engine.generate.reset_mock()
|
||||
mock_engine.add_lora.reset_mock()
|
||||
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
models = OpenAIServingModels(
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
serving_completion = OpenAIServingCompletion(
|
||||
mock_engine, models, online_renderer=online_renderer, request_logger=None
|
||||
)
|
||||
|
||||
return mock_engine, serving_completion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_with_lora_resolver(mock_serving_setup, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
lora_model_name = "test-lora"
|
||||
req_found = CompletionRequest(
|
||||
model=lora_model_name,
|
||||
prompt="Generate with LoRA",
|
||||
)
|
||||
|
||||
# Suppress potential errors during the mocked generate call,
|
||||
# as we are primarily checking for add_lora and generate calls
|
||||
with suppress(Exception):
|
||||
await serving_completion.create_completion(req_found)
|
||||
|
||||
mock_engine.add_lora.assert_awaited_once()
|
||||
called_lora_request = mock_engine.add_lora.call_args[0][0]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == lora_model_name
|
||||
|
||||
mock_engine.generate.assert_called_once()
|
||||
called_lora_request = mock_engine.generate.call_args[1]["lora_request"]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == lora_model_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_resolver_not_found(mock_serving_setup, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
non_existent_model = "non-existent-lora-adapter"
|
||||
req = CompletionRequest(
|
||||
model=non_existent_model,
|
||||
prompt="what is 1+1?",
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(req)
|
||||
|
||||
mock_engine.add_lora.assert_not_awaited()
|
||||
mock_engine.generate.assert_not_called()
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == HTTPStatus.NOT_FOUND.value
|
||||
assert non_existent_model in response.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_resolver_add_lora_fails(
|
||||
mock_serving_setup, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "true")
|
||||
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
invalid_model = "invalid-lora"
|
||||
req = CompletionRequest(
|
||||
model=invalid_model,
|
||||
prompt="what is 1+1?",
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(req)
|
||||
|
||||
# Assert add_lora was called before the failure
|
||||
mock_engine.add_lora.assert_awaited_once()
|
||||
called_lora_request = mock_engine.add_lora.call_args[0][0]
|
||||
assert isinstance(called_lora_request, LoRARequest)
|
||||
assert called_lora_request.lora_name == invalid_model
|
||||
|
||||
# Assert generate was *not* called due to the failure
|
||||
mock_engine.generate.assert_not_called()
|
||||
|
||||
# Assert the correct error response
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST.value
|
||||
assert invalid_model in response.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_completion_flag_not_set(mock_serving_setup):
|
||||
mock_engine, serving_completion = mock_serving_setup
|
||||
|
||||
lora_model_name = "test-lora"
|
||||
req_found = CompletionRequest(
|
||||
model=lora_model_name,
|
||||
prompt="Generate with LoRA",
|
||||
)
|
||||
|
||||
await serving_completion.create_completion(req_found)
|
||||
|
||||
mock_engine.add_lora.assert_not_called()
|
||||
mock_engine.generate.assert_not_called()
|
||||
@@ -396,7 +396,6 @@ def server_with_store(default_server_args):
|
||||
default_server_args,
|
||||
env_dict={
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
class TestWorkerExtension:
|
||||
def get_model_name(self) -> str:
|
||||
"""Test non-pydantic return type."""
|
||||
return MODEL_NAME
|
||||
|
||||
def echo_args_kwargs(self, *args, **kwargs) -> dict[str, Any]:
|
||||
"""Echo back both args and kwargs."""
|
||||
return dict(
|
||||
args=list(args),
|
||||
kwargs=kwargs,
|
||||
total_items=len(args) + len(kwargs),
|
||||
)
|
||||
|
||||
def return_none(self, *args, **kwargs) -> None:
|
||||
"""Test method that does not return anything."""
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--worker-extension-cls",
|
||||
"tests.entrypoints.serve.dev.rpc.test_collective_rpc.TestWorkerExtension",
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_get_model_name(server):
|
||||
"""Test basic response."""
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"), json={"method": "get_model_name"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
assert "results" in results
|
||||
assert results["results"] == [MODEL_NAME]
|
||||
|
||||
|
||||
def test_return_none(server):
|
||||
"""Test return none."""
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"), json={"method": "return_none"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
assert results["results"] == [None]
|
||||
|
||||
|
||||
def test_echo_args_kwargs(server):
|
||||
"""Test args, kwargs, and dict response."""
|
||||
args = ["arg1", "arg2"]
|
||||
kwargs = {"key1": "value1", "key2": "value2"}
|
||||
response = requests.post(
|
||||
server.url_for("collective_rpc"),
|
||||
json={"method": "echo_args_kwargs", "args": args, "kwargs": kwargs},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()
|
||||
result = results["results"][0]
|
||||
assert result["args"] == args
|
||||
assert result["kwargs"] == kwargs
|
||||
assert result["total_items"] == len(args) + len(kwargs)
|
||||
@@ -70,19 +70,17 @@ class MockModelConfig:
|
||||
|
||||
|
||||
@pytest.fixture(params=get_args(SupportedTask))
|
||||
def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]:
|
||||
def task_routes(request) -> tuple[str, list[tuple[str, list[str]]]]:
|
||||
"""For each supported task, build an app with only that task's routers,
|
||||
extract all routes, and return the task name and routes."""
|
||||
task = request.param
|
||||
# Enable development mode to register all routes (including dev-only routes).
|
||||
monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1")
|
||||
|
||||
app = FastAPI()
|
||||
args = Namespace()
|
||||
app.state = Namespace()
|
||||
app.state.args = args
|
||||
|
||||
# Register routers for this specific task (development mode already enabled).
|
||||
# Register routers for this specific task.
|
||||
register_api_routers(
|
||||
args,
|
||||
app,
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from argparse import Namespace
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.tokenize.api_router import attach_router
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
task = "generate"
|
||||
runner_type = "generate"
|
||||
model = MODEL_NAME
|
||||
tokenizer = MODEL_NAME
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = MockHFConfig()
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
return ServingTokenization(
|
||||
models,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=(
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
)
|
||||
|
||||
request = TokenizeChatRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
request = TokenizeCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
class TestDetokenizeClientErrorResponses:
|
||||
"""Client-caused errors from /detokenize are 4xx, not 500.
|
||||
|
||||
/detokenize used to wrap ``create_detokenize`` in a blanket
|
||||
``except Exception`` that reported every failure as 500, while its
|
||||
sibling /tokenize let exceptions propagate to the global exception
|
||||
handlers, which map client-caused errors to 4xx and keep genuine
|
||||
server errors at 500. These tests pin the shared contract for both
|
||||
endpoints. See #52246 for the same fix in the Anthropic entrypoint.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _make_api_app(handler: MagicMock) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.state.serving_tokenization = handler
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
return app
|
||||
|
||||
def _post(self, handler: MagicMock, path: str = "/detokenize"):
|
||||
app = self._make_api_app(handler)
|
||||
body: dict[str, Any] = {"model": "test-model"}
|
||||
if path == "/detokenize":
|
||||
body["tokens"] = [1, 2, 3]
|
||||
else:
|
||||
body["prompt"] = "Hello"
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
return client.post(path, json=body)
|
||||
|
||||
def test_vllm_validation_error_returns_bad_request(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = VLLMValidationError(
|
||||
"invalid token ids", parameter="tokens"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "BadRequestError"
|
||||
assert error["param"] == "tokens"
|
||||
|
||||
def test_vllm_not_found_error_returns_not_found(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = VLLMNotFoundError(
|
||||
"LoRA adapter nonexistent-lora not found"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
assert response.json()["error"]["type"] == "NotFoundError"
|
||||
|
||||
def test_value_error_returns_bad_request(self):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = ValueError("bad input")
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
def test_overflow_error_returns_bad_request(self):
|
||||
"""Fast tokenizers raise OverflowError for out-of-range token ids.
|
||||
|
||||
This used to be special-cased into a RequestValidationError; the
|
||||
global OverflowError handler keeps the same 400 status code.
|
||||
"""
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = OverflowError(
|
||||
"out of range integral type conversion attempted"
|
||||
)
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
def test_server_error_still_returns_internal_server_error(self):
|
||||
"""Genuine server bugs keep the existing 500 behaviour."""
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
handler.create_detokenize.side_effect = RuntimeError("boom")
|
||||
|
||||
response = self._post(handler)
|
||||
|
||||
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
assert response.json()["error"]["type"] == "InternalServerError"
|
||||
|
||||
def test_tokenize_and_detokenize_agree_on_client_errors(self):
|
||||
"""The two sibling endpoints report the same client error alike."""
|
||||
statuses = {}
|
||||
for path, method in (
|
||||
("/tokenize", "create_tokenize"),
|
||||
("/detokenize", "create_detokenize"),
|
||||
):
|
||||
handler = MagicMock(spec=ServingTokenization)
|
||||
getattr(handler, method).side_effect = VLLMValidationError(
|
||||
"invalid input", parameter="model"
|
||||
)
|
||||
statuses[path] = self._post(handler, path=path).status_code
|
||||
|
||||
assert statuses["/tokenize"] == HTTPStatus.BAD_REQUEST
|
||||
assert statuses["/detokenize"] == HTTPStatus.BAD_REQUEST
|
||||
Reference in New Issue
Block a user