[Frontend][CI] Consolidate instrumentator entrypoints (#34123)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-02-10 07:30:19 +00:00
committed by GitHub
parent 8d48d0a9d9
commit dab1de9f38
16 changed files with 64 additions and 74 deletions
@@ -0,0 +1,250 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from http import HTTPStatus
from unittest.mock import AsyncMock, Mock
import openai
import pytest
import pytest_asyncio
import requests
from fastapi import Request
from vllm.v1.engine.exceptions import EngineDeadError
from vllm.version import __version__ as VLLM_VERSION
from ...utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def server_args(request: pytest.FixtureRequest) -> list[str]:
"""Provide extra arguments to the server via indirect parametrization
Usage:
>>> @pytest.mark.parametrize(
>>> "server_args",
>>> [
>>> ["--disable-frontend-multiprocessing"],
>>> [
>>> "--model=NousResearch/Hermes-3-Llama-3.1-70B",
>>> "--enable-auto-tool-choice",
>>> ],
>>> ],
>>> indirect=True,
>>> )
>>> def test_foo(server, client):
>>> ...
This will run `test_foo` twice with servers with:
- `--disable-frontend-multiprocessing`
- `--model=NousResearch/Hermes-3-Llama-3.1-70B --enable-auto-tool-choice`.
"""
if not hasattr(request, "param"):
return []
val = request.param
if isinstance(val, str):
return [val]
return request.param
@pytest.fixture(scope="module")
def server(server_args):
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
"--max-num-seqs",
"128",
*server_args,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.parametrize(
"server_args",
[
pytest.param([], id="default-frontend-multiprocessing"),
pytest.param(
["--disable-frontend-multiprocessing"],
id="disable-frontend-multiprocessing",
),
],
indirect=True,
)
@pytest.mark.asyncio
async def test_show_version(server: RemoteOpenAIServer):
response = requests.get(server.url_for("version"))
response.raise_for_status()
assert response.json() == {"version": VLLM_VERSION}
@pytest.mark.parametrize(
"server_args",
[
pytest.param([], id="default-frontend-multiprocessing"),
pytest.param(
["--disable-frontend-multiprocessing"],
id="disable-frontend-multiprocessing",
),
],
indirect=True,
)
@pytest.mark.asyncio
async def test_check_health(server: RemoteOpenAIServer):
response = requests.get(server.url_for("health"))
assert response.status_code == HTTPStatus.OK
@pytest.mark.parametrize(
"server_args",
[
pytest.param(
["--max-model-len", "10100"], id="default-frontend-multiprocessing"
),
pytest.param(
["--disable-frontend-multiprocessing", "--max-model-len", "10100"],
id="disable-frontend-multiprocessing",
),
],
indirect=True,
)
@pytest.mark.asyncio
async def test_request_cancellation(server: RemoteOpenAIServer):
# clunky test: send an ungodly amount of load in with short timeouts
# then ensure that it still responds quickly afterwards
chat_input = [{"role": "user", "content": "Write a long story"}]
client = server.get_async_client(timeout=0.5)
tasks = []
# Request about 2 million tokens
for _ in range(200):
task = asyncio.create_task(
client.chat.completions.create(
messages=chat_input,
model=MODEL_NAME,
max_tokens=10000,
extra_body={"min_tokens": 10000},
)
)
tasks.append(task)
done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
# Make sure all requests were sent to the server and timed out
# (We don't want to hide other errors like 400s that would invalidate this
# test)
assert len(pending) == 0
for d in done:
with pytest.raises(openai.APITimeoutError):
d.result()
# If the server had not cancelled all the other requests, then it would not
# be able to respond to this one within the timeout
client = server.get_async_client(timeout=5)
response = await client.chat.completions.create(
messages=chat_input, model=MODEL_NAME, max_tokens=10
)
assert len(response.choices) == 1
@pytest.mark.asyncio
async def test_request_wrong_content_type(server: RemoteOpenAIServer):
chat_input = [{"role": "user", "content": "Write a long story"}]
client = server.get_async_client()
with pytest.raises(openai.APIStatusError):
await client.chat.completions.create(
messages=chat_input,
model=MODEL_NAME,
max_tokens=10000,
extra_headers={"Content-Type": "application/x-www-form-urlencoded"},
)
@pytest.mark.parametrize(
"server_args",
[pytest.param(["--enable-server-load-tracking"], id="enable-server-load-tracking")],
indirect=True,
)
@pytest.mark.asyncio
async def test_server_load(server: RemoteOpenAIServer):
# Check initial server load
response = requests.get(server.url_for("load"))
assert response.status_code == HTTPStatus.OK
assert response.json().get("server_load") == 0
def make_long_completion_request():
return requests.post(
server.url_for("v1/completions"),
headers={"Content-Type": "application/json"},
json={
"prompt": "Give me a long story",
"max_tokens": 1000,
"temperature": 0,
},
)
# Start the completion request in a background thread.
completion_future = asyncio.create_task(
asyncio.to_thread(make_long_completion_request)
)
# Give a short delay to ensure the request has started.
await asyncio.sleep(0.1)
# Check server load while the completion request is running.
response = requests.get(server.url_for("load"))
assert response.status_code == HTTPStatus.OK
assert response.json().get("server_load") == 1
# Wait for the completion request to finish.
await completion_future
await asyncio.sleep(0.1)
# Check server load after the completion request has finished.
response = requests.get(server.url_for("load"))
assert response.status_code == HTTPStatus.OK
assert response.json().get("server_load") == 0
@pytest.mark.asyncio
async def test_health_check_engine_dead_error():
# Import the health function directly to test it in isolation
from vllm.entrypoints.serve.instrumentator.health import health
# Create a mock request that simulates what FastAPI would provide
mock_request = Mock(spec=Request)
mock_app_state = Mock()
mock_engine_client = AsyncMock()
mock_engine_client.check_health.side_effect = EngineDeadError()
mock_app_state.engine_client = mock_engine_client
mock_request.app.state = mock_app_state
# Test the health function directly with our mocked request
# This simulates what would happen if the engine dies
response = await health(mock_request)
# Assert that it returns 503 Service Unavailable
assert response.status_code == 503
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for middleware that's off by default and can be toggled through
server arguments, mainly --api-key and --enable-request-id-headers.
"""
from http import HTTPStatus
import pytest
import requests
from ...utils import RemoteOpenAIServer
# Use a small embeddings model for faster startup and smaller memory footprint.
# Since we are not testing any chat functionality,
# using a chat capable model is overkill.
MODEL_NAME = "intfloat/multilingual-e5-small"
@pytest.fixture(scope="module")
def server(request: pytest.FixtureRequest):
passed_params = []
if hasattr(request, "param"):
passed_params = request.param
if isinstance(passed_params, str):
passed_params = [passed_params]
args = [
"--runner",
"pooling",
# use half precision for speed and memory savings in CI environment
"--dtype",
"float16",
"--max-model-len",
"512",
"--enforce-eager",
"--max-num-seqs",
"2",
*passed_params,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_no_api_token(server: RemoteOpenAIServer):
response = requests.get(server.url_for("v1/models"))
assert response.status_code == HTTPStatus.OK
@pytest.mark.asyncio
async def test_no_request_id_header(server: RemoteOpenAIServer):
response = requests.get(server.url_for("health"))
assert "X-Request-Id" not in response.headers
@pytest.mark.parametrize(
"server",
[["--api-key", "test"]],
indirect=True,
)
@pytest.mark.asyncio
async def test_missing_api_token(server: RemoteOpenAIServer):
response = requests.get(server.url_for("v1/models"))
assert response.status_code == HTTPStatus.UNAUTHORIZED
@pytest.mark.parametrize(
"server",
[["--api-key", "test"]],
indirect=True,
)
@pytest.mark.asyncio
async def test_passed_api_token(server: RemoteOpenAIServer):
response = requests.get(
server.url_for("v1/models"), headers={"Authorization": "Bearer test"}
)
assert response.status_code == HTTPStatus.OK
@pytest.mark.parametrize(
"server",
[["--api-key", "test"]],
indirect=True,
)
@pytest.mark.asyncio
async def test_not_v1_api_token(server: RemoteOpenAIServer):
# Authorization check is skipped for any paths that
# don't start with /v1 (e.g. /v1/chat/completions).
response = requests.get(server.url_for("health"))
assert response.status_code == HTTPStatus.OK
@pytest.mark.parametrize(
"server",
["--enable-request-id-headers"],
indirect=True,
)
@pytest.mark.asyncio
async def test_enable_request_id_header(server: RemoteOpenAIServer):
response = requests.get(server.url_for("health"))
assert "X-Request-Id" in response.headers
assert len(response.headers.get("X-Request-Id", "")) == 32
@pytest.mark.parametrize(
"server",
["--enable-request-id-headers"],
indirect=True,
)
@pytest.mark.asyncio
async def test_custom_request_id_header(server: RemoteOpenAIServer):
response = requests.get(
server.url_for("health"), headers={"X-Request-Id": "Custom"}
)
assert "X-Request-Id" in response.headers
assert response.headers.get("X-Request-Id") == "Custom"
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import openai
import pytest
import pytest_asyncio
from ...utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def monkeypatch_module():
from _pytest.monkeypatch import MonkeyPatch
mpatch = MonkeyPatch()
yield mpatch
mpatch.undo()
@pytest.fixture(scope="module", params=[True])
def server(request, monkeypatch_module):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_chat_completion_with_orca_header(server: RemoteOpenAIServer):
messages = [
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "what is 1+1?"},
]
client = openai.OpenAI(
api_key="EMPTY",
base_url=f"http://localhost:{server.port}/v1",
default_headers={"endpoint-load-metrics-format": "TEXT"},
)
# 1. Use raw client to get response headers.
raw_client = client.with_raw_response
# 2. Make the API call using the raw_client
response_with_raw = raw_client.chat.completions.create(
model=MODEL_NAME,
messages=messages,
extra_headers={"endpoint-load-metrics-format": "TEXT"},
)
# 3. Access the raw httpx.Response object
raw_http_response = response_with_raw.http_response
# 4. Get the headers from the httpx.Response object
response_headers = raw_http_response.headers
assert "endpoint-load-metrics" in response_headers
@pytest.mark.asyncio
async def test_completion_with_orca_header(client: openai.AsyncOpenAI):
# 1. Use raw client to get response headers.
raw_client = client.with_raw_response
# 2. Make the API call using the raw_client
completion = await raw_client.completions.create(
model=MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
extra_headers={"endpoint-load-metrics-format": "JSON"},
)
# 3. Access the raw httpx.Response object
raw_http_response = completion.http_response
# 4. Get the headers from the httpx.Response object
response_headers = raw_http_response.headers
assert "endpoint-load-metrics" in response_headers
@pytest.mark.asyncio
async def test_single_completion(client: openai.AsyncOpenAI):
completion = await client.completions.create(
model=MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
extra_headers={"endpoint-load-metrics-format": "JSON"},
temperature=0.0,
)
assert completion.id is not None
assert completion.choices is not None and len(completion.choices) == 1
choice = completion.choices[0]
assert len(choice.text) >= 5
assert choice.finish_reason == "length"
# When using Qwen3-0.6B, prompt tokens=[9707, 11, 847, 829, 374]
assert completion.usage == openai.types.CompletionUsage(
completion_tokens=5, prompt_tokens=5, total_tokens=10
)
# test using token IDs
completion = await client.completions.create(
model=MODEL_NAME,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
)
assert len(completion.choices[0].text) >= 1
assert completion.choices[0].prompt_logprobs is None
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "meta-llama/Llama-3.2-1B"
def test_sleep_mode():
# dtype, max-len etc set so that this can run in CI
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--max-num-seqs",
"128",
"--enable-sleep-mode",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
) as remote_server:
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 0
assert weights_offloaded == 1
assert discard_all == 0
response = requests.post(remote_server.url_for("wake_up"))
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
# test wake up with tags
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["weights"]}
)
assert response.status_code == 200
# is sleeping should be false after waking up any part of the engine
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["kv_cache"]}
)
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
def _get_sleep_metrics_from_api(response: requests.Response):
"""Return (awake, weights_offloaded, discard_all)"""
awake, weights_offloaded, discard_all = None, None, None
for family in text_string_to_metric_families(response.text):
if family.name == "vllm:engine_sleep_state":
for sample in family.samples:
if sample.name == "vllm:engine_sleep_state":
for label_name, label_value in sample.labels.items():
if label_value == "awake":
awake = sample.value
elif label_value == "weights_offloaded":
weights_offloaded = sample.value
elif label_value == "discard_all":
discard_all = sample.value
assert awake is not None
assert weights_offloaded is not None
assert discard_all is not None
return awake, weights_offloaded, discard_all