[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
from fastapi import Request
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm import envs
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args(request: pytest.FixtureRequest) -> list[str]:
|
||||
"""Provide extra arguments to the server via indirect parametrization.
|
||||
|
||||
Usage:
|
||||
|
||||
>>> @pytest.mark.parametrize(
|
||||
>>> "server_args",
|
||||
>>> [
|
||||
>>> ["--max-model-len", "10100"],
|
||||
>>> [
|
||||
>>> "--model=NousResearch/Hermes-3-Llama-3.1-70B",
|
||||
>>> "--enable-auto-tool-choice",
|
||||
>>> ],
|
||||
>>> ],
|
||||
>>> indirect=True,
|
||||
>>> )
|
||||
>>> def test_foo(server, client):
|
||||
>>> ...
|
||||
|
||||
This will run `test_foo` twice with servers with:
|
||||
- `--max-model-len 10100`
|
||||
- `--model=NousResearch/Hermes-3-Llama-3.1-70B --enable-auto-tool-choice`.
|
||||
|
||||
"""
|
||||
if not hasattr(request, "param"):
|
||||
return []
|
||||
|
||||
val = request.param
|
||||
|
||||
if isinstance(val, str):
|
||||
return [val]
|
||||
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args):
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
*server_args,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_show_version(server: RemoteOpenAIServer):
|
||||
response = requests.get(server.url_for("version"))
|
||||
response.raise_for_status()
|
||||
|
||||
payload = response.json()
|
||||
assert payload["version"] == VLLM_VERSION
|
||||
if envs.VLLM_USE_RUST_FRONTEND:
|
||||
assert payload["rust_frontend_version"] == VLLM_VERSION
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_health(server: RemoteOpenAIServer):
|
||||
response = requests.get(server.url_for("health"))
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"server_args",
|
||||
[
|
||||
pytest.param(["--max-model-len", "10100"]),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_cancellation(server: RemoteOpenAIServer):
|
||||
# clunky test: send an ungodly amount of load in with short timeouts
|
||||
# then ensure that it still responds quickly afterwards
|
||||
|
||||
chat_input = [{"role": "user", "content": "Write a long story"}]
|
||||
client = server.get_async_client(timeout=0.5)
|
||||
tasks = []
|
||||
# Request about 2 million tokens
|
||||
for _ in range(200):
|
||||
task = asyncio.create_task(
|
||||
client.chat.completions.create(
|
||||
messages=chat_input,
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10000,
|
||||
extra_body={"min_tokens": 10000},
|
||||
temperature=0.0,
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
|
||||
|
||||
# Make sure all requests were sent to the server and timed out
|
||||
# (We don't want to hide other errors like 400s that would invalidate this
|
||||
# test)
|
||||
assert len(pending) == 0
|
||||
for d in done:
|
||||
with pytest.raises(openai.APITimeoutError):
|
||||
d.result()
|
||||
|
||||
# If the server had not cancelled all the other requests, then it would not
|
||||
# be able to respond to this one within the timeout
|
||||
client = server.get_async_client(timeout=5)
|
||||
response = await client.chat.completions.create(
|
||||
messages=chat_input, model=MODEL_NAME, max_tokens=10, temperature=0.0
|
||||
)
|
||||
|
||||
assert len(response.choices) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_wrong_content_type(server: RemoteOpenAIServer):
|
||||
chat_input = [{"role": "user", "content": "Write a long story"}]
|
||||
client = server.get_async_client()
|
||||
|
||||
with pytest.raises(openai.APIStatusError):
|
||||
await client.chat.completions.create(
|
||||
messages=chat_input,
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10000,
|
||||
extra_headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"server_args",
|
||||
[pytest.param(["--enable-server-load-tracking"], id="enable-server-load-tracking")],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_load(server: RemoteOpenAIServer):
|
||||
# Check initial server load
|
||||
response = requests.get(server.url_for("load"))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json().get("server_load") == 0
|
||||
|
||||
def make_long_completion_request():
|
||||
return requests.post(
|
||||
server.url_for("v1/completions"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
json={
|
||||
"prompt": "Give me a long story",
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0,
|
||||
},
|
||||
)
|
||||
|
||||
# Start the completion request in a background thread.
|
||||
completion_future = asyncio.create_task(
|
||||
asyncio.to_thread(make_long_completion_request)
|
||||
)
|
||||
|
||||
# Give a short delay to ensure the request has started.
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Check server load while the completion request is running.
|
||||
response = requests.get(server.url_for("load"))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json().get("server_load") == 1
|
||||
|
||||
# Wait for the completion request to finish.
|
||||
await completion_future
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Check server load after the completion request has finished.
|
||||
response = requests.get(server.url_for("load"))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json().get("server_load") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_engine_dead_error():
|
||||
# Import the health function directly to test it in isolation
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
|
||||
# Create a mock request that simulates what FastAPI would provide
|
||||
mock_request = Mock(spec=Request)
|
||||
mock_app_state = Mock()
|
||||
mock_engine_client = AsyncMock()
|
||||
mock_engine_client.check_health.side_effect = EngineDeadError()
|
||||
mock_app_state.engine_client = mock_engine_client
|
||||
mock_request.app.state = mock_app_state
|
||||
|
||||
# Test the health function directly with our mocked request
|
||||
# This simulates what would happen if the engine dies
|
||||
response = await health(mock_request)
|
||||
|
||||
# Assert that it returns 503 Service Unavailable
|
||||
assert response.status_code == 503
|
||||
@@ -1,500 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.conftest import LocalAssetServer
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm import version
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
MODELS = {
|
||||
"text": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||||
"multimodal": "HuggingFaceTB/SmolVLM-256M-Instruct",
|
||||
}
|
||||
PREV_MINOR_VERSION = version._prev_minor_version()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=list(MODELS.keys()))
|
||||
def model_key(request):
|
||||
yield request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
scope="module",
|
||||
params=[
|
||||
"",
|
||||
"--enable-chunked-prefill",
|
||||
f"--show-hidden-metrics-for-version={PREV_MINOR_VERSION}",
|
||||
],
|
||||
)
|
||||
def server(model_key, default_server_args, request):
|
||||
if request.param:
|
||||
default_server_args.append(request.param)
|
||||
|
||||
model_name = MODELS[model_key]
|
||||
with RemoteOpenAIServer(model_name, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as cl:
|
||||
yield cl
|
||||
|
||||
|
||||
_PROMPT = "Hello my name is Robert and I love magic"
|
||||
|
||||
|
||||
def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: int):
|
||||
num_prompt_tokens = len(prompt_ids)
|
||||
|
||||
# {metric_family: [(suffix, expected_value)]}
|
||||
return {
|
||||
"vllm:time_to_first_token_seconds": [("_count", num_requests)],
|
||||
"vllm:inter_token_latency_seconds": [
|
||||
("_count", num_requests * (max_tokens - 1))
|
||||
],
|
||||
"vllm:e2e_request_latency_seconds": [("_count", num_requests)],
|
||||
"vllm:request_queue_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_inference_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_prefill_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_decode_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_num_preemptions": [("_count", num_requests)],
|
||||
"vllm:request_prompt_tokens": [
|
||||
("_sum", num_requests * num_prompt_tokens),
|
||||
("_count", num_requests),
|
||||
],
|
||||
"vllm:request_generation_tokens": [
|
||||
("_sum", num_requests * max_tokens),
|
||||
("_count", num_requests),
|
||||
],
|
||||
"vllm:request_params_n": [("_count", num_requests)],
|
||||
"vllm:request_params_max_tokens": [
|
||||
("_sum", num_requests * max_tokens),
|
||||
("_count", num_requests),
|
||||
],
|
||||
"vllm:iteration_tokens_total": [
|
||||
(
|
||||
"_sum",
|
||||
num_requests * (num_prompt_tokens + max_tokens),
|
||||
),
|
||||
("_count", num_requests * max_tokens),
|
||||
],
|
||||
"vllm:prompt_tokens": [("_total", num_requests * num_prompt_tokens)],
|
||||
"vllm:generation_tokens": [("_total", num_requests * max_tokens)],
|
||||
"vllm:request_success": [("_total", num_requests)],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_counts(
|
||||
server: RemoteOpenAIServer,
|
||||
client: openai.AsyncClient,
|
||||
model_key: str,
|
||||
):
|
||||
if model_key == "multimodal":
|
||||
pytest.skip("Unnecessary test")
|
||||
|
||||
model_name = MODELS[model_key]
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
prompt_ids = tokenizer.encode(_PROMPT)
|
||||
num_requests = 10
|
||||
max_tokens = 10
|
||||
|
||||
for _ in range(num_requests):
|
||||
# sending a request triggers the metrics to be logged.
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_ids,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
response = requests.get(server.url_for("metrics"))
|
||||
print(response.text)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# Loop over all expected metric_families
|
||||
expected_values = _get_expected_values(num_requests, prompt_ids, max_tokens)
|
||||
for metric_family, suffix_values_list in expected_values.items():
|
||||
if metric_family not in EXPECTED_METRICS_V1 or (
|
||||
not server.show_hidden_metrics
|
||||
and metric_family in HIDDEN_DEPRECATED_METRICS
|
||||
):
|
||||
continue
|
||||
|
||||
found_metric = False
|
||||
|
||||
# Check to see if the metric_family is found in the prom endpoint.
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
if family.name == metric_family:
|
||||
found_metric = True
|
||||
|
||||
# Check that each suffix is found in the prom endpoint.
|
||||
for suffix, expected_value in suffix_values_list:
|
||||
metric_name_w_suffix = f"{metric_family}{suffix}"
|
||||
found_suffix = False
|
||||
|
||||
for sample in family.samples:
|
||||
if sample.name == metric_name_w_suffix:
|
||||
found_suffix = True
|
||||
|
||||
# For each suffix, value sure the value matches
|
||||
# what we expect.
|
||||
assert sample.value == expected_value, (
|
||||
f"{metric_name_w_suffix} expected value of "
|
||||
f"{expected_value} did not match found value "
|
||||
f"{sample.value}"
|
||||
)
|
||||
break
|
||||
assert found_suffix, (
|
||||
f"Did not find {metric_name_w_suffix} in prom endpoint"
|
||||
)
|
||||
break
|
||||
|
||||
assert found_metric, f"Did not find {metric_family} in prom endpoint"
|
||||
|
||||
|
||||
EXPECTED_METRICS_V1 = [
|
||||
"vllm:num_requests_running",
|
||||
"vllm:num_requests_waiting",
|
||||
"vllm:num_requests_waiting_by_reason",
|
||||
"vllm:kv_cache_usage_perc",
|
||||
"vllm:prefix_cache_queries",
|
||||
"vllm:prefix_cache_hits",
|
||||
"vllm:num_preemptions_total",
|
||||
"vllm:admission_rejections_total",
|
||||
"vllm:prompt_tokens_total",
|
||||
"vllm:generation_tokens_total",
|
||||
"vllm:iteration_tokens_total",
|
||||
"vllm:cache_config_info",
|
||||
"vllm:request_success_total",
|
||||
"vllm:request_prompt_tokens_sum",
|
||||
"vllm:request_prompt_tokens_bucket",
|
||||
"vllm:request_prompt_tokens_count",
|
||||
"vllm:request_generation_tokens_sum",
|
||||
"vllm:request_generation_tokens_bucket",
|
||||
"vllm:request_generation_tokens_count",
|
||||
"vllm:request_params_n_sum",
|
||||
"vllm:request_params_n_bucket",
|
||||
"vllm:request_params_n_count",
|
||||
"vllm:request_params_max_tokens_sum",
|
||||
"vllm:request_params_max_tokens_bucket",
|
||||
"vllm:request_params_max_tokens_count",
|
||||
"vllm:time_to_first_token_seconds_sum",
|
||||
"vllm:time_to_first_token_seconds_bucket",
|
||||
"vllm:time_to_first_token_seconds_count",
|
||||
"vllm:inter_token_latency_seconds_sum",
|
||||
"vllm:inter_token_latency_seconds_bucket",
|
||||
"vllm:inter_token_latency_seconds_count",
|
||||
"vllm:e2e_request_latency_seconds_sum",
|
||||
"vllm:e2e_request_latency_seconds_bucket",
|
||||
"vllm:e2e_request_latency_seconds_count",
|
||||
"vllm:request_queue_time_seconds_sum",
|
||||
"vllm:request_queue_time_seconds_bucket",
|
||||
"vllm:request_queue_time_seconds_count",
|
||||
"vllm:request_inference_time_seconds_sum",
|
||||
"vllm:request_inference_time_seconds_bucket",
|
||||
"vllm:request_inference_time_seconds_count",
|
||||
"vllm:request_prefill_time_seconds_sum",
|
||||
"vllm:request_prefill_time_seconds_bucket",
|
||||
"vllm:request_prefill_time_seconds_count",
|
||||
"vllm:request_decode_time_seconds_sum",
|
||||
"vllm:request_decode_time_seconds_bucket",
|
||||
"vllm:request_decode_time_seconds_count",
|
||||
"vllm:request_num_preemptions_sum",
|
||||
"vllm:request_num_preemptions_bucket",
|
||||
"vllm:request_num_preemptions_count",
|
||||
]
|
||||
|
||||
EXPECTED_METRICS_MM = [
|
||||
"vllm:mm_cache_queries",
|
||||
"vllm:mm_cache_hits",
|
||||
]
|
||||
|
||||
HIDDEN_DEPRECATED_METRICS: list[str] = []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_exist(
|
||||
local_asset_server: LocalAssetServer,
|
||||
server: RemoteOpenAIServer,
|
||||
client: openai.AsyncClient,
|
||||
model_key: str,
|
||||
):
|
||||
model_name = MODELS[model_key]
|
||||
|
||||
# sending a request triggers the metrics to be logged.
|
||||
if model_key == "text":
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
else:
|
||||
# https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": local_asset_server.url_for(
|
||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
),
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
response = requests.get(server.url_for("metrics"))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
expected_metrics = EXPECTED_METRICS_V1
|
||||
if model_key == "multimodal":
|
||||
# NOTE: Don't use in-place assignment
|
||||
expected_metrics = expected_metrics + EXPECTED_METRICS_MM
|
||||
|
||||
for metric in expected_metrics:
|
||||
if metric in HIDDEN_DEPRECATED_METRICS and not server.show_hidden_metrics:
|
||||
continue
|
||||
assert metric in response.text
|
||||
|
||||
cache_config_samples = [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
if family.name == "vllm:cache_config_info"
|
||||
for sample in family.samples
|
||||
]
|
||||
assert cache_config_samples
|
||||
for sample in cache_config_samples:
|
||||
assert sample.labels.get("kv_cache_size_tokens") not in (None, "None", "")
|
||||
assert sample.labels.get("kv_cache_max_concurrency") not in (None, "None", "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_metrics_reset(
|
||||
server: RemoteOpenAIServer,
|
||||
client: openai.AsyncClient,
|
||||
model_key: str,
|
||||
):
|
||||
model_name = MODELS[model_key]
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
prompt_ids = tokenizer.encode(_PROMPT)
|
||||
|
||||
running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
|
||||
server,
|
||||
)
|
||||
|
||||
# Expect no running requests or kvcache usage
|
||||
assert running_requests == 0
|
||||
assert waiting_requests == 0
|
||||
assert kv_cache_usage == 0.0
|
||||
|
||||
# Start some long-running requests that we can abort
|
||||
tasks = []
|
||||
for _ in range(3):
|
||||
task = asyncio.create_task(
|
||||
client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_ids,
|
||||
max_tokens=500, # Long generation to give time to abort
|
||||
temperature=0.0,
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# Poll until we see running requests rather than using a fixed sleep,
|
||||
# since generation speed varies across hardware.
|
||||
try:
|
||||
await _poll_until(
|
||||
lambda: _get_running_metrics_from_api(server)[0] > 0,
|
||||
timeout=10.0,
|
||||
interval=0.1,
|
||||
description="running_requests > 0",
|
||||
)
|
||||
except TimeoutError:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
pytest.fail("Requests never appeared as running in metrics")
|
||||
|
||||
# Check that we have running requests
|
||||
running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
|
||||
server,
|
||||
)
|
||||
|
||||
# Expect running requests and kvcache usage
|
||||
assert running_requests > 0
|
||||
assert kv_cache_usage > 0
|
||||
|
||||
# Cancel all tasks to abort the requests
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Poll until metrics reset rather than using a fixed sleep
|
||||
await _poll_until(
|
||||
lambda: _get_running_metrics_from_api(server) == (0, 0, 0),
|
||||
timeout=10.0,
|
||||
interval=0.2,
|
||||
description="gauge metrics back to zero",
|
||||
)
|
||||
|
||||
# Verify running and waiting requests counts and KV cache usage are zero
|
||||
running_requests_after, waiting_requests_after, kv_cache_usage_after = (
|
||||
_get_running_metrics_from_api(server)
|
||||
)
|
||||
|
||||
assert running_requests_after == 0, (
|
||||
f"Expected 0 running requests after abort, got {running_requests_after}"
|
||||
)
|
||||
assert waiting_requests_after == 0, (
|
||||
f"Expected 0 waiting requests after abort, got {waiting_requests_after}"
|
||||
)
|
||||
assert kv_cache_usage_after == 0, (
|
||||
f"Expected 0% KV cache usage after abort, got {kv_cache_usage_after}"
|
||||
)
|
||||
|
||||
|
||||
async def _poll_until(
|
||||
predicate, *, timeout: float, interval: float = 0.5, description: str = "condition"
|
||||
):
|
||||
"""Poll until predicate() returns True, or raise TimeoutError."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if predicate():
|
||||
return
|
||||
await asyncio.sleep(interval)
|
||||
raise TimeoutError(f"Timed out after {timeout}s waiting for: {description}")
|
||||
|
||||
|
||||
def _get_running_metrics_from_api(server: RemoteOpenAIServer):
|
||||
"""Return (running_count, waiting_count, kv_cache_usage)."""
|
||||
response = requests.get(server.url_for("metrics"))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# Verify running and waiting requests counts and KV cache usage are zero
|
||||
running_requests, waiting_requests, kv_cache_usage = None, None, None
|
||||
|
||||
kv_cache_usage_metric = "vllm:kv_cache_usage_perc"
|
||||
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
if family.name == "vllm:num_requests_running":
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:num_requests_running":
|
||||
running_requests = sample.value
|
||||
break
|
||||
elif family.name == "vllm:num_requests_waiting":
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:num_requests_waiting":
|
||||
waiting_requests = sample.value
|
||||
break
|
||||
elif family.name == kv_cache_usage_metric:
|
||||
for sample in family.samples:
|
||||
if sample.name == kv_cache_usage_metric:
|
||||
kv_cache_usage = sample.value
|
||||
break
|
||||
|
||||
assert running_requests is not None
|
||||
assert waiting_requests is not None
|
||||
assert kv_cache_usage is not None
|
||||
|
||||
return running_requests, waiting_requests, kv_cache_usage
|
||||
|
||||
|
||||
def test_metrics_exist_run_batch():
|
||||
input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}""" # noqa: E501
|
||||
|
||||
base_url = "0.0.0.0"
|
||||
port = str(get_open_port())
|
||||
server_url = f"http://{base_url}:{port}"
|
||||
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(input_batch)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.launchers.run_batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
"intfloat/multilingual-e5-small",
|
||||
"--enable-metrics",
|
||||
"--host",
|
||||
base_url,
|
||||
"--port",
|
||||
port,
|
||||
],
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
def is_server_up(url):
|
||||
try:
|
||||
response = requests.get(url)
|
||||
return response.status_code == 200
|
||||
except requests.ConnectionError:
|
||||
return False
|
||||
|
||||
start = time.time()
|
||||
timeout = 120
|
||||
while not is_server_up(server_url):
|
||||
if proc.poll() is not None:
|
||||
pytest.fail(
|
||||
f"Batch process exited early with returncode={proc.returncode}"
|
||||
)
|
||||
if time.time() - start > timeout:
|
||||
pytest.fail("Batch server did not start within timeout")
|
||||
time.sleep(1)
|
||||
|
||||
response = requests.get(server_url + "/metrics")
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
finally:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm import envs
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with TemporaryDirectory() as tmpdir:
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--uds",
|
||||
f"{tmpdir}/vllm.sock",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_show_version(server: RemoteOpenAIServer):
|
||||
transport = httpx.HTTPTransport(uds=server.uds)
|
||||
client = httpx.Client(transport=transport)
|
||||
response = client.get(server.url_for("version"))
|
||||
response.raise_for_status()
|
||||
|
||||
payload = response.json()
|
||||
assert payload["version"] == VLLM_VERSION
|
||||
if envs.VLLM_USE_RUST_FRONTEND:
|
||||
assert payload["rust_frontend_version"] == VLLM_VERSION
|
||||
@@ -1,334 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enable-tokenizer-info-endpoint",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer_name(model_name: str):
|
||||
return model_name
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_tokenize_completions(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
|
||||
|
||||
for add_special in [False, True]:
|
||||
prompt = "vllm1 This is a test prompt."
|
||||
tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("tokenize"),
|
||||
json={
|
||||
"add_special_tokens": add_special,
|
||||
"model": model_name,
|
||||
"prompt": prompt,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
assert result["tokens"] == tokens
|
||||
assert result["count"] == len(tokens)
|
||||
assert result["max_model_len"] == 8192
|
||||
assert result["token_strs"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_tokenize_chat(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
|
||||
|
||||
for add_generation in [False, True]:
|
||||
for add_special in [False, True]:
|
||||
conversation = [
|
||||
{"role": "user", "content": "Hi there!"},
|
||||
{"role": "assistant", "content": "Nice to meet you!"},
|
||||
{"role": "user", "content": "Can I ask a question? vllm1"},
|
||||
]
|
||||
for continue_final in [False, True]:
|
||||
if add_generation and continue_final:
|
||||
continue
|
||||
if continue_final:
|
||||
conversation.append({"role": "assistant", "content": "Sure,"})
|
||||
|
||||
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
|
||||
add_generation_prompt=add_generation,
|
||||
continue_final_message=continue_final,
|
||||
conversation=conversation,
|
||||
tokenize=False,
|
||||
)
|
||||
tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("tokenize"),
|
||||
json={
|
||||
"add_generation_prompt": add_generation,
|
||||
"continue_final_message": continue_final,
|
||||
"add_special_tokens": add_special,
|
||||
"messages": conversation,
|
||||
"model": model_name,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
assert result["tokens"] == tokens
|
||||
assert result["count"] == len(tokens)
|
||||
assert result["max_model_len"] == 8192
|
||||
assert result["token_strs"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_tokenize_chat_with_tools(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
|
||||
|
||||
for add_generation in [False, True]:
|
||||
for add_special in [False, True]:
|
||||
conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Paris today?",
|
||||
}
|
||||
]
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
for continue_final in [False, True]:
|
||||
if add_generation and continue_final:
|
||||
continue
|
||||
if continue_final:
|
||||
conversation.append({"role": "assistant", "content": "Sure,"})
|
||||
|
||||
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
|
||||
add_generation_prompt=add_generation,
|
||||
continue_final_message=continue_final,
|
||||
conversation=conversation,
|
||||
tools=tools,
|
||||
tokenize=False,
|
||||
)
|
||||
tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("tokenize"),
|
||||
json={
|
||||
"add_generation_prompt": add_generation,
|
||||
"continue_final_message": continue_final,
|
||||
"add_special_tokens": add_special,
|
||||
"messages": conversation,
|
||||
"model": model_name,
|
||||
"tools": tools,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
assert result["tokens"] == tokens
|
||||
assert result["count"] == len(tokens)
|
||||
assert result["max_model_len"] == 8192
|
||||
assert result["token_strs"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_tokenize_with_return_token_strs(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
|
||||
|
||||
prompt = "This is a token_strs test prompt! vllm1"
|
||||
response = requests.post(
|
||||
server.url_for("tokenize"),
|
||||
json={"prompt": prompt, "model": model_name, "return_token_strs": True},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
tokens = tokenizer.encode(prompt, add_special_tokens=True)
|
||||
tokens_str = tokenizer.convert_ids_to_tokens(tokens)
|
||||
|
||||
result = response.json()
|
||||
assert result["tokens"] == tokens
|
||||
assert result["count"] == len(tokens)
|
||||
assert result["max_model_len"] == 8192
|
||||
assert result["token_strs"] == tokens_str
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_detokenize(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
|
||||
|
||||
prompt = "This is a test prompt. vllm1"
|
||||
tokens = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("detokenize"), json={"model": model_name, "tokens": tokens}
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
assert response.json() == {"prompt": prompt}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,tokenizer_name",
|
||||
[(MODEL_NAME, MODEL_NAME)],
|
||||
indirect=["tokenizer_name"],
|
||||
)
|
||||
async def test_tokenizer_info_basic(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
tokenizer_name: str,
|
||||
):
|
||||
"""Test basic tokenizer info endpoint functionality."""
|
||||
response = requests.get(server.url_for("tokenizer_info"))
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
assert "tokenizer_class" in result
|
||||
assert isinstance(result["tokenizer_class"], str)
|
||||
assert result["tokenizer_class"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenizer_info_schema(server: RemoteOpenAIServer):
|
||||
"""Test that the response matches expected schema types."""
|
||||
response = requests.get(server.url_for("tokenizer_info"))
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
field_types = {
|
||||
"add_bos_token": bool,
|
||||
"add_prefix_space": bool,
|
||||
"clean_up_tokenization_spaces": bool,
|
||||
"split_special_tokens": bool,
|
||||
"bos_token": str,
|
||||
"eos_token": str,
|
||||
"pad_token": str,
|
||||
"unk_token": str,
|
||||
"chat_template": str,
|
||||
"errors": str,
|
||||
"model_max_length": int,
|
||||
"additional_special_tokens": list,
|
||||
"added_tokens_decoder": dict,
|
||||
}
|
||||
for field, expected_type in field_types.items():
|
||||
if field in result and result[field] is not None:
|
||||
assert isinstance(result[field], expected_type), (
|
||||
f"{field} should be {expected_type.__name__}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenizer_info_consistency_with_tokenize(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that tokenizer info is consistent with tokenization endpoint."""
|
||||
info_response = requests.get(server.url_for("tokenizer_info"))
|
||||
info_response.raise_for_status()
|
||||
info = info_response.json()
|
||||
tokenize_response = requests.post(
|
||||
server.url_for("tokenize"),
|
||||
json={"model": MODEL_NAME, "prompt": "Hello world!"},
|
||||
)
|
||||
tokenize_response.raise_for_status()
|
||||
tokenize_result = tokenize_response.json()
|
||||
info_max_len = info.get("model_max_length")
|
||||
tokenize_max_len = tokenize_result.get("max_model_len")
|
||||
if info_max_len and tokenize_max_len:
|
||||
assert info_max_len >= tokenize_max_len, (
|
||||
"Info max length should be >= tokenize max length"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenizer_info_chat_template(server: RemoteOpenAIServer):
|
||||
"""Test chat template is properly included."""
|
||||
response = requests.get(server.url_for("tokenizer_info"))
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
chat_template = result.get("chat_template")
|
||||
if chat_template:
|
||||
assert isinstance(chat_template, str), "Chat template should be a string"
|
||||
assert chat_template.strip(), "Chat template should not be empty"
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``system_fingerprint`` construction."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils import fingerprint as fp
|
||||
|
||||
|
||||
def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
|
||||
c = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=tp,
|
||||
pipeline_parallel_size=pp,
|
||||
data_parallel_size=dp,
|
||||
enable_expert_parallel=ep,
|
||||
)
|
||||
)
|
||||
c.compute_hash = lambda: digest
|
||||
return c
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset():
|
||||
fp.set_default_fingerprint_mode("full")
|
||||
yield
|
||||
fp.set_default_fingerprint_mode("full")
|
||||
|
||||
|
||||
def test_four_modes_produce_expected_shapes():
|
||||
from vllm import __version__ as v
|
||||
|
||||
cfg = _cfg(tp=8, ep=True)
|
||||
|
||||
assert fp.build_system_fingerprint(cfg, "full") == (f"vllm-{v}-tp8-ep-a3b21f94")
|
||||
assert fp.build_system_fingerprint(cfg, "hash") == f"vllm-{v}-a3b21f94"
|
||||
assert fp.build_system_fingerprint(cfg, "custom", "my-fp") == "my-fp"
|
||||
assert fp.build_system_fingerprint(cfg, "none") is None
|
||||
|
||||
|
||||
def test_full_mode_emits_only_non_trivial_parallelism():
|
||||
from vllm import __version__ as v
|
||||
|
||||
# Single-GPU: nothing between version and hash.
|
||||
assert fp.build_system_fingerprint(_cfg(), "full") == f"vllm-{v}-a3b21f94"
|
||||
# All parallelism axes.
|
||||
assert (
|
||||
fp.build_system_fingerprint(_cfg(tp=8, pp=2, dp=4, ep=True), "full")
|
||||
== f"vllm-{v}-tp8-pp2-dp4-ep-a3b21f94"
|
||||
)
|
||||
|
||||
|
||||
def test_get_respects_set_default():
|
||||
cfg = _cfg(tp=8)
|
||||
full = fp.get_system_fingerprint(cfg)
|
||||
assert full == fp.get_system_fingerprint(cfg)
|
||||
|
||||
fp.set_default_fingerprint_mode("hash")
|
||||
hashed = fp.get_system_fingerprint(cfg)
|
||||
assert hashed is not None
|
||||
assert hashed != full
|
||||
assert "tp8" not in hashed
|
||||
|
||||
fp.set_default_fingerprint_mode("custom", "deploy-42")
|
||||
assert fp.get_system_fingerprint(cfg) == "deploy-42"
|
||||
|
||||
fp.set_default_fingerprint_mode("none")
|
||||
assert fp.get_system_fingerprint(cfg) is None
|
||||
|
||||
|
||||
def test_compute_hash_failure_does_not_raise():
|
||||
cfg = _cfg()
|
||||
cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
|
||||
full_fp = fp.build_system_fingerprint(cfg, "full")
|
||||
assert full_fp is not None and full_fp.endswith("-nohash")
|
||||
hash_fp = fp.build_system_fingerprint(cfg, "hash")
|
||||
assert hash_fp is not None and hash_fp.endswith("-nohash")
|
||||
Reference in New Issue
Block a user