add --insecure arg to the vllm bench to skip TLS (#34026)
Signed-off-by: Fan Yang <yan9fan@meta.com> Co-authored-by: Fan Yang <yan9fan@meta.com>
This commit is contained in:
@@ -26,6 +26,7 @@ import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import ssl
|
||||
import time
|
||||
import uuid
|
||||
import warnings
|
||||
@@ -60,11 +61,14 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a
|
||||
|
||||
|
||||
async def get_first_model_from_server(
|
||||
base_url: str, headers: dict | None = None
|
||||
base_url: str,
|
||||
headers: dict | None = None,
|
||||
ssl_context: ssl.SSLContext | bool | None = None,
|
||||
) -> tuple[str, str]:
|
||||
"""Fetch the first model from the server's /v1/models endpoint."""
|
||||
models_url = f"{base_url}/v1/models"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
connector = aiohttp.TCPConnector(ssl=ssl_context)
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
try:
|
||||
async with session.get(models_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
@@ -619,6 +623,7 @@ async def benchmark(
|
||||
ramp_up_start_rps: int | None = None,
|
||||
ramp_up_end_rps: int | None = None,
|
||||
ready_check_timeout_sec: int = 600,
|
||||
ssl_context: ssl.SSLContext | bool | None = None,
|
||||
):
|
||||
try:
|
||||
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
|
||||
@@ -626,6 +631,8 @@ async def benchmark(
|
||||
raise ValueError(f"Unknown backend: {endpoint_type}") from None
|
||||
|
||||
# Reuses connections across requests to reduce TLS handshake overhead.
|
||||
# Use ssl_context if provided, otherwise default to True for https URLs
|
||||
ssl_setting = ssl_context if ssl_context is not None else ("https://" in api_url)
|
||||
connector = aiohttp.TCPConnector(
|
||||
limit=max_concurrency or 0,
|
||||
limit_per_host=max_concurrency or 0,
|
||||
@@ -634,7 +641,7 @@ async def benchmark(
|
||||
keepalive_timeout=60,
|
||||
enable_cleanup_closed=True,
|
||||
force_close=False,
|
||||
ssl=("https://" in api_url),
|
||||
ssl=ssl_setting,
|
||||
)
|
||||
|
||||
session = aiohttp.ClientSession(
|
||||
@@ -1513,6 +1520,14 @@ def add_cli_args(parser: argparse.ArgumentParser):
|
||||
default=None,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--insecure",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Disable SSL certificate verification. Use this option when "
|
||||
"connecting to servers with self-signed certificates.",
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||
return asyncio.run(main_async(args))
|
||||
@@ -1564,10 +1579,21 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
else:
|
||||
raise ValueError("Invalid header format. Please use KEY=VALUE format.")
|
||||
|
||||
# SSL context configuration
|
||||
ssl_context: ssl.SSLContext | bool | None = None
|
||||
if args.insecure:
|
||||
# Disable SSL certificate verification
|
||||
ssl_context = False
|
||||
elif "https://" in base_url:
|
||||
# Use default SSL context for HTTPS
|
||||
ssl_context = True
|
||||
|
||||
# Fetch model from server if not specified
|
||||
if args.model is None:
|
||||
print("Model not specified, fetching first model from server...")
|
||||
model_name, model_id = await get_first_model_from_server(base_url, headers)
|
||||
model_name, model_id = await get_first_model_from_server(
|
||||
base_url, headers, ssl_context
|
||||
)
|
||||
print(f"First model name: {model_name}, first model id: {model_id}")
|
||||
else:
|
||||
model_name = args.served_model_name
|
||||
@@ -1691,6 +1717,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
ramp_up_start_rps=args.ramp_up_start_rps,
|
||||
ramp_up_end_rps=args.ramp_up_end_rps,
|
||||
ready_check_timeout_sec=args.ready_check_timeout_sec,
|
||||
ssl_context=ssl_context,
|
||||
)
|
||||
|
||||
# Save config and results to json
|
||||
|
||||
Reference in New Issue
Block a user