[Frontend] Add vllm launch command for GPU-less preprocessing serving (#34551)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
+1
-1
@@ -7,7 +7,7 @@ import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E tests for render endpoints via `vllm launch` (GPU-less serving)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from ...utils import RemoteLaunchRenderServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = []
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args, max_wait_seconds=120) as srv:
|
||||
yield srv
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# -- Chat Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
conversation, engine_prompts = data
|
||||
|
||||
assert isinstance(conversation, list)
|
||||
assert conversation[0]["role"] == "user"
|
||||
|
||||
assert isinstance(engine_prompts, list)
|
||||
assert len(engine_prompts) > 0
|
||||
first_prompt = engine_prompts[0]
|
||||
assert "prompt_token_ids" in first_prompt
|
||||
assert "prompt" in first_prompt
|
||||
assert isinstance(first_prompt["prompt_token_ids"], list)
|
||||
assert all(isinstance(t, int) for t in first_prompt["prompt_token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_multi_turn(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
conversation, engine_prompts = response.json()
|
||||
|
||||
assert len(conversation) == 3
|
||||
assert conversation[0]["role"] == "user"
|
||||
assert conversation[1]["role"] == "assistant"
|
||||
assert conversation[2]["role"] == "user"
|
||||
assert len(engine_prompts) > 0
|
||||
assert len(engine_prompts[0]["prompt_token_ids"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_invalid_model(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": "nonexistent-model",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "error" in response.json()
|
||||
|
||||
|
||||
# -- Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Once upon a time",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) > 0
|
||||
|
||||
first_prompt = data[0]
|
||||
assert "prompt_token_ids" in first_prompt
|
||||
assert "prompt" in first_prompt
|
||||
assert isinstance(first_prompt["prompt_token_ids"], list)
|
||||
assert len(first_prompt["prompt_token_ids"]) > 0
|
||||
assert "Once upon a time" in first_prompt["prompt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
for prompt in data:
|
||||
assert "prompt_token_ids" in prompt
|
||||
assert "prompt" in prompt
|
||||
assert len(prompt["prompt_token_ids"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_invalid_model(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": "nonexistent-model",
|
||||
"prompt": "Hello",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "error" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_is_fast(client):
|
||||
"""Render should complete quickly since there is no inference."""
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Tell me a very long story about " * 10,
|
||||
},
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert response.status_code == 200
|
||||
assert elapsed < 2.0
|
||||
|
||||
|
||||
# -- Health & Models --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint(client):
|
||||
response = await client.get("/health")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_endpoint(client):
|
||||
response = await client.get("/v1/models")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "data" in data
|
||||
model_ids = [m["id"] for m in data["data"]]
|
||||
assert MODEL_NAME in model_ids
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the `vllm launch` CLI subcommand."""
|
||||
|
||||
import argparse
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.cli.launch import (
|
||||
LaunchSubcommand,
|
||||
RenderSubcommand,
|
||||
cmd_init,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def launch_parser():
|
||||
parser = FlexibleArgumentParser(description="test")
|
||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||
LaunchSubcommand().subparser_init(subparsers)
|
||||
return parser
|
||||
|
||||
|
||||
def test_subcommand_name():
|
||||
assert LaunchSubcommand().name == "launch"
|
||||
|
||||
|
||||
def test_cmd_init_returns_subcommand():
|
||||
result = cmd_init()
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], LaunchSubcommand)
|
||||
|
||||
|
||||
# -- Parsing: `vllm launch render` --
|
||||
|
||||
|
||||
def test_parse_launch_render(launch_parser):
|
||||
args = launch_parser.parse_args(["launch", "render", "--model", "test-model"])
|
||||
assert args.launch_component == "render"
|
||||
|
||||
|
||||
def test_parse_launch_requires_component(launch_parser):
|
||||
with pytest.raises(SystemExit):
|
||||
launch_parser.parse_args(["launch", "--model", "test-model"])
|
||||
|
||||
|
||||
def test_parse_launch_invalid_component(launch_parser):
|
||||
with pytest.raises(SystemExit):
|
||||
launch_parser.parse_args(["launch", "unknown", "--model", "test-model"])
|
||||
|
||||
|
||||
# -- Dispatch --
|
||||
|
||||
|
||||
def test_cmd_launch_render_calls_run():
|
||||
args = argparse.Namespace(model_tag=None, model="test-model")
|
||||
with patch("vllm.entrypoints.cli.launch.uvloop.run") as mock_uvloop_run:
|
||||
RenderSubcommand.cmd(args)
|
||||
mock_uvloop_run.assert_called_once()
|
||||
|
||||
|
||||
def test_cmd_launch_model_tag_overrides():
|
||||
args = argparse.Namespace(
|
||||
model_tag="tag-model",
|
||||
model="original-model",
|
||||
launch_command=lambda a: None,
|
||||
)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert args.model == "tag-model"
|
||||
|
||||
|
||||
def test_cmd_launch_model_tag_none():
|
||||
args = argparse.Namespace(
|
||||
model_tag=None,
|
||||
model="original-model",
|
||||
launch_command=lambda a: None,
|
||||
)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert args.model == "original-model"
|
||||
|
||||
|
||||
def test_cmd_dispatches():
|
||||
called = {}
|
||||
|
||||
def fake_dispatch(args):
|
||||
called["args"] = args
|
||||
|
||||
args = argparse.Namespace(launch_command=fake_dispatch)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert "args" in called
|
||||
|
||||
|
||||
# -- Module registration --
|
||||
|
||||
|
||||
def test_subparser_init_returns_parser():
|
||||
parser = FlexibleArgumentParser(description="test")
|
||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||
result = LaunchSubcommand().subparser_init(subparsers)
|
||||
assert isinstance(result, FlexibleArgumentParser)
|
||||
|
||||
|
||||
def test_launch_registered_in_main():
|
||||
"""Verify that launch module is importable as a CLI module."""
|
||||
import vllm.entrypoints.cli.launch as launch_module
|
||||
|
||||
assert hasattr(launch_module, "cmd_init")
|
||||
subcmds = launch_module.cmd_init()
|
||||
assert any(s.name == "launch" for s in subcmds)
|
||||
+76
-23
@@ -110,31 +110,25 @@ VLLM_PATH = Path(__file__).parent.parent
|
||||
"""Path to root of the vLLM repository."""
|
||||
|
||||
|
||||
class RemoteOpenAIServer:
|
||||
class RemoteVLLMServer:
|
||||
"""Base class for launching vLLM server subprocesses for testing.
|
||||
|
||||
Subclasses must override ``_create_cli_subcommand`` and
|
||||
``_start_server``.
|
||||
"""
|
||||
|
||||
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
|
||||
proc: subprocess.Popen
|
||||
|
||||
def _create_cli_subcommand(self):
|
||||
"""Return a CLISubcommand instance used to parse CLI args."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _start_server(
|
||||
self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
|
||||
) -> None:
|
||||
"""Subclasses override this method to customize server process launch"""
|
||||
env = os.environ.copy()
|
||||
# the current process might initialize cuda,
|
||||
# to be safe, we should use spawn method
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
|
||||
print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
|
||||
print(f"Environment variables: {env}")
|
||||
self.proc: subprocess.Popen = subprocess.Popen(
|
||||
serve_cmd,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
# Create a dedicated process group so we can kill
|
||||
# the entire tree (parent + EngineCore + workers) at once.
|
||||
start_new_session=True,
|
||||
)
|
||||
raise NotImplementedError
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -171,9 +165,9 @@ class RemoteOpenAIServer:
|
||||
json.dumps(override_hf_configs),
|
||||
]
|
||||
|
||||
parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
|
||||
parser = FlexibleArgumentParser(description="vLLM's remote server.")
|
||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||
parser = ServeSubcommand().subparser_init(subparsers)
|
||||
parser = self._create_cli_subcommand().subparser_init(subparsers)
|
||||
args = parser.parse_args(["--model", model, *vllm_serve_args])
|
||||
self.uds = args.uds
|
||||
if args.uds:
|
||||
@@ -183,7 +177,9 @@ class RemoteOpenAIServer:
|
||||
self.host = str(args.host or "127.0.0.1")
|
||||
self.port = int(args.port)
|
||||
|
||||
self.show_hidden_metrics = args.show_hidden_metrics_for_version is not None
|
||||
self.show_hidden_metrics = (
|
||||
getattr(args, "show_hidden_metrics_for_version", None) is not None
|
||||
)
|
||||
|
||||
# download the model before starting the server to avoid timeout
|
||||
is_local = os.path.isdir(model)
|
||||
@@ -201,7 +197,8 @@ class RemoteOpenAIServer:
|
||||
if self._pre_server_gpu_memory is not None:
|
||||
pre_gb = self._pre_server_gpu_memory / 1e9
|
||||
print(
|
||||
f"[RemoteOpenAIServer] GPU memory before server start: {pre_gb:.2f} GB"
|
||||
f"[{type(self).__name__}] GPU memory before server start: "
|
||||
f"{pre_gb:.2f} GB"
|
||||
)
|
||||
|
||||
self._start_server(model, vllm_serve_args, env_dict)
|
||||
@@ -452,6 +449,62 @@ class RemoteOpenAIServer:
|
||||
)
|
||||
|
||||
|
||||
class RemoteOpenAIServer(RemoteVLLMServer):
|
||||
"""Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
|
||||
|
||||
def _create_cli_subcommand(self):
|
||||
return ServeSubcommand()
|
||||
|
||||
def _start_server(
|
||||
self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
|
||||
) -> None:
|
||||
env = os.environ.copy()
|
||||
# the current process might initialize cuda,
|
||||
# to be safe, we should use spawn method
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
|
||||
print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
|
||||
print(f"Environment variables: {env}")
|
||||
self.proc: subprocess.Popen = subprocess.Popen(
|
||||
serve_cmd,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
# Create a dedicated process group so we can kill
|
||||
# the entire tree (parent + EngineCore + workers) at once.
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
class RemoteLaunchRenderServer(RemoteVLLMServer):
|
||||
"""Launches ``vllm launch render`` for GPU-less serving tests."""
|
||||
|
||||
def _create_cli_subcommand(self):
|
||||
return ServeSubcommand()
|
||||
|
||||
def _start_server(
|
||||
self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
|
||||
) -> None:
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if env_dict is not None:
|
||||
env.update(env_dict)
|
||||
serve_cmd = ["vllm", "launch", "render", model, *vllm_serve_args]
|
||||
print(f"Launching RemoteLaunchRenderServer with: {' '.join(serve_cmd)}")
|
||||
self.proc: subprocess.Popen = subprocess.Popen(
|
||||
serve_cmd,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
|
||||
pass # No GPU used
|
||||
|
||||
|
||||
class RemoteOpenAIServerCustom(RemoteOpenAIServer):
|
||||
"""Launch test server with custom child process"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user