[CI][RL] Consolidate RL entrypoint tests under tests/entrypoints/rl (#59948)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-07 08:29:09 +00:00
committed by GitHub
co-authored by wang.yuqi Claude Opus 5.5
parent bc8dd4ad27
commit 42ecc8eebe
20 changed files with 46 additions and 22 deletions
@@ -1,349 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared fixtures and helpers for the RL lifecycle test suite.
All test modules under this directory import from here to avoid duplication.
RFC: https://github.com/vllm-project/vllm/issues/45585
PR: https://github.com/vllm-project/vllm/pull/45586
"""
import contextlib
import json
import os
import subprocess
import sys
import threading
import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any
import requests
# ---------------------------------------------------------------------------
# Model / server defaults
# ---------------------------------------------------------------------------
MODEL_NAME = os.environ.get("VLLM_TEST_MODEL", "Qwen/Qwen3-0.6B")
_BASE_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"32",
"--gpu-memory-utilization",
"0.75",
"--enable-sleep-mode",
"--enforce-eager",
]
# Lightweight args for state-machine / protocol tests that don't need real
# weights (avoids spending time downloading a 1B checkpoint in T0 tests).
_DUMMY_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"128",
"--max-num-seqs",
"8",
"--gpu-memory-utilization",
"0.5",
"--enable-sleep-mode",
"--enforce-eager",
"--load-format",
"dummy",
]
# ---------------------------------------------------------------------------
# Server harness
# ---------------------------------------------------------------------------
def _warm_up(url: str) -> None:
"""Put one request through the engine before tests start timing things.
/health turns green before any request has travelled the request path, and
that first pass costs seconds on a loaded machine.
"""
response = gen(url, max_tokens=4, timeout=120)
assert ok(response), f"warm-up generation failed: {response}"
@contextmanager
def server(
extra_args=None,
port: int = 8770,
timeout: float = 180.0,
dummy_weights: bool = False,
):
"""Launch a vLLM server with the dev router; yield its base URL.
Args:
extra_args: Additional CLI flags appended after the base args.
port: HTTP port to bind (caller is responsible for uniqueness).
timeout: Seconds to wait for /health before giving up.
dummy_weights: If True, use --load-format dummy (fast, no real weights).
"""
env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"}
base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS
cmd = [
sys.executable,
"-m",
"vllm.entrypoints.openai.api_server",
"--model",
MODEL_NAME,
"--port",
str(port),
"--served-model-name",
"m",
*(base + (extra_args or [])),
]
proc = subprocess.Popen(
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
)
url = f"http://localhost:{port}"
try:
deadline = time.time() + timeout
while time.time() < deadline:
if proc.poll() is not None:
err = (
proc.stderr.read(4000).decode(errors="replace")
if proc.stderr
else ""
)
raise RuntimeError(f"vllm server exited during startup:\n{err}")
with contextlib.suppress(Exception):
if requests.get(f"{url}/health", timeout=3).status_code == 200:
break
time.sleep(1)
else:
proc.terminate()
raise RuntimeError("vllm server did not start in time")
_warm_up(url)
yield url
finally:
proc.terminate()
with contextlib.suppress(subprocess.TimeoutExpired):
proc.wait(timeout=10)
if proc.poll() is None:
proc.kill()
# ---------------------------------------------------------------------------
# HTTP helpers — generation
# ---------------------------------------------------------------------------
def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30):
"""Fire a /v1/completions request; return JSON or None on any error."""
try:
r = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": 0,
},
timeout=timeout,
)
return r.json()
except Exception:
return None
def ok(resp) -> bool:
"""True iff resp is a successful completion (has choices, no error key)."""
return (
resp is not None
and "choices" in resp
and bool(resp["choices"])
and "error" not in resp
)
# ---------------------------------------------------------------------------
# HTTP helpers — stream generation
# ---------------------------------------------------------------------------
# First-token wait for a streaming request; loaded machines need the slack.
STREAM_START_TIMEOUT = 20.0
@dataclass
class StreamResult:
started: threading.Event = field(default_factory=threading.Event)
done: threading.Event = field(default_factory=threading.Event)
chunks: list[dict[str, Any]] = field(default_factory=list)
finish_reason: str | None = None
error: Exception | None = None
def stream_completion(url: str, result: StreamResult, max_tokens: int) -> None:
try:
with requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": "Count upward slowly: one, two, three,",
"max_tokens": max_tokens,
"temperature": 0,
"ignore_eos": True,
"stream": True,
},
stream=True,
timeout=(5, 60),
) as response:
response.raise_for_status()
for line in response.iter_lines(decode_unicode=True):
if not line or line == "data: [DONE]":
continue
assert line.startswith("data: ")
chunk = json.loads(line.removeprefix("data: "))
result.chunks.append(chunk)
choice = chunk["choices"][0]
if choice.get("text"):
result.started.set()
if choice.get("finish_reason") is not None:
result.finish_reason = choice["finish_reason"]
except Exception as error:
result.error = error
finally:
result.done.set()
def start_stream(url: str, max_tokens: int) -> tuple[StreamResult, threading.Thread]:
result = StreamResult()
thread = threading.Thread(
target=stream_completion,
args=(url, result, max_tokens),
)
thread.start()
started = result.started.wait(timeout=STREAM_START_TIMEOUT)
if not started or result.done.is_set():
# Best-effort: on a stalled server these time out too, and would then
# mask the assertions below.
with contextlib.suppress(requests.RequestException):
pause(url, mode="abort")
resume(url)
thread.join(timeout=10)
assert started, (
f"request did not start generating within {STREAM_START_TIMEOUT}s "
f"(stream error: {result.error})"
)
assert not result.done.is_set(), "request completed before it could be paused"
return result, thread
# ---------------------------------------------------------------------------
# HTTP helpers — pause / resume
# ---------------------------------------------------------------------------
def pause(url, mode="abort", clear_cache=True):
return requests.post(
f"{url}/pause",
params={"mode": mode, "clear_cache": clear_cache},
timeout=15,
).status_code
def resume(url):
return requests.post(f"{url}/resume", timeout=10).status_code
def completion_with_cache_details(url: str, prompt: str) -> dict[str, Any]:
response = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": 8,
"temperature": 0,
"logprobs": 1,
},
timeout=30,
)
response.raise_for_status()
return response.json()
def golden_output(response: dict[str, Any]) -> dict[str, Any]:
choice = response["choices"][0]
usage = response["usage"]
return {
"text": choice["text"],
"finish_reason": choice["finish_reason"],
"tokens": choice["logprobs"]["tokens"],
"prompt_tokens": usage["prompt_tokens"],
"completion_tokens": usage["completion_tokens"],
}
def cached_tokens(response: dict[str, Any]) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
# ---------------------------------------------------------------------------
# HTTP helpers — sleep / wake
# ---------------------------------------------------------------------------
def sleep(url, level=1, mode="abort"):
return requests.post(
f"{url}/sleep", params={"level": level, "mode": mode}, timeout=15
).status_code
def wake(url, tags=None):
params = {"tags": tags} if tags else {}
return requests.post(f"{url}/wake_up", params=params, timeout=20).status_code
def is_sleeping(url) -> bool:
return requests.get(f"{url}/is_sleeping", timeout=5).json()["is_sleeping"]
def is_paused(url) -> bool:
return requests.get(f"{url}/is_paused", timeout=5).json()["is_paused"]
def health(url) -> int:
try:
return requests.get(f"{url}/health", timeout=5).status_code
except Exception:
return 0
# ---------------------------------------------------------------------------
# HTTP helpers — weight transfer
# ---------------------------------------------------------------------------
def start_weight_update(url, is_checkpoint_format=True):
return requests.post(
f"{url}/start_weight_update",
json={"is_checkpoint_format": is_checkpoint_format},
timeout=10,
)
def finish_weight_update(url):
return requests.post(f"{url}/finish_weight_update", timeout=10)
def get_world_size(url, include_dp=True):
return requests.get(
f"{url}/get_world_size",
params={"include_dp": include_dp},
timeout=5,
)
@@ -1,168 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the vLLM RL pause/resume lifecycle."""
import os
import threading
from typing import Any
from unittest.mock import patch
import pytest
import requests
from tests.entrypoints.serve.dev.rlhf.conftest import (
cached_tokens,
completion_with_cache_details,
gen,
golden_output,
is_paused,
ok,
pause,
resume,
server,
start_stream,
)
@pytest.fixture(scope="module", params=[False, True], ids=["MRV1", "MRV2"])
def use_v2(request):
return request.param
@pytest.fixture(scope="module")
def server_url(use_v2):
env_vars = {
"VLLM_USE_V2_MODEL_RUNNER": "1" if use_v2 else "0",
}
with (
patch.dict(os.environ, env_vars),
server(
extra_args=[
"--enable-prefix-caching",
"--enable-prompt-tokens-details",
]
) as url,
):
yield url
@pytest.fixture(autouse=True)
def restore_unpaused_state(server_url):
assert resume(server_url) == 200
yield
assert resume(server_url) == 200
class TestPauseResume:
def test_state_and_idempotency_across_cycles(self, server_url):
assert not is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
for _ in range(2):
for mode in ("abort", "wait", "keep"):
assert pause(server_url, mode=mode) == 200
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
def test_invalid_mode_preserves_state(self, server_url):
for paused in (False, True):
if paused:
assert pause(server_url) == 200
assert is_paused(server_url) is paused
response = requests.post(
f"{server_url}/pause",
params={"mode": "invalid"},
timeout=10,
)
assert response.status_code == 400
assert response.json()["error"]["param"] == "query.mode"
assert is_paused(server_url) is paused
assert resume(server_url) == 200
@pytest.mark.parametrize(
("mode", "max_tokens", "inflight_finish_reason"),
[
pytest.param("abort", 256, "abort", id="abort"),
pytest.param("wait", 256, "length", id="wait"),
pytest.param("keep", 256, "length", id="keep"),
],
)
def test_mode_request_lifecycle(
self,
server_url,
mode,
max_tokens,
inflight_finish_reason,
):
inflight, inflight_thread = start_stream(server_url, max_tokens)
new_result: dict[str, Any] = {}
new_done = threading.Event()
def _new_request():
new_result["response"] = gen(server_url, max_tokens=4, timeout=60)
new_done.set()
new_thread = threading.Thread(target=_new_request)
try:
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
if mode in ("abort", "wait"):
assert inflight.done.is_set()
else:
chunks_after_pause = len(inflight.chunks)
assert not inflight.done.wait(timeout=5)
assert len(inflight.chunks) == chunks_after_pause, (
"in-flight request continued generating in keep mode"
)
new_thread.start()
assert not new_done.wait(timeout=0.3), (
"new request completed while generation was paused"
)
finally:
assert resume(server_url) == 200
inflight_thread.join(timeout=30)
if new_thread.ident is not None:
new_thread.join(timeout=30)
assert not inflight_thread.is_alive()
assert inflight.error is None
assert inflight.finish_reason == inflight_finish_reason
assert not new_thread.is_alive()
assert ok(new_result.get("response"))
def test_clear_cache_preserves_output_and_controls_prefix_cache(self, server_url):
prompt = (
"Paris is the capital of France. Berlin is the capital of Germany. "
) * 20
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
baseline = completion_with_cache_details(server_url, prompt)
warmed = completion_with_cache_details(server_url, prompt)
assert cached_tokens(baseline) == 0
assert cached_tokens(warmed) > 0
assert pause(server_url, clear_cache=False) == 200
assert resume(server_url) == 200
preserved = completion_with_cache_details(server_url, prompt)
assert golden_output(preserved) == golden_output(baseline)
assert cached_tokens(preserved) > 0
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
cleared = completion_with_cache_details(server_url, prompt)
assert golden_output(cleared) == golden_output(baseline)
assert cached_tokens(cleared) == 0
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from unittest.mock import patch
import pytest
import regex as re
import requests
import torch
from transformers import AutoModelForCausalLM
from tests.entrypoints.serve.dev.rlhf.conftest import (
MODEL_NAME,
gen,
health,
ok,
server,
)
from tests.utils import multi_gpu_test
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
from vllm.platforms import current_platform
def weight_checker(url: str, action: str, baseline=None) -> requests.Response:
body = {"action": action}
if baseline is not None:
body["baseline"] = baseline
return requests.post(f"{url}/weight_checker", json=body, timeout=300)
def checksums(url: str) -> dict[str, str]:
response = weight_checker(url, "checksum")
response.raise_for_status()
return response.json()["checksums"]
def ranks(keys) -> set[str]:
return {key.rsplit(":", 1)[0] for key in keys}
def reset_reload_and_compare(url: str) -> None:
baseline = checksums(url)
try:
weight_checker(url, "reset").raise_for_status()
comparison = weight_checker(url, "compare", baseline).json()
assert ranks(comparison["mismatches"]) == ranks(baseline)
finally:
requests.post(
f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300
).raise_for_status()
assert weight_checker(url, "compare", baseline).json() == {
"match": True,
"mismatches": [],
}
@pytest.fixture(scope="class")
def server_url():
with server() as url:
yield url
class TestSingleGPU:
@pytest.mark.parametrize(
"payload",
[{}, {"action": "frobnicate"}, {"action": "compare"}, [], None],
)
def test_invalid_request_returns_400(self, server_url, payload):
response = requests.post(
f"{server_url}/weight_checker", json=payload, timeout=10
)
assert response.status_code == 400, response.text
assert health(server_url) == 200
def test_checksum_is_stable_and_compare_reports_differences(self, server_url):
baseline = checksums(server_url)
assert all(
re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values()
)
assert checksums(server_url) == baseline
changed, missing, *_ = baseline
altered = {**baseline, changed: "0" * 64}
del altered[missing]
assert weight_checker(server_url, "compare", altered).json() == {
"match": False,
"mismatches": sorted([changed, missing]),
}
def test_reset_changes_weights_and_reload_restores_them(self, server_url):
reset_reload_and_compare(server_url)
assert ok(gen(server_url))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="IPC weight transfer uses CUDA IPC."
)
def test_ipc_weight_transfer_restores_reset_weights():
"""The documented RL flow: reset, then transfer the checkpoint back."""
if current_platform.is_xpu():
pytest.skip("IPC weight transfer backend uses CUDA IPC handles")
args = ["--weight-transfer-config", '{"backend": "ipc"}']
with (
patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}),
server(extra_args=args, port=8772) as url,
):
baseline = checksums(url)
weight_checker(url, "reset").raise_for_status()
trainer_model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, torch_dtype=torch.bfloat16
).cuda()
WeightTransferTrainerFactory.trainer_init(
IPCTrainerInitInfo(rank=0),
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer_model),
).send_weights()
assert weight_checker(url, "compare", baseline).json() == {
"match": True,
"mismatches": [],
}
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"args, expected",
[
(["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}),
(["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}),
],
)
def test_checksum_and_reset_cover_every_worker(args, expected):
with server(extra_args=args, port=8771, timeout=600) as url:
assert ranks(checksums(url)) == expected
reset_reload_and_compare(url)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
import requests
import torch
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoModelForCausalLM
from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME
from tests.utils import RemoteOpenAIServer
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
SYNCS = 8
def scrape(url: str) -> dict[tuple[str, str], float]:
text = requests.get(f"{url}/metrics", timeout=30).text
return {
(sample.name, sample.labels["operation"]): sample.value
for family in text_string_to_metric_families(text)
for sample in family.samples
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
}
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
args = [
"--load-format",
"dummy",
"--enforce-eager",
"--max-model-len",
"512",
"--gpu-memory-utilization",
"0.5",
"--api-server-count",
"2",
"--weight-transfer-config",
'{"backend": "ipc"}',
]
env = {
"VLLM_SERVER_DEV_MODE": "1",
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
url = server.url_root
trainer = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, torch_dtype=torch.bfloat16
).cuda()
engine = WeightTransferTrainerFactory.trainer_init(
IPCTrainerInitInfo(rank=0),
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer),
)
for _ in range(SYNCS):
engine.send_weights()
metrics = scrape(url)
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
for operation, count in calls.items():
assert metrics[(f"{DURATION}_count", operation)] == count, operation
assert metrics[(IN_FLIGHT, operation)] == 0, operation
-306
View File
@@ -1,306 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.instrumentator.metrics import (
attach_router as attach_metrics_router,
)
from vllm.v1.metrics import prometheus as prometheus_metrics
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.fixture
def sleep_route_app(monkeypatch):
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
metrics = SleepModeOperationMetrics(CollectorRegistry())
monkeypatch.setattr(
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
lambda: metrics,
)
attach_router(app)
init_exception_handler(app)
return app, metrics
@pytest.mark.cpu_test
@pytest.mark.parametrize("level", [0, 1, 2])
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
app, _ = sleep_route_app
with TestClient(app) as client:
response = client.post("/sleep", params={"level": level})
assert response.status_code == 200
assert response.json() == {"status": "sleeping", "level": level}
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("query", "expected_param"),
[
("level=invalid", "query.level"),
("level=-1", "query.level"),
("level=3", "query.level"),
("mode=invalid", "query.mode"),
],
)
def test_sleep_route_rejects_invalid_query_before_dispatch(
sleep_route_app, query, expected_param
):
app, metrics = sleep_route_app
with TestClient(app) as client:
response = client.post(f"/sleep?{query}")
assert response.status_code == 400
assert response.json()["error"]["param"] == expected_param
app.state.engine_client.sleep.assert_not_awaited()
assert list(metrics.duration.collect()[0].samples) == []
assert list(metrics.in_flight.collect()[0].samples) == []
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("fully_awake", "tags", "expected"),
[
(True, "", {"status": "awake", "tags": None}),
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
],
)
def test_wake_route_maps_engine_result_without_state_query(
sleep_route_app, fully_awake, tags, expected
):
app, _ = sleep_route_app
app.state.engine_client.wake_up.return_value = fully_awake
with TestClient(app) as client:
response = client.post(f"/wake_up{tags}")
assert response.status_code == 200
assert response.json() == expected
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
app.state.engine_client.is_sleeping.assert_not_awaited()
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(sleep_route_app, fails):
app, metrics = sleep_route_app
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
if not fails:
assert response.json() == {"status": "kv_cache_released"}
release.assert_awaited_once_with()
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
assert _duration_count(metrics, "release_kv_cache_memory") == 1
@pytest.mark.cpu_test
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
metrics = SleepModeOperationMetrics(CollectorRegistry())
with metrics.record(operation):
assert metrics.in_flight.labels(operation)._value.get() == 1
with pytest.raises(exception), metrics.record(operation):
raise exception()
assert metrics.in_flight.labels(operation)._value.get() == 0
assert metrics.duration.labels(operation)._sum.get() >= 0
assert _duration_count(metrics, operation) == 2
def _duration_count(metrics, operation):
return next(
sample.value
for sample in metrics.duration.collect()[0].samples
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("path", "method", "operation", "status", "fails"),
[
("/sleep", "POST", "sleep", 200, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
("/wake_up", "POST", "wake", 200, False),
("/is_sleeping", "GET", None, 200, False),
("/sleep?level=3", "POST", None, 400, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
],
)
def test_sleep_routes_visible_on_production_metrics_endpoint(
monkeypatch, path, method, operation, status, fails
):
registry = CollectorRegistry()
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
monkeypatch.setattr(sleep_metrics, "_metrics", None)
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
app.state.engine_client.wake_up.return_value = True
app.state.engine_client.is_sleeping.return_value = False
if fails:
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
"engine failed"
)
attach_router(app)
init_exception_handler(app)
attach_metrics_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
assert client.request(method, path).status_code == status
response = client.get("/metrics")
assert response.status_code == 200
samples = [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
]
assert any(
sample.name == "http_requests_total"
and sample.labels
== {
"handler": path.split("?")[0],
"method": method,
"status": f"{status // 100}xx",
}
and sample.value == 1
for sample in samples
)
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
assert any(
sample.name == "vllm:sleep_mode_operations_in_flight"
and sample.labels == {"operation": operation}
and sample.value == 0
for sample in samples
) == (operation is not None)
assert any(
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
and sample.labels == {"operation": operation}
and sample.value == 1
for sample in samples
) == (operation is not None)
def test_sleep_mode():
# dtype, max-len etc set so that this can run in CI
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--max-num-seqs",
"128",
"--enable-sleep-mode",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
) as remote_server:
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 0
assert weights_offloaded == 1
assert discard_all == 0
response = requests.post(remote_server.url_for("wake_up"))
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
# test wake up with tags
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["weights"]}
)
assert response.status_code == 200
# Partial wake keeps the engine sleeping.
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["kv_cache"]}
)
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
def _get_sleep_metrics_from_api(response: requests.Response):
"""Return (awake, weights_offloaded, discard_all)."""
awake, weights_offloaded, discard_all = None, None, None
for family in text_string_to_metric_families(response.text):
if family.name == "vllm:engine_sleep_state":
for sample in family.samples:
if sample.name == "vllm:engine_sleep_state":
for label_name, label_value in sample.labels.items():
if label_value == "awake":
awake = sample.value
elif label_value == "weights_offloaded":
weights_offloaded = sample.value
elif label_value == "discard_all":
discard_all = sample.value
assert awake is not None
assert weights_offloaded is not None
assert discard_all is not None
return awake, weights_offloaded, discard_all