[Test] Add pause/resume E2E tests (#52144)

Signed-off-by: flb_ <floatlibai@gmail.com>
This commit is contained in:
flb_
2026-08-18 14:46:44 +08:00
committed by GitHub
parent c89d692cb5
commit d785eb51ce
3 changed files with 582 additions and 0 deletions
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Shared fixtures and helpers for the RL lifecycle test suite.
All test modules under this directory import from here to avoid duplication.
RFC: https://github.com/vllm-project/vllm/issues/45585
PR: https://github.com/vllm-project/vllm/pull/45586
"""
import contextlib
import json
import os
import subprocess
import sys
import threading
import time
from collections.abc import Callable
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any
import requests
# ---------------------------------------------------------------------------
# Model / server defaults
# ---------------------------------------------------------------------------
MODEL_NAME = os.environ.get("VLLM_TEST_MODEL", "Qwen/Qwen3-0.6B")
_BASE_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"32",
"--gpu-memory-utilization",
"0.75",
"--enable-sleep-mode",
"--enforce-eager",
]
# Lightweight args for state-machine / protocol tests that don't need real
# weights (avoids spending time downloading a 1B checkpoint in T0 tests).
_DUMMY_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"128",
"--max-num-seqs",
"8",
"--gpu-memory-utilization",
"0.5",
"--enable-sleep-mode",
"--enforce-eager",
"--load-format",
"dummy",
]
# ---------------------------------------------------------------------------
# Server harness
# ---------------------------------------------------------------------------
@contextmanager
def server(
extra_args=None,
port: int = 8770,
timeout: float = 180.0,
dummy_weights: bool = False,
):
"""Launch a vLLM server with the dev router; yield its base URL.
Args:
extra_args: Additional CLI flags appended after the base args.
port: HTTP port to bind (caller is responsible for uniqueness).
timeout: Seconds to wait for /health before giving up.
dummy_weights: If True, use --load-format dummy (fast, no real weights).
"""
env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"}
base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS
cmd = [
sys.executable,
"-m",
"vllm.entrypoints.openai.api_server",
"--model",
MODEL_NAME,
"--port",
str(port),
"--served-model-name",
"m",
*(base + (extra_args or [])),
]
proc = subprocess.Popen(
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
)
url = f"http://localhost:{port}"
try:
deadline = time.time() + timeout
while time.time() < deadline:
if proc.poll() is not None:
err = (
proc.stderr.read(4000).decode(errors="replace")
if proc.stderr
else ""
)
raise RuntimeError(f"vllm server exited during startup:\n{err}")
with contextlib.suppress(Exception):
if requests.get(f"{url}/health", timeout=3).status_code == 200:
break
time.sleep(1)
else:
proc.terminate()
raise RuntimeError("vllm server did not start in time")
yield url
finally:
proc.terminate()
with contextlib.suppress(subprocess.TimeoutExpired):
proc.wait(timeout=10)
if proc.poll() is None:
proc.kill()
# ---------------------------------------------------------------------------
# Polling helper (200-lie workaround)
# ---------------------------------------------------------------------------
def poll_until(
predicate: Callable[[], bool],
timeout: float = 10.0,
interval: float = 0.5,
) -> bool:
"""Poll predicate() until it returns True or timeout expires.
Workaround for the vLLM sleep/wake "200-lie" — the HTTP endpoints may
return 200 before the underlying operation is complete, so callers that
need to verify state *after* an operation can use this helper instead of
assuming the 200 means completion.
Returns True if predicate became true within timeout, False otherwise.
"""
deadline = time.time() + timeout
while time.time() < deadline:
try:
if predicate():
return True
except Exception:
pass
time.sleep(interval)
return False
# ---------------------------------------------------------------------------
# HTTP helpers — generation
# ---------------------------------------------------------------------------
def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30):
"""Fire a /v1/completions request; return JSON or None on any error."""
try:
r = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": 0,
},
timeout=timeout,
)
return r.json()
except Exception:
return None
def gen_with_logprobs(
url, prompt="The capital of France is", max_tokens=8, logprobs=5, timeout=30
):
"""Fire a /v1/completions request with logprobs; return JSON or None."""
try:
r = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": 0,
"logprobs": logprobs,
},
timeout=timeout,
)
return r.json()
except Exception:
return None
def ok(resp) -> bool:
"""True iff resp is a successful completion (has choices, no error key)."""
return (
resp is not None
and "choices" in resp
and bool(resp["choices"])
and "error" not in resp
)
# ---------------------------------------------------------------------------
# HTTP helpers — stream generation
# ---------------------------------------------------------------------------
@dataclass
class StreamResult:
started: threading.Event = field(default_factory=threading.Event)
done: threading.Event = field(default_factory=threading.Event)
chunks: list[dict[str, Any]] = field(default_factory=list)
finish_reason: str | None = None
error: Exception | None = None
def stream_completion(url: str, result: StreamResult, max_tokens: int) -> None:
try:
with requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": "Count upward slowly: one, two, three,",
"max_tokens": max_tokens,
"temperature": 0,
"ignore_eos": True,
"stream": True,
},
stream=True,
timeout=(5, 60),
) as response:
response.raise_for_status()
for line in response.iter_lines(decode_unicode=True):
if not line or line == "data: [DONE]":
continue
assert line.startswith("data: ")
chunk = json.loads(line.removeprefix("data: "))
result.chunks.append(chunk)
choice = chunk["choices"][0]
if choice.get("text"):
result.started.set()
if choice.get("finish_reason") is not None:
result.finish_reason = choice["finish_reason"]
except Exception as error:
result.error = error
finally:
result.done.set()
def start_stream(url: str, max_tokens: int) -> tuple[StreamResult, threading.Thread]:
result = StreamResult()
thread = threading.Thread(
target=stream_completion,
args=(url, result, max_tokens),
)
thread.start()
started = result.started.wait(timeout=10)
if not started or result.done.is_set():
pause(url, mode="abort")
resume(url)
thread.join(timeout=10)
assert started, "request did not start generating"
assert not result.done.is_set(), "request completed before it could be paused"
return result, thread
# ---------------------------------------------------------------------------
# HTTP helpers — pause / resume
# ---------------------------------------------------------------------------
def pause(url, mode="abort", clear_cache=True):
return requests.post(
f"{url}/pause",
params={"mode": mode, "clear_cache": clear_cache},
timeout=15,
).status_code
def resume(url):
return requests.post(f"{url}/resume", timeout=10).status_code
def completion_with_cache_details(url: str, prompt: str) -> dict[str, Any]:
response = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": 8,
"temperature": 0,
"logprobs": 1,
},
timeout=30,
)
response.raise_for_status()
return response.json()
def golden_output(response: dict[str, Any]) -> dict[str, Any]:
choice = response["choices"][0]
usage = response["usage"]
return {
"text": choice["text"],
"finish_reason": choice["finish_reason"],
"tokens": choice["logprobs"]["tokens"],
"prompt_tokens": usage["prompt_tokens"],
"completion_tokens": usage["completion_tokens"],
}
def cached_tokens(response: dict[str, Any]) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
# ---------------------------------------------------------------------------
# HTTP helpers — sleep / wake
# ---------------------------------------------------------------------------
def sleep(url, level=1, mode="abort"):
return requests.post(
f"{url}/sleep", params={"level": level, "mode": mode}, timeout=15
).status_code
def wake(url, tags=None):
params = {"tags": tags} if tags else {}
return requests.post(f"{url}/wake_up", params=params, timeout=20).status_code
def is_sleeping(url) -> bool:
return requests.get(f"{url}/is_sleeping", timeout=5).json()["is_sleeping"]
def is_paused(url) -> bool:
return requests.get(f"{url}/is_paused", timeout=5).json()["is_paused"]
def health(url) -> int:
try:
return requests.get(f"{url}/health", timeout=5).status_code
except Exception:
return 0
# ---------------------------------------------------------------------------
# HTTP helpers — weight transfer
# ---------------------------------------------------------------------------
def start_weight_update(url, is_checkpoint_format=True):
return requests.post(
f"{url}/start_weight_update",
json={"is_checkpoint_format": is_checkpoint_format},
timeout=10,
)
def finish_weight_update(url):
return requests.post(f"{url}/finish_weight_update", timeout=10)
def get_world_size(url, include_dp=True):
return requests.get(
f"{url}/get_world_size",
params={"include_dp": include_dp},
timeout=5,
)
# ---------------------------------------------------------------------------
# GPU / metrics helpers
# ---------------------------------------------------------------------------
def gpu_free_bytes(device: int = 0) -> int:
"""Read GPU free bytes via subprocess to avoid import-time torch init."""
out = subprocess.check_output(
[
sys.executable,
"-c",
f"import torch; f,_=torch.accelerator.get_memory_info({device}); print(f)",
],
timeout=10,
)
return int(out.strip())
def sleep_metrics(url):
"""Return (awake, weights_offloaded, discard_all) from /metrics."""
try:
from prometheus_client.parser import text_string_to_metric_families
except ImportError:
return None, None, None
r = requests.get(f"{url}/metrics", timeout=5)
vals: dict = {}
for family in text_string_to_metric_families(r.text):
if family.name == "vllm:engine_sleep_state":
for s in family.samples:
vals[s.labels.get("sleep_state", "")] = s.value
return vals.get("awake"), vals.get("weights_offloaded"), vals.get("discard_all")
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the vLLM RL pause/resume lifecycle."""
import os
import threading
from typing import Any
from unittest.mock import patch
import pytest
import requests
from tests.entrypoints.serve.dev.rlhf.conftest import (
cached_tokens,
completion_with_cache_details,
gen,
golden_output,
is_paused,
ok,
pause,
resume,
server,
start_stream,
)
@pytest.fixture(scope="module", params=[False, True], ids=["MRV1", "MRV2"])
def use_v2(request):
return request.param
@pytest.fixture(scope="module")
def server_url(use_v2):
env_vars = {
"VLLM_USE_V2_MODEL_RUNNER": "1" if use_v2 else "0",
}
with (
patch.dict(os.environ, env_vars),
server(
extra_args=[
"--enable-prefix-caching",
"--enable-prompt-tokens-details",
]
) as url,
):
yield url
@pytest.fixture(autouse=True)
def restore_unpaused_state(server_url):
assert resume(server_url) == 200
yield
assert resume(server_url) == 200
class TestPauseResume:
def test_state_and_idempotency_across_cycles(self, server_url):
assert not is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
for _ in range(2):
for mode in ("abort", "wait", "keep"):
assert pause(server_url, mode=mode) == 200
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
def test_invalid_mode_preserves_state(self, server_url):
for paused in (False, True):
if paused:
assert pause(server_url) == 200
assert is_paused(server_url) is paused
response = requests.post(
f"{server_url}/pause",
params={"mode": "invalid"},
timeout=10,
)
assert response.status_code == 400
assert response.json()["error"]["param"] == "query.mode"
assert is_paused(server_url) is paused
assert resume(server_url) == 200
@pytest.mark.parametrize(
("mode", "max_tokens", "inflight_finish_reason"),
[
pytest.param("abort", 256, "abort", id="abort"),
pytest.param("wait", 256, "length", id="wait"),
pytest.param("keep", 256, "length", id="keep"),
],
)
def test_mode_request_lifecycle(
self,
server_url,
mode,
max_tokens,
inflight_finish_reason,
):
inflight, inflight_thread = start_stream(server_url, max_tokens)
new_result: dict[str, Any] = {}
new_done = threading.Event()
def _new_request():
new_result["response"] = gen(server_url, max_tokens=4, timeout=60)
new_done.set()
new_thread = threading.Thread(target=_new_request)
try:
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
if mode in ("abort", "wait"):
assert inflight.done.is_set()
else:
chunks_after_pause = len(inflight.chunks)
assert not inflight.done.wait(timeout=5)
assert len(inflight.chunks) == chunks_after_pause, (
"in-flight request continued generating in keep mode"
)
new_thread.start()
assert not new_done.wait(timeout=0.3), (
"new request completed while generation was paused"
)
finally:
assert resume(server_url) == 200
inflight_thread.join(timeout=30)
if new_thread.ident is not None:
new_thread.join(timeout=30)
assert not inflight_thread.is_alive()
assert inflight.error is None
assert inflight.finish_reason == inflight_finish_reason
assert not new_thread.is_alive()
assert ok(new_result.get("response"))
def test_clear_cache_preserves_output_and_controls_prefix_cache(self, server_url):
prompt = (
"Paris is the capital of France. Berlin is the capital of Germany. "
) * 20
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
baseline = completion_with_cache_details(server_url, prompt)
warmed = completion_with_cache_details(server_url, prompt)
assert cached_tokens(baseline) == 0
assert cached_tokens(warmed) > 0
assert pause(server_url, clear_cache=False) == 200
assert resume(server_url) == 200
preserved = completion_with_cache_details(server_url, prompt)
assert golden_output(preserved) == golden_output(baseline)
assert cached_tokens(preserved) > 0
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
cleared = completion_with_cache_details(server_url, prompt)
assert golden_output(cleared) == golden_output(baseline)
assert cached_tokens(cleared) == 0