[CI][RL] Consolidate RL entrypoint tests under tests/entrypoints/rl (#59948)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
wang.yuqi
Claude Opus 5.5
parent
bc8dd4ad27
commit
42ecc8eebe
@@ -1,349 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared fixtures and helpers for the RL lifecycle test suite.
|
||||
|
||||
All test modules under this directory import from here to avoid duplication.
|
||||
|
||||
RFC: https://github.com/vllm-project/vllm/issues/45585
|
||||
PR: https://github.com/vllm-project/vllm/pull/45586
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model / server defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
MODEL_NAME = os.environ.get("VLLM_TEST_MODEL", "Qwen/Qwen3-0.6B")
|
||||
|
||||
|
||||
_BASE_ARGS = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--gpu-memory-utilization",
|
||||
"0.75",
|
||||
"--enable-sleep-mode",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
# Lightweight args for state-machine / protocol tests that don't need real
|
||||
# weights (avoids spending time downloading a 1B checkpoint in T0 tests).
|
||||
_DUMMY_ARGS = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"128",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--enable-sleep-mode",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server harness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _warm_up(url: str) -> None:
|
||||
"""Put one request through the engine before tests start timing things.
|
||||
|
||||
/health turns green before any request has travelled the request path, and
|
||||
that first pass costs seconds on a loaded machine.
|
||||
"""
|
||||
response = gen(url, max_tokens=4, timeout=120)
|
||||
assert ok(response), f"warm-up generation failed: {response}"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def server(
|
||||
extra_args=None,
|
||||
port: int = 8770,
|
||||
timeout: float = 180.0,
|
||||
dummy_weights: bool = False,
|
||||
):
|
||||
"""Launch a vLLM server with the dev router; yield its base URL.
|
||||
|
||||
Args:
|
||||
extra_args: Additional CLI flags appended after the base args.
|
||||
port: HTTP port to bind (caller is responsible for uniqueness).
|
||||
timeout: Seconds to wait for /health before giving up.
|
||||
dummy_weights: If True, use --load-format dummy (fast, no real weights).
|
||||
|
||||
"""
|
||||
env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"}
|
||||
base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.openai.api_server",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--port",
|
||||
str(port),
|
||||
"--served-model-name",
|
||||
"m",
|
||||
*(base + (extra_args or [])),
|
||||
]
|
||||
proc = subprocess.Popen(
|
||||
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
||||
)
|
||||
url = f"http://localhost:{port}"
|
||||
try:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
err = (
|
||||
proc.stderr.read(4000).decode(errors="replace")
|
||||
if proc.stderr
|
||||
else ""
|
||||
)
|
||||
raise RuntimeError(f"vllm server exited during startup:\n{err}")
|
||||
with contextlib.suppress(Exception):
|
||||
if requests.get(f"{url}/health", timeout=3).status_code == 200:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
proc.terminate()
|
||||
raise RuntimeError("vllm server did not start in time")
|
||||
_warm_up(url)
|
||||
yield url
|
||||
finally:
|
||||
proc.terminate()
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
proc.wait(timeout=10)
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30):
|
||||
"""Fire a /v1/completions request; return JSON or None on any error."""
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
return r.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def ok(resp) -> bool:
|
||||
"""True iff resp is a successful completion (has choices, no error key)."""
|
||||
return (
|
||||
resp is not None
|
||||
and "choices" in resp
|
||||
and bool(resp["choices"])
|
||||
and "error" not in resp
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — stream generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# First-token wait for a streaming request; loaded machines need the slack.
|
||||
STREAM_START_TIMEOUT = 20.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamResult:
|
||||
started: threading.Event = field(default_factory=threading.Event)
|
||||
done: threading.Event = field(default_factory=threading.Event)
|
||||
chunks: list[dict[str, Any]] = field(default_factory=list)
|
||||
finish_reason: str | None = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
def stream_completion(url: str, result: StreamResult, max_tokens: int) -> None:
|
||||
try:
|
||||
with requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": "Count upward slowly: one, two, three,",
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
"ignore_eos": True,
|
||||
"stream": True,
|
||||
},
|
||||
stream=True,
|
||||
timeout=(5, 60),
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
for line in response.iter_lines(decode_unicode=True):
|
||||
if not line or line == "data: [DONE]":
|
||||
continue
|
||||
assert line.startswith("data: ")
|
||||
chunk = json.loads(line.removeprefix("data: "))
|
||||
result.chunks.append(chunk)
|
||||
choice = chunk["choices"][0]
|
||||
if choice.get("text"):
|
||||
result.started.set()
|
||||
if choice.get("finish_reason") is not None:
|
||||
result.finish_reason = choice["finish_reason"]
|
||||
except Exception as error:
|
||||
result.error = error
|
||||
finally:
|
||||
result.done.set()
|
||||
|
||||
|
||||
def start_stream(url: str, max_tokens: int) -> tuple[StreamResult, threading.Thread]:
|
||||
result = StreamResult()
|
||||
thread = threading.Thread(
|
||||
target=stream_completion,
|
||||
args=(url, result, max_tokens),
|
||||
)
|
||||
thread.start()
|
||||
started = result.started.wait(timeout=STREAM_START_TIMEOUT)
|
||||
if not started or result.done.is_set():
|
||||
# Best-effort: on a stalled server these time out too, and would then
|
||||
# mask the assertions below.
|
||||
with contextlib.suppress(requests.RequestException):
|
||||
pause(url, mode="abort")
|
||||
resume(url)
|
||||
thread.join(timeout=10)
|
||||
assert started, (
|
||||
f"request did not start generating within {STREAM_START_TIMEOUT}s "
|
||||
f"(stream error: {result.error})"
|
||||
)
|
||||
assert not result.done.is_set(), "request completed before it could be paused"
|
||||
return result, thread
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — pause / resume
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pause(url, mode="abort", clear_cache=True):
|
||||
return requests.post(
|
||||
f"{url}/pause",
|
||||
params={"mode": mode, "clear_cache": clear_cache},
|
||||
timeout=15,
|
||||
).status_code
|
||||
|
||||
|
||||
def resume(url):
|
||||
return requests.post(f"{url}/resume", timeout=10).status_code
|
||||
|
||||
|
||||
def completion_with_cache_details(url: str, prompt: str) -> dict[str, Any]:
|
||||
response = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": prompt,
|
||||
"max_tokens": 8,
|
||||
"temperature": 0,
|
||||
"logprobs": 1,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def golden_output(response: dict[str, Any]) -> dict[str, Any]:
|
||||
choice = response["choices"][0]
|
||||
usage = response["usage"]
|
||||
return {
|
||||
"text": choice["text"],
|
||||
"finish_reason": choice["finish_reason"],
|
||||
"tokens": choice["logprobs"]["tokens"],
|
||||
"prompt_tokens": usage["prompt_tokens"],
|
||||
"completion_tokens": usage["completion_tokens"],
|
||||
}
|
||||
|
||||
|
||||
def cached_tokens(response: dict[str, Any]) -> int:
|
||||
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — sleep / wake
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def sleep(url, level=1, mode="abort"):
|
||||
return requests.post(
|
||||
f"{url}/sleep", params={"level": level, "mode": mode}, timeout=15
|
||||
).status_code
|
||||
|
||||
|
||||
def wake(url, tags=None):
|
||||
params = {"tags": tags} if tags else {}
|
||||
return requests.post(f"{url}/wake_up", params=params, timeout=20).status_code
|
||||
|
||||
|
||||
def is_sleeping(url) -> bool:
|
||||
return requests.get(f"{url}/is_sleeping", timeout=5).json()["is_sleeping"]
|
||||
|
||||
|
||||
def is_paused(url) -> bool:
|
||||
return requests.get(f"{url}/is_paused", timeout=5).json()["is_paused"]
|
||||
|
||||
|
||||
def health(url) -> int:
|
||||
try:
|
||||
return requests.get(f"{url}/health", timeout=5).status_code
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — weight transfer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def start_weight_update(url, is_checkpoint_format=True):
|
||||
return requests.post(
|
||||
f"{url}/start_weight_update",
|
||||
json={"is_checkpoint_format": is_checkpoint_format},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
def finish_weight_update(url):
|
||||
return requests.post(f"{url}/finish_weight_update", timeout=10)
|
||||
|
||||
|
||||
def get_world_size(url, include_dp=True):
|
||||
return requests.get(
|
||||
f"{url}/get_world_size",
|
||||
params={"include_dp": include_dp},
|
||||
timeout=5,
|
||||
)
|
||||
@@ -1,168 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the vLLM RL pause/resume lifecycle."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import (
|
||||
cached_tokens,
|
||||
completion_with_cache_details,
|
||||
gen,
|
||||
golden_output,
|
||||
is_paused,
|
||||
ok,
|
||||
pause,
|
||||
resume,
|
||||
server,
|
||||
start_stream,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[False, True], ids=["MRV1", "MRV2"])
|
||||
def use_v2(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_url(use_v2):
|
||||
env_vars = {
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1" if use_v2 else "0",
|
||||
}
|
||||
|
||||
with (
|
||||
patch.dict(os.environ, env_vars),
|
||||
server(
|
||||
extra_args=[
|
||||
"--enable-prefix-caching",
|
||||
"--enable-prompt-tokens-details",
|
||||
]
|
||||
) as url,
|
||||
):
|
||||
yield url
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_unpaused_state(server_url):
|
||||
assert resume(server_url) == 200
|
||||
yield
|
||||
assert resume(server_url) == 200
|
||||
|
||||
|
||||
class TestPauseResume:
|
||||
def test_state_and_idempotency_across_cycles(self, server_url):
|
||||
assert not is_paused(server_url)
|
||||
|
||||
assert resume(server_url) == 200
|
||||
assert resume(server_url) == 200
|
||||
assert not is_paused(server_url)
|
||||
|
||||
for _ in range(2):
|
||||
for mode in ("abort", "wait", "keep"):
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert is_paused(server_url)
|
||||
|
||||
assert resume(server_url) == 200
|
||||
assert resume(server_url) == 200
|
||||
assert not is_paused(server_url)
|
||||
|
||||
def test_invalid_mode_preserves_state(self, server_url):
|
||||
for paused in (False, True):
|
||||
if paused:
|
||||
assert pause(server_url) == 200
|
||||
assert is_paused(server_url) is paused
|
||||
|
||||
response = requests.post(
|
||||
f"{server_url}/pause",
|
||||
params={"mode": "invalid"},
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["param"] == "query.mode"
|
||||
assert is_paused(server_url) is paused
|
||||
|
||||
assert resume(server_url) == 200
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "max_tokens", "inflight_finish_reason"),
|
||||
[
|
||||
pytest.param("abort", 256, "abort", id="abort"),
|
||||
pytest.param("wait", 256, "length", id="wait"),
|
||||
pytest.param("keep", 256, "length", id="keep"),
|
||||
],
|
||||
)
|
||||
def test_mode_request_lifecycle(
|
||||
self,
|
||||
server_url,
|
||||
mode,
|
||||
max_tokens,
|
||||
inflight_finish_reason,
|
||||
):
|
||||
inflight, inflight_thread = start_stream(server_url, max_tokens)
|
||||
new_result: dict[str, Any] = {}
|
||||
new_done = threading.Event()
|
||||
|
||||
def _new_request():
|
||||
new_result["response"] = gen(server_url, max_tokens=4, timeout=60)
|
||||
new_done.set()
|
||||
|
||||
new_thread = threading.Thread(target=_new_request)
|
||||
try:
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert is_paused(server_url)
|
||||
|
||||
if mode in ("abort", "wait"):
|
||||
assert inflight.done.is_set()
|
||||
else:
|
||||
chunks_after_pause = len(inflight.chunks)
|
||||
assert not inflight.done.wait(timeout=5)
|
||||
assert len(inflight.chunks) == chunks_after_pause, (
|
||||
"in-flight request continued generating in keep mode"
|
||||
)
|
||||
|
||||
new_thread.start()
|
||||
assert not new_done.wait(timeout=0.3), (
|
||||
"new request completed while generation was paused"
|
||||
)
|
||||
finally:
|
||||
assert resume(server_url) == 200
|
||||
inflight_thread.join(timeout=30)
|
||||
if new_thread.ident is not None:
|
||||
new_thread.join(timeout=30)
|
||||
|
||||
assert not inflight_thread.is_alive()
|
||||
assert inflight.error is None
|
||||
assert inflight.finish_reason == inflight_finish_reason
|
||||
assert not new_thread.is_alive()
|
||||
assert ok(new_result.get("response"))
|
||||
|
||||
def test_clear_cache_preserves_output_and_controls_prefix_cache(self, server_url):
|
||||
prompt = (
|
||||
"Paris is the capital of France. Berlin is the capital of Germany. "
|
||||
) * 20
|
||||
|
||||
assert pause(server_url, clear_cache=True) == 200
|
||||
assert resume(server_url) == 200
|
||||
baseline = completion_with_cache_details(server_url, prompt)
|
||||
warmed = completion_with_cache_details(server_url, prompt)
|
||||
|
||||
assert cached_tokens(baseline) == 0
|
||||
assert cached_tokens(warmed) > 0
|
||||
|
||||
assert pause(server_url, clear_cache=False) == 200
|
||||
assert resume(server_url) == 200
|
||||
preserved = completion_with_cache_details(server_url, prompt)
|
||||
assert golden_output(preserved) == golden_output(baseline)
|
||||
assert cached_tokens(preserved) > 0
|
||||
|
||||
assert pause(server_url, clear_cache=True) == 200
|
||||
assert resume(server_url) == 200
|
||||
cleared = completion_with_cache_details(server_url, prompt)
|
||||
assert golden_output(cleared) == golden_output(baseline)
|
||||
assert cached_tokens(cleared) == 0
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import requests
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import (
|
||||
MODEL_NAME,
|
||||
gen,
|
||||
health,
|
||||
ok,
|
||||
server,
|
||||
)
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def weight_checker(url: str, action: str, baseline=None) -> requests.Response:
|
||||
body = {"action": action}
|
||||
if baseline is not None:
|
||||
body["baseline"] = baseline
|
||||
return requests.post(f"{url}/weight_checker", json=body, timeout=300)
|
||||
|
||||
|
||||
def checksums(url: str) -> dict[str, str]:
|
||||
response = weight_checker(url, "checksum")
|
||||
response.raise_for_status()
|
||||
return response.json()["checksums"]
|
||||
|
||||
|
||||
def ranks(keys) -> set[str]:
|
||||
return {key.rsplit(":", 1)[0] for key in keys}
|
||||
|
||||
|
||||
def reset_reload_and_compare(url: str) -> None:
|
||||
baseline = checksums(url)
|
||||
try:
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
comparison = weight_checker(url, "compare", baseline).json()
|
||||
assert ranks(comparison["mismatches"]) == ranks(baseline)
|
||||
finally:
|
||||
requests.post(
|
||||
f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300
|
||||
).raise_for_status()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def server_url():
|
||||
with server() as url:
|
||||
yield url
|
||||
|
||||
|
||||
class TestSingleGPU:
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[{}, {"action": "frobnicate"}, {"action": "compare"}, [], None],
|
||||
)
|
||||
def test_invalid_request_returns_400(self, server_url, payload):
|
||||
response = requests.post(
|
||||
f"{server_url}/weight_checker", json=payload, timeout=10
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert health(server_url) == 200
|
||||
|
||||
def test_checksum_is_stable_and_compare_reports_differences(self, server_url):
|
||||
baseline = checksums(server_url)
|
||||
assert all(
|
||||
re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values()
|
||||
)
|
||||
assert checksums(server_url) == baseline
|
||||
|
||||
changed, missing, *_ = baseline
|
||||
altered = {**baseline, changed: "0" * 64}
|
||||
del altered[missing]
|
||||
assert weight_checker(server_url, "compare", altered).json() == {
|
||||
"match": False,
|
||||
"mismatches": sorted([changed, missing]),
|
||||
}
|
||||
|
||||
def test_reset_changes_weights_and_reload_restores_them(self, server_url):
|
||||
reset_reload_and_compare(server_url)
|
||||
assert ok(gen(server_url))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="IPC weight transfer uses CUDA IPC."
|
||||
)
|
||||
def test_ipc_weight_transfer_restores_reset_weights():
|
||||
"""The documented RL flow: reset, then transfer the checkpoint back."""
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("IPC weight transfer backend uses CUDA IPC handles")
|
||||
args = ["--weight-transfer-config", '{"backend": "ipc"}']
|
||||
with (
|
||||
patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}),
|
||||
server(extra_args=args, port=8772) as url,
|
||||
):
|
||||
baseline = checksums(url)
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
trainer_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer_model),
|
||||
).send_weights()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"args, expected",
|
||||
[
|
||||
(["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}),
|
||||
(["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}),
|
||||
],
|
||||
)
|
||||
def test_checksum_and_reset_cover_every_worker(args, expected):
|
||||
with server(extra_args=args, port=8771, timeout=600) as url:
|
||||
assert ranks(checksums(url)) == expected
|
||||
reset_reload_and_compare(url)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
SYNCS = 8
|
||||
|
||||
|
||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
text = requests.get(f"{url}/metrics", timeout=30).text
|
||||
return {
|
||||
(sample.name, sample.labels["operation"]): sample.value
|
||||
for family in text_string_to_metric_families(text)
|
||||
for sample in family.samples
|
||||
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
|
||||
}
|
||||
|
||||
|
||||
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
args = [
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--api-server-count",
|
||||
"2",
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
|
||||
url = server.url_root
|
||||
trainer = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer),
|
||||
)
|
||||
for _ in range(SYNCS):
|
||||
engine.send_weights()
|
||||
metrics = scrape(url)
|
||||
|
||||
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
|
||||
for operation, count in calls.items():
|
||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||
@@ -1,306 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.instrumentator.metrics import (
|
||||
attach_router as attach_metrics_router,
|
||||
)
|
||||
from vllm.v1.metrics import prometheus as prometheus_metrics
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sleep_route_app(monkeypatch):
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
|
||||
lambda: metrics,
|
||||
)
|
||||
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
return app, metrics
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("level", [0, 1, 2])
|
||||
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
|
||||
app, _ = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/sleep", params={"level": level})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "sleeping", "level": level}
|
||||
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_param"),
|
||||
[
|
||||
("level=invalid", "query.level"),
|
||||
("level=-1", "query.level"),
|
||||
("level=3", "query.level"),
|
||||
("mode=invalid", "query.mode"),
|
||||
],
|
||||
)
|
||||
def test_sleep_route_rejects_invalid_query_before_dispatch(
|
||||
sleep_route_app, query, expected_param
|
||||
):
|
||||
app, metrics = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/sleep?{query}")
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["param"] == expected_param
|
||||
app.state.engine_client.sleep.assert_not_awaited()
|
||||
assert list(metrics.duration.collect()[0].samples) == []
|
||||
assert list(metrics.in_flight.collect()[0].samples) == []
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("fully_awake", "tags", "expected"),
|
||||
[
|
||||
(True, "", {"status": "awake", "tags": None}),
|
||||
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
|
||||
],
|
||||
)
|
||||
def test_wake_route_maps_engine_result_without_state_query(
|
||||
sleep_route_app, fully_awake, tags, expected
|
||||
):
|
||||
app, _ = sleep_route_app
|
||||
app.state.engine_client.wake_up.return_value = fully_awake
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/wake_up{tags}")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == expected
|
||||
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
|
||||
app.state.engine_client.is_sleeping.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(sleep_route_app, fails):
|
||||
app, metrics = sleep_route_app
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
if not fails:
|
||||
assert response.json() == {"status": "kv_cache_released"}
|
||||
release.assert_awaited_once_with()
|
||||
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
|
||||
assert _duration_count(metrics, "release_kv_cache_memory") == 1
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
|
||||
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
|
||||
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
with metrics.record(operation):
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 1
|
||||
with pytest.raises(exception), metrics.record(operation):
|
||||
raise exception()
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 0
|
||||
assert metrics.duration.labels(operation)._sum.get() >= 0
|
||||
assert _duration_count(metrics, operation) == 2
|
||||
|
||||
|
||||
def _duration_count(metrics, operation):
|
||||
return next(
|
||||
sample.value
|
||||
for sample in metrics.duration.collect()[0].samples
|
||||
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("path", "method", "operation", "status", "fails"),
|
||||
[
|
||||
("/sleep", "POST", "sleep", 200, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
|
||||
("/wake_up", "POST", "wake", 200, False),
|
||||
("/is_sleeping", "GET", None, 200, False),
|
||||
("/sleep?level=3", "POST", None, 400, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
|
||||
],
|
||||
)
|
||||
def test_sleep_routes_visible_on_production_metrics_endpoint(
|
||||
monkeypatch, path, method, operation, status, fails
|
||||
):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
|
||||
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
|
||||
monkeypatch.setattr(sleep_metrics, "_metrics", None)
|
||||
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
app.state.engine_client.wake_up.return_value = True
|
||||
app.state.engine_client.is_sleeping.return_value = False
|
||||
if fails:
|
||||
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
|
||||
"engine failed"
|
||||
)
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
attach_metrics_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
assert client.request(method, path).status_code == status
|
||||
response = client.get("/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
samples = [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
]
|
||||
assert any(
|
||||
sample.name == "http_requests_total"
|
||||
and sample.labels
|
||||
== {
|
||||
"handler": path.split("?")[0],
|
||||
"method": method,
|
||||
"status": f"{status // 100}xx",
|
||||
}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
)
|
||||
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operations_in_flight"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 0
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enable-sleep-mode",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
|
||||
) as remote_server:
|
||||
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
|
||||
assert response.status_code == 200
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 0
|
||||
assert weights_offloaded == 1
|
||||
assert discard_all == 0
|
||||
|
||||
response = requests.post(remote_server.url_for("wake_up"))
|
||||
assert response.status_code == 200
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is False
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 1
|
||||
assert weights_offloaded == 0
|
||||
assert discard_all == 0
|
||||
|
||||
# test wake up with tags
|
||||
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
|
||||
assert response.status_code == 200
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("wake_up"), params={"tags": ["weights"]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Partial wake keeps the engine sleeping.
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("wake_up"), params={"tags": ["kv_cache"]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is False
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 1
|
||||
assert weights_offloaded == 0
|
||||
assert discard_all == 0
|
||||
|
||||
|
||||
def _get_sleep_metrics_from_api(response: requests.Response):
|
||||
"""Return (awake, weights_offloaded, discard_all)."""
|
||||
awake, weights_offloaded, discard_all = None, None, None
|
||||
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
if family.name == "vllm:engine_sleep_state":
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:engine_sleep_state":
|
||||
for label_name, label_value in sample.labels.items():
|
||||
if label_value == "awake":
|
||||
awake = sample.value
|
||||
elif label_value == "weights_offloaded":
|
||||
weights_offloaded = sample.value
|
||||
elif label_value == "discard_all":
|
||||
discard_all = sample.value
|
||||
|
||||
assert awake is not None
|
||||
assert weights_offloaded is not None
|
||||
assert discard_all is not None
|
||||
|
||||
return awake, weights_offloaded, discard_all
|
||||
Reference in New Issue
Block a user