[Frontend][RL] Track HTTP weight operation outcomes and concurrency (#55781)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
aoshen02
parent
ae66f9a97c
commit
713ec07655
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from prometheus_client.multiprocess import MultiProcessCollector
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
# Enough requests that the kernel spreads them over both API servers.
|
||||
SYNCS = 8
|
||||
|
||||
|
||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
text = requests.get(f"{url}/metrics", timeout=30).text
|
||||
return {
|
||||
(sample.name, sample.labels["operation"]): sample.value
|
||||
for family in text_string_to_metric_families(text)
|
||||
for sample in family.samples
|
||||
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
|
||||
}
|
||||
|
||||
|
||||
def pids_that_recorded(multiproc_dir: Path) -> set[str]:
|
||||
return {
|
||||
path.stem.rsplit("_", 1)[1]
|
||||
for path in multiproc_dir.glob("histogram_*.db")
|
||||
if any(
|
||||
metric.name == DURATION
|
||||
for metric in MultiProcessCollector.merge([str(path)])
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
args = [
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--api-server-count",
|
||||
"2",
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
|
||||
url = server.url_root
|
||||
trainer = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer),
|
||||
)
|
||||
for _ in range(SYNCS):
|
||||
engine.send_weights()
|
||||
metrics = scrape(url)
|
||||
|
||||
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
|
||||
for operation, count in calls.items():
|
||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||
# The counts above are sums over processes only if both servers recorded.
|
||||
assert len(pids_that_recorded(tmp_path)) == 2
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -29,7 +29,9 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
|
||||
)
|
||||
|
||||
|
||||
async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
async def _async_serving_models_init_with_mock() -> tuple[
|
||||
OpenAIServingModels, MagicMock
|
||||
]:
|
||||
mock_engine_client = MagicMock(spec=EngineClient)
|
||||
# Set the max_model_len attribute to avoid missing attribute
|
||||
mock_model_config = MagicMock(spec=ModelConfig)
|
||||
@@ -45,6 +47,13 @@ async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
)
|
||||
await serving_models.init_static_loras()
|
||||
|
||||
# The mock is returned separately: `engine_client` is typed as the protocol,
|
||||
# so assertion helpers are not visible through it.
|
||||
return serving_models, mock_engine_client
|
||||
|
||||
|
||||
async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
serving_models, _ = await _async_serving_models_init_with_mock()
|
||||
return serving_models
|
||||
|
||||
|
||||
@@ -103,7 +112,7 @@ async def test_load_lora_adapter_duplicate():
|
||||
@pytest.mark.parametrize("load_inplace", [False, True])
|
||||
async def test_load_lora_adapter_base_model_name(load_inplace: bool):
|
||||
"""A LoRA named after a served model would shadow it for all requests."""
|
||||
serving_models = await _async_serving_models_init()
|
||||
serving_models, mock_engine_client = await _async_serving_models_init_with_mock()
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace
|
||||
)
|
||||
@@ -112,8 +121,7 @@ async def test_load_lora_adapter_base_model_name(load_inplace: bool):
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
assert len(serving_models.lora_requests) == 0
|
||||
assert isinstance(serving_models.engine_client.add_lora, AsyncMock)
|
||||
serving_models.engine_client.add_lora.assert_not_called()
|
||||
mock_engine_client.add_lora.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
|
||||
from vllm.entrypoints.serve.dev.rlhf import api_router
|
||||
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
|
||||
from vllm.entrypoints.serve.dev.rlhf.metrics import WeightOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.v1.metrics.prometheus import get_prometheus_registry
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds_count"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
|
||||
|
||||
def sample(registry, name, operation):
|
||||
return registry.get_sample_value(name, {"operation": operation})
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, engine):
|
||||
self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine))
|
||||
|
||||
|
||||
class Engine:
|
||||
"""Holds every engine call open until the test releases it."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
|
||||
def __getattr__(self, name):
|
||||
async def call(*args, **kwargs):
|
||||
self.started.set()
|
||||
await self.release.wait()
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.setattr(rlhf_metrics, "_metrics", WeightOperationMetrics(registry))
|
||||
return registry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
def test_record_releases_in_flight_and_observes_duration(registry, fail):
|
||||
metrics = rlhf_metrics.weight_operation_metrics()
|
||||
with (
|
||||
pytest.raises(RuntimeError) if fail else nullcontext(),
|
||||
metrics.record("update"),
|
||||
):
|
||||
assert sample(registry, IN_FLIGHT, "update") == 1
|
||||
if fail:
|
||||
raise RuntimeError
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"operation,route,body",
|
||||
[
|
||||
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
|
||||
("start", api_router.start_weight_update, None),
|
||||
("start_draft", api_router.start_draft_weight_update, None),
|
||||
("update", api_router.update_weights, {"update_info": {}}),
|
||||
("finish", api_router.finish_weight_update, None),
|
||||
],
|
||||
)
|
||||
async def test_route_records_its_operation(registry, operation, route, body):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(route(Request(engine), **(body or {})))
|
||||
await engine.started.wait()
|
||||
assert sample(registry, IN_FLIGHT, operation) == 1
|
||||
engine.release.set()
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, operation) == 0
|
||||
assert sample(registry, DURATION, operation) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_route_releases_in_flight(registry):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(
|
||||
api_router.update_weights(Request(engine), update_info={})
|
||||
)
|
||||
await engine.started.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
HOLD_OPERATION = """
|
||||
import sys
|
||||
from vllm.entrypoints.serve.dev.rlhf.metrics import weight_operation_metrics
|
||||
with weight_operation_metrics().record("update"):
|
||||
print("entered", flush=True)
|
||||
sys.stdin.readline()
|
||||
"""
|
||||
|
||||
|
||||
def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch):
|
||||
"""Two API-server processes each hold an operation; /metrics sums them."""
|
||||
monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path))
|
||||
servers = [
|
||||
subprocess.Popen(
|
||||
[sys.executable, "-c", HOLD_OPERATION],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
env=os.environ.copy(),
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
try:
|
||||
for server in servers:
|
||||
assert server.stdout is not None
|
||||
assert server.stdout.readline().strip() == "entered"
|
||||
assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2
|
||||
finally:
|
||||
for server in servers:
|
||||
server.communicate("\n", timeout=60)
|
||||
|
||||
scraped = get_prometheus_registry()
|
||||
assert sample(scraped, IN_FLIGHT, "update") == 0
|
||||
assert sample(scraped, DURATION, "update") == 2
|
||||
|
||||
|
||||
def test_rejected_request_is_not_recorded(registry):
|
||||
app = FastAPI()
|
||||
app.include_router(api_router.router)
|
||||
init_exception_handler(app)
|
||||
app.state.engine_client = Engine()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
response = TestClient(app).post("/update_weights", json={})
|
||||
assert response.status_code == 400
|
||||
assert sample(registry, DURATION, "update") is None
|
||||
|
||||
|
||||
def test_default_recorder_is_created_lazily_on_the_default_registry():
|
||||
"""The default recorder registers on the process registry only when first used.
|
||||
|
||||
Runs in a fresh interpreter: the registry is process-global, so a route test
|
||||
elsewhere in the session may already have registered these collectors, and
|
||||
"not registered yet" would then be unobservable.
|
||||
"""
|
||||
probe = """
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
|
||||
|
||||
NAMES = [
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
]
|
||||
assert rlhf_metrics._metrics is None
|
||||
assert not set(NAMES) & set(REGISTRY._names_to_collectors)
|
||||
|
||||
metrics = rlhf_metrics.weight_operation_metrics()
|
||||
assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration
|
||||
assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight
|
||||
assert rlhf_metrics.weight_operation_metrics() is metrics
|
||||
print("lazy registration ok")
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", probe],
|
||||
cwd=Path(__file__).parents[3],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "lazy registration ok" in result.stdout
|
||||
Reference in New Issue
Block a user