[Metrics] Add admission rejection metrics (#55812)
Signed-off-by: Thuongvu Ho <thuongvuho@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
5bed289707
commit
99b44c7a42
@@ -188,6 +188,7 @@ EXPECTED_METRICS_V1 = [
|
||||
"vllm:prefix_cache_queries",
|
||||
"vllm:prefix_cache_hits",
|
||||
"vllm:num_preemptions_total",
|
||||
"vllm:admission_rejections_total",
|
||||
"vllm:prompt_tokens_total",
|
||||
"vllm:generation_tokens_total",
|
||||
"vllm:iteration_tokens_total",
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
These tests cover:
|
||||
- OutputProcessor.get_num_queued_tokens() token counting
|
||||
- AsyncLLM.check_admission() admission control logic
|
||||
- Admission rejection Prometheus samples
|
||||
- Exception classes (GracefulHTTPError, QueueOverflowError, MaxQueuedTokensError)
|
||||
- create_error_response() mapping GracefulHTTPError to HTTP 503
|
||||
- SchedulerConfig field defaults and validation
|
||||
@@ -14,6 +15,7 @@ These tests cover:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import multiprocessing
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
@@ -21,6 +23,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
@@ -40,6 +43,10 @@ from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.metrics.loggers import StatLoggerManager
|
||||
|
||||
_ADMISSION_METRIC_MODEL = "admission-metric-test"
|
||||
_ADMISSION_LOGGER = "vllm.v1.engine.async_llm"
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -70,10 +77,43 @@ def _make_async_llm(
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
|
||||
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
|
||||
llm.logger_manager = None
|
||||
llm.admission_stats = None
|
||||
return llm
|
||||
|
||||
|
||||
def _attach_prometheus_logger(
|
||||
llm: AsyncLLM, engine_indexes: list[int] | None = None
|
||||
) -> None:
|
||||
"""Attach a real Prometheus logger manager to a lightweight AsyncLLM."""
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.show_hidden_metrics = False
|
||||
vllm_config.observability_config.kv_cache_metrics = False
|
||||
vllm_config.observability_config.custom_histogram_buckets = None
|
||||
vllm_config.model_config.served_model_name = _ADMISSION_METRIC_MODEL
|
||||
vllm_config.model_config.max_model_len = 2048
|
||||
vllm_config.model_config.is_diffusion = False
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
vllm_config.ec_transfer_config = None
|
||||
vllm_config.lora_config = None
|
||||
llm.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
engine_idxs=engine_indexes,
|
||||
enable_default_loggers=False,
|
||||
)
|
||||
|
||||
|
||||
def _admission_rejection_count(reason: str) -> float:
|
||||
"""Read the Prometheus sample for one admission rejection reason."""
|
||||
value = REGISTRY.get_sample_value(
|
||||
"vllm:admission_rejections_total",
|
||||
{"model_name": _ADMISSION_METRIC_MODEL, "reason": reason},
|
||||
)
|
||||
assert value is not None
|
||||
return value
|
||||
|
||||
|
||||
def _make_output_processor(**request_states) -> OutputProcessor:
|
||||
op = OutputProcessor.__new__(OutputProcessor)
|
||||
op.request_states = request_states
|
||||
@@ -157,11 +197,13 @@ def test_shared_request_snapshot_enforces_global_limit():
|
||||
server_0.set_num_requests(2)
|
||||
llm = _make_async_llm(max_num_queued_reqs=2)
|
||||
llm.admission_stats = server_1
|
||||
_attach_prometheus_logger(llm)
|
||||
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
|
||||
assert server_0.get_num_requests() == 2
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -247,19 +289,35 @@ def test_admission_no_limits_allows_everything():
|
||||
|
||||
def test_admission_reqs_allows_when_under_limit():
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=5)
|
||||
_attach_prometheus_logger(llm)
|
||||
llm.check_admission()
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 0
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
def test_admission_reqs_rejects_at_limit():
|
||||
def test_admission_reqs_rejects_at_limit(caplog):
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=10)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
_attach_prometheus_logger(llm, engine_indexes=[0, 1, 2])
|
||||
with (
|
||||
caplog.at_level(logging.INFO, logger=_ADMISSION_LOGGER),
|
||||
pytest.raises(QueueOverflowError),
|
||||
):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
assert not any(
|
||||
"Request queue full" in record.getMessage() for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_admission_reqs_rejects_with_n():
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=8)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission(3)
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
def test_admission_reqs_allows_n_at_boundary():
|
||||
@@ -283,8 +341,11 @@ def test_admission_tokens_allows_when_under_limit():
|
||||
|
||||
def test_admission_tokens_rejects_at_limit():
|
||||
llm = _make_async_llm(max_num_queued_tokens=1000, num_queued_tokens=1000)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(MaxQueuedTokensError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 0
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 1
|
||||
|
||||
|
||||
def test_admission_tokens_rejects_over_limit():
|
||||
@@ -320,8 +381,11 @@ def test_admission_req_limit_checked_before_token_limit():
|
||||
num_unfinished=10,
|
||||
num_queued_tokens=1000,
|
||||
)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
# -- n derived from params at the add_request call site ---------------------
|
||||
@@ -381,6 +445,17 @@ async def test_concurrent_single_request_admission_respects_limit():
|
||||
assert len(request_states) == 1
|
||||
|
||||
|
||||
def test_admission_preflight_pass_then_reject_records_once():
|
||||
llm = _make_async_llm(max_num_queued_reqs=1, num_unfinished=0)
|
||||
_attach_prometheus_logger(llm)
|
||||
llm.check_admission()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = 1
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("first_n", [1, 3])
|
||||
async def test_parallel_admission_is_all_or_nothing(first_n: int):
|
||||
|
||||
Reference in New Issue
Block a user