[V1][Metrics] Add API for accessing in-memory Prometheus metrics (#17010)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -6,6 +6,7 @@ from typing import Optional
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
|
||||
MODEL = "facebook/opt-125m"
|
||||
DTYPE = "half"
|
||||
@@ -97,3 +98,67 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:
|
||||
raise AssertionError(
|
||||
f"{len(completion_counts)} unique completions; expected"
|
||||
f" {n}. Repeats: {repeats}")
|
||||
|
||||
|
||||
def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):
|
||||
max_tokens = 100
|
||||
# Use spec decoding to test num_accepted_tokens_per_pos
|
||||
speculative_config = {
|
||||
"method": "ngram",
|
||||
"prompt_lookup_max": 5,
|
||||
"prompt_lookup_min": 3,
|
||||
"num_speculative_tokens": 5,
|
||||
}
|
||||
monkeypatch.setenv("VLLM_USE_V1", "1")
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
speculative_config=speculative_config,
|
||||
disable_log_stats=False,
|
||||
) as vllm_model:
|
||||
model: LLM = vllm_model.model
|
||||
sampling_params = SamplingParams(temperature=0.0,
|
||||
max_tokens=max_tokens)
|
||||
outputs = model.generate(example_prompts, sampling_params)
|
||||
|
||||
n_prompts = len(example_prompts)
|
||||
assert len(outputs) == n_prompts
|
||||
|
||||
total_tokens = 0
|
||||
for out in outputs:
|
||||
assert len(out.outputs) == 1
|
||||
total_tokens += len(out.outputs[0].token_ids)
|
||||
assert total_tokens == max_tokens * n_prompts
|
||||
|
||||
metrics = model.get_metrics()
|
||||
|
||||
def find_metric(name) -> list[Metric]:
|
||||
found = []
|
||||
for metric in metrics:
|
||||
if metric.name == name:
|
||||
found.append(metric)
|
||||
return found
|
||||
|
||||
num_requests_running = find_metric("vllm:num_requests_running")
|
||||
assert len(num_requests_running) == 1
|
||||
assert isinstance(num_requests_running[0], Gauge)
|
||||
assert num_requests_running[0].value == .0
|
||||
|
||||
generation_tokens = find_metric("vllm:generation_tokens")
|
||||
assert len(generation_tokens) == 1
|
||||
assert isinstance(generation_tokens[0], Counter)
|
||||
assert generation_tokens[0].value == total_tokens
|
||||
|
||||
request_generation_tokens = find_metric(
|
||||
"vllm:request_generation_tokens")
|
||||
assert len(request_generation_tokens) == 1
|
||||
assert isinstance(request_generation_tokens[0], Histogram)
|
||||
assert "+Inf" in request_generation_tokens[0].buckets
|
||||
assert request_generation_tokens[0].buckets["+Inf"] == n_prompts
|
||||
assert request_generation_tokens[0].count == n_prompts
|
||||
assert request_generation_tokens[0].sum == total_tokens
|
||||
|
||||
num_accepted_tokens_per_pos = find_metric(
|
||||
"vllm:spec_decode_num_accepted_tokens_per_pos")
|
||||
assert len(num_accepted_tokens_per_pos) == 1
|
||||
assert isinstance(num_accepted_tokens_per_pos[0], Vector)
|
||||
assert len(num_accepted_tokens_per_pos[0].values) == 5
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import prometheus_client
|
||||
import pytest
|
||||
|
||||
from vllm.v1.metrics.reader import (Counter, Gauge, Histogram, Vector,
|
||||
get_metrics_snapshot)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def test_registry(monkeypatch):
|
||||
# Use a custom registry for tests
|
||||
test_registry = prometheus_client.CollectorRegistry(auto_describe=True)
|
||||
monkeypatch.setattr("vllm.v1.metrics.reader.REGISTRY", test_registry)
|
||||
return test_registry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_engines", [1, 4])
|
||||
def test_gauge_metric(test_registry, num_engines):
|
||||
g = prometheus_client.Gauge("vllm:test_gauge",
|
||||
"Test gauge metric",
|
||||
labelnames=["model", "engine_index"],
|
||||
registry=test_registry)
|
||||
for i in range(num_engines):
|
||||
g.labels(model="foo", engine_index=str(i)).set(98.5)
|
||||
|
||||
metrics = get_metrics_snapshot()
|
||||
assert len(metrics) == num_engines
|
||||
engine_labels = [str(i) for i in range(num_engines)]
|
||||
for m in metrics:
|
||||
assert isinstance(m, Gauge)
|
||||
assert m.name == "vllm:test_gauge"
|
||||
assert m.value == 98.5
|
||||
assert m.labels["model"] == "foo"
|
||||
assert m.labels["engine_index"] in engine_labels
|
||||
engine_labels.remove(m.labels["engine_index"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_engines", [1, 4])
|
||||
def test_counter_metric(test_registry, num_engines):
|
||||
c = prometheus_client.Counter("vllm:test_counter",
|
||||
"Test counter metric",
|
||||
labelnames=["model", "engine_index"],
|
||||
registry=test_registry)
|
||||
for i in range(num_engines):
|
||||
c.labels(model="bar", engine_index=str(i)).inc(19)
|
||||
|
||||
metrics = get_metrics_snapshot()
|
||||
assert len(metrics) == num_engines
|
||||
engine_labels = [str(i) for i in range(num_engines)]
|
||||
for m in metrics:
|
||||
assert isinstance(m, Counter)
|
||||
assert m.name == "vllm:test_counter"
|
||||
assert m.value == 19
|
||||
assert m.labels["model"] == "bar"
|
||||
assert m.labels["engine_index"] in engine_labels
|
||||
engine_labels.remove(m.labels["engine_index"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_engines", [1, 4])
|
||||
def test_histogram_metric(test_registry, num_engines):
|
||||
h = prometheus_client.Histogram("vllm:test_histogram",
|
||||
"Test histogram metric",
|
||||
labelnames=["model", "engine_index"],
|
||||
buckets=[10, 20, 30, 40, 50],
|
||||
registry=test_registry)
|
||||
for i in range(num_engines):
|
||||
hist = h.labels(model="blaa", engine_index=str(i))
|
||||
hist.observe(42)
|
||||
hist.observe(21)
|
||||
hist.observe(7)
|
||||
|
||||
metrics = get_metrics_snapshot()
|
||||
assert len(metrics) == num_engines
|
||||
engine_labels = [str(i) for i in range(num_engines)]
|
||||
for m in metrics:
|
||||
assert isinstance(m, Histogram)
|
||||
assert m.name == "vllm:test_histogram"
|
||||
assert m.count == 3
|
||||
assert m.sum == 70
|
||||
assert m.buckets["10.0"] == 1
|
||||
assert m.buckets["20.0"] == 1
|
||||
assert m.buckets["30.0"] == 2
|
||||
assert m.buckets["40.0"] == 2
|
||||
assert m.buckets["50.0"] == 3
|
||||
assert m.labels["model"] == "blaa"
|
||||
assert m.labels["engine_index"] in engine_labels
|
||||
engine_labels.remove(m.labels["engine_index"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_engines", [1, 4])
|
||||
def test_vector_metric(test_registry, num_engines):
|
||||
c = prometheus_client.Counter(
|
||||
"vllm:spec_decode_num_accepted_tokens_per_pos",
|
||||
"Vector-like counter metric",
|
||||
labelnames=["position", "model", "engine_index"],
|
||||
registry=test_registry)
|
||||
for i in range(num_engines):
|
||||
c.labels(position="0", model="llama", engine_index=str(i)).inc(10)
|
||||
c.labels(position="1", model="llama", engine_index=str(i)).inc(5)
|
||||
c.labels(position="2", model="llama", engine_index=str(i)).inc(1)
|
||||
|
||||
metrics = get_metrics_snapshot()
|
||||
assert len(metrics) == num_engines
|
||||
engine_labels = [str(i) for i in range(num_engines)]
|
||||
for m in metrics:
|
||||
assert isinstance(m, Vector)
|
||||
assert m.name == "vllm:spec_decode_num_accepted_tokens_per_pos"
|
||||
assert m.values == [10, 5, 1]
|
||||
assert m.labels["model"] == "llama"
|
||||
assert m.labels["engine_index"] in engine_labels
|
||||
engine_labels.remove(m.labels["engine_index"])
|
||||
Reference in New Issue
Block a user