[Metrics] Count finished requests by watermark status (#58715)

Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
simon-veitner-redhat
2026-10-12 17:21:22 +00:00
committed by GitHub
co-authored by Claude Lucas Wilkinson
parent 45bb710eca
commit a7b7bc1fbe
7 changed files with 203 additions and 1 deletions
+28
View File
@@ -56,6 +56,34 @@ watermarking algorithm that does not support it natively, at the cost of
weaker detectability. See weaker detectability. See
[Speculative decoding](#speculative-decoding). [Speculative decoding](#speculative-decoding).
## Monitoring
When `watermark_config` is set, the Prometheus counter
`vllm:watermark_requests_total` counts finished requests by `status`:
- `watermarked`: the request was sampled with the watermark switched on.
- `skipped`: the request asked for the watermark (omitted or `true`) but could
not use it, such as `temperature=0` and trace replay requests. A beam search
request that asks for the watermark runs without it and logs a warning. Its
beam steps count as `not_watermarked`, not `skipped`.
- `not_watermarked`: all other finished requests. This includes explicit
opt-outs, pooling requests, and the internal requests of beam search (one
per beam step), generative scoring and language detection.
Each finished request has exactly one status, so the sum over `status` equals
`vllm:request_success_total` on the same instance. Requests that the client
aborts are in neither counter. The counter does not exist without
`watermark_config`. With `n > 1`, each sequence counts once. In a
disaggregated prefill/decode deployment, both instances count a request: sum
over decode instances only. For streaming input, the first input chunk sets
the status.
The status is per request, not per token. A `watermarked` request can contain
tokens without the watermark: accepted draft tokens with
`allow_target_only_watermarking`, tokens whose context repeats
(`deduplicate_contexts`), the first `context_width` generated tokens with
`deduplicate_contexts: "all"`, and tokens that a grammar forces.
## Architecture ## Architecture
`WatermarkConfig` selects an algorithm and PRF. Model Runner V2 constructs the `WatermarkConfig` selects an algorithm and PRF. Model Runner V2 constructs the
+94
View File
@@ -6,6 +6,7 @@ import time
from unittest.mock import MagicMock, Mock from unittest.mock import MagicMock, Mock
import numpy as np import numpy as np
import prometheus_client
import pytest import pytest
import torch import torch
@@ -17,6 +18,7 @@ from tests.v1.engine.utils import (
MockEngineCore, MockEngineCore,
) )
from vllm import PoolingParams from vllm import PoolingParams
from vllm.config.watermarking import WatermarkConfig
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
from vllm.lora.request import LoRARequest from vllm.lora.request import LoRARequest
from vllm.outputs import ( from vllm.outputs import (
@@ -41,6 +43,9 @@ from vllm.v1.engine.output_processor import (
RequestOutputCollector, RequestOutputCollector,
RequestState, RequestState,
) )
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.loggers import PrometheusStatLogger
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
from vllm.v1.outputs import SamplingMaskLists from vllm.v1.outputs import SamplingMaskLists
@@ -1169,6 +1174,95 @@ def test_iteration_stats(dummy_test_vectors):
assert iteration_stats.num_generation_tokens == num_active assert iteration_stats.num_generation_tokens == num_active
def test_watermark_status_partitions_finished_requests(dummy_test_vectors):
"""Each finished request gets exactly one watermark status."""
output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=True)
prompt_token_ids = dummy_test_vectors.prompt_tokens[0]
def make_request(
request_id: str,
sampling_params: SamplingParams | None = None,
pooling_params: PoolingParams | None = None,
) -> EngineCoreRequest:
return EngineCoreRequest(
request_id=request_id,
external_req_id=f"{request_id}-ext",
prompt_token_ids=prompt_token_ids,
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=sampling_params,
pooling_params=pooling_params,
)
# The frontend resolves the flags before the request reaches the
# OutputProcessor; set them as it does.
def resolved(watermarking: bool, skipped: bool, n: int = 1) -> SamplingParams:
params = SamplingParams(n=n, watermarking=watermarking)
params._watermarking_skipped = skipped
return params
finish_reasons = {
"wm": FinishReason.ABORT,
"skip": FinishReason.STOP,
# A re-resolved request can carry both flags; it is watermarked.
"sticky": FinishReason.STOP,
"off": FinishReason.ERROR,
"pool": FinishReason.STOP,
}
output_processor.add_request(make_request("wm", resolved(True, False)), None)
output_processor.add_request(make_request("skip", resolved(False, True)), None)
output_processor.add_request(make_request("sticky", resolved(True, True)), None)
output_processor.add_request(make_request("off", resolved(False, False)), None)
output_processor.add_request(
make_request("pool", pooling_params=PoolingParams(task="embed")), None
)
parent_req = ParentRequest(make_request("parent", resolved(True, False, n=2)))
for idx in range(2):
child_id, child_params = parent_req.get_child_info(idx)
output_processor.add_request(
make_request(child_id, child_params), None, parent_req, idx
)
finish_reasons[child_id] = FinishReason.LENGTH
outputs = [
EngineCoreOutput(
request_id=request_id,
new_token_ids=[] if request_id == "pool" else [prompt_token_ids[0]],
pooling_output=torch.tensor([1.0]) if request_id == "pool" else None,
finish_reason=finish_reason,
)
for request_id, finish_reason in finish_reasons.items()
]
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, time.monotonic(), iteration_stats)
vllm_config = dummy_test_vectors.vllm_config
vllm_config.watermark_config = WatermarkConfig(key=42)
try:
PrometheusStatLogger(vllm_config).record(
scheduler_stats=None, iteration_stats=iteration_stats
)
totals: dict[str, dict[str, float]] = {}
for metric in prometheus_client.REGISTRY.collect():
for sample in metric.samples:
if sample.name in (
"vllm:watermark_requests_total",
"vllm:request_success_total",
):
label = sample.labels.get("status", "all")
by_label = totals.setdefault(sample.name, {})
by_label[label] = by_label.get(label, 0) + sample.value
finally:
unregister_vllm_metrics()
status = totals["vllm:watermark_requests_total"]
assert status == {"watermarked": 4, "skipped": 1, "not_watermarked": 2}
assert sum(status.values()) == totals["vllm:request_success_total"]["all"]
@pytest.mark.parametrize("log_stats", [True, False]) @pytest.mark.parametrize("log_stats", [True, False])
def test_lora_request_tracking(log_stats: bool, dummy_test_vectors): def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
"""Test LoRA request lifecycle tracking through waiting -> running -> finished.""" """Test LoRA request lifecycle tracking through waiting -> running -> finished."""
@@ -11,6 +11,7 @@ import pytest
from vllm.config import ModelConfig, VllmConfig from vllm.config import ModelConfig, VllmConfig
from vllm.config.observability import ObservabilityConfig from vllm.config.observability import ObservabilityConfig
from vllm.config.watermarking import WatermarkConfig
from vllm.v1.metrics.buckets import ( from vllm.v1.metrics.buckets import (
BUCKET_FAMILY_KEYS, BUCKET_FAMILY_KEYS,
build_1_2_5_buckets, build_1_2_5_buckets,
@@ -317,3 +318,16 @@ def test_prometheus_logger_applies_overrides():
assert found[metric_name] == overridden[family], metric_name assert found[metric_name] == overridden[family], metric_name
finally: finally:
unregister_vllm_metrics() unregister_vllm_metrics()
@pytest.mark.parametrize("with_config", [True, False])
def test_watermark_requests_counter_registration(with_config: bool):
"""The watermark counter must exist only with watermark_config."""
config = build_logger_config(ObservabilityConfig())
config.watermark_config = WatermarkConfig(key=42) if with_config else None
try:
PrometheusStatLogger(config)
names = {m.name for m in prometheus_client.REGISTRY.collect()}
assert ("vllm:watermark_requests" in names) == with_config
finally:
unregister_vllm_metrics()
@@ -5,6 +5,7 @@ import pytest
from vllm import SamplingParams from vllm import SamplingParams
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.v1.metrics.reader import Counter
from vllm.v1.watermarking import GumbelWatermarkDetector from vllm.v1.watermarking import GumbelWatermarkDetector
@@ -23,14 +24,29 @@ def test_llm_generated_sequence_is_watermarked(vllm_runner):
enforce_eager=True, enforce_eager=True,
max_model_len=512, max_model_len=512,
gpu_memory_utilization=0.2, gpu_memory_utilization=0.2,
disable_log_stats=False,
) )
request = "Tell me a story about an explorer who discovers a mysterious island." request = "Tell me a story about an explorer who discovers a mysterious island."
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True) params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False) params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
params_greedy = SamplingParams(temperature=0.0, max_tokens=16)
with runner: with runner:
output_use_wm = runner.llm.generate(request, params_use_wm) output_use_wm = runner.llm.generate(request, params_use_wm)
output_no_wm = runner.llm.generate(request, params_no_wm) output_no_wm = runner.llm.generate(request, params_no_wm)
runner.llm.generate(request, params_greedy)
metrics = runner.llm.get_metrics()
# The greedy request is skipped; the opt-out one is not watermarked.
counters = [m for m in metrics if isinstance(m, Counter)]
status = {
m.labels["status"]: m.value
for m in counters
if m.name == "vllm:watermark_requests"
}
assert status == {"watermarked": 1, "skipped": 1, "not_watermarked": 1}
num_success = sum(m.value for m in counters if m.name == "vllm:request_success")
assert sum(status.values()) == num_success
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids)) result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids)) result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
+5 -1
View File
@@ -276,7 +276,7 @@ class RequestState:
output_kind = request.pooling_params.output_kind output_kind = request.pooling_params.output_kind
assert request.external_req_id is not None assert request.external_req_id is not None
return cls( state = cls(
request_id=request.request_id, request_id=request.request_id,
external_req_id=request.external_req_id, external_req_id=request.external_req_id,
parent_req=parent_req, parent_req=parent_req,
@@ -299,6 +299,10 @@ class RequestState:
stream_input=request.resumable, stream_input=request.resumable,
remote_prefill_cached_tokens=remote_prefill_cached_tokens, remote_prefill_cached_tokens=remote_prefill_cached_tokens,
) )
if state.stats is not None and sampling_params is not None:
state.stats.is_watermarked = bool(sampling_params.watermarking)
state.stats.is_watermark_skipped = sampling_params._watermarking_skipped
return state
def make_request_output( def make_request_output(
self, self,
+30
View File
@@ -639,6 +639,25 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
counter_corrupted_requests, per_engine_labelvalues counter_corrupted_requests, per_engine_labelvalues
) )
if vllm_config.watermark_config is not None:
counter_watermark_requests = self._counter_cls(
name="vllm:watermark_requests",
documentation=(
"Finished requests by watermark status "
"(only with watermark_config)."
),
labelnames=labelnames + ["status"],
)
self.counter_watermark_requests: dict[str, dict[int, Counter]] = {}
for status in ("watermarked", "skipped", "not_watermarked"):
per_engine_labelvalues_with_status = {
idx: labelvalues + [status]
for idx, labelvalues in per_engine_labelvalues.items()
}
self.counter_watermark_requests[status] = create_metric_per_engine(
counter_watermark_requests, per_engine_labelvalues_with_status
)
counter_prefix_cache_queries = self._counter_cls( counter_prefix_cache_queries = self._counter_cls(
name="vllm:prefix_cache_queries", name="vllm:prefix_cache_queries",
documentation=( documentation=(
@@ -1226,6 +1245,17 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
self.counter_corrupted_requests[engine_idx].inc( self.counter_corrupted_requests[engine_idx].inc(
iteration_stats.num_corrupted_reqs iteration_stats.num_corrupted_reqs
) )
if self.vllm_config.watermark_config is not None:
counter_watermark = self.counter_watermark_requests
counter_watermark["watermarked"][engine_idx].inc(
iteration_stats.num_watermarked_reqs
)
counter_watermark["skipped"][engine_idx].inc(
iteration_stats.num_watermark_skipped_reqs
)
counter_watermark["not_watermarked"][engine_idx].inc(
iteration_stats.num_not_watermarked_reqs
)
self.counter_num_preempted_reqs[engine_idx].inc( self.counter_num_preempted_reqs[engine_idx].inc(
iteration_stats.num_preempted_reqs iteration_stats.num_preempted_reqs
) )
+16
View File
@@ -277,6 +277,12 @@ class RequestStateStats:
# Track if this request is corrupted (NaNs in logits) # Track if this request is corrupted (NaNs in logits)
is_corrupted: bool = False is_corrupted: bool = False
# Track if this request was sampled with the engine watermark
is_watermarked: bool = False
# Track if this request asked for the watermark but could not use it
is_watermark_skipped: bool = False
@dataclass @dataclass
class FinishedRequestStats: class FinishedRequestStats:
@@ -479,6 +485,9 @@ class IterationStats:
self.time_to_first_tokens_iter: list[float] = [] self.time_to_first_tokens_iter: list[float] = []
self.inter_token_latencies_iter: list[float] = [] self.inter_token_latencies_iter: list[float] = []
self.num_corrupted_reqs: int = 0 self.num_corrupted_reqs: int = 0
self.num_watermarked_reqs: int = 0
self.num_watermark_skipped_reqs: int = 0
self.num_not_watermarked_reqs: int = 0
def __repr__(self) -> str: def __repr__(self) -> str:
field_to_value_str = ", ".join(f"{k}={v}" for k, v in vars(self).items()) field_to_value_str = ", ".join(f"{k}={v}" for k, v in vars(self).items())
@@ -624,6 +633,13 @@ class IterationStats:
if req_stats.is_corrupted: if req_stats.is_corrupted:
self.num_corrupted_reqs += 1 self.num_corrupted_reqs += 1
if req_stats.is_watermarked:
self.num_watermarked_reqs += 1
elif req_stats.is_watermark_skipped:
self.num_watermark_skipped_reqs += 1
else:
self.num_not_watermarked_reqs += 1
class LoRAStats: class LoRAStats:
"""Tracks waiting and running request IDs for a single LoRA.""" """Tracks waiting and running request IDs for a single LoRA."""