[watermarking hardening] Add e2e test and basic GSM8K quality tests (#56309)

Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tomas Ruiz
2026-09-14 11:29:13 -04:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1713a09491
commit b7e8dd8f37
4 changed files with 81 additions and 0 deletions
+16
View File
@@ -30,6 +30,22 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking"
device: h200_35gb
key: lm-eval-watermarking
# Recorded runs took 80–140 seconds
timeout_in_minutes: 10
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- vllm/config/watermarking.py
- vllm/sampling_params.py
- vllm/v1/watermarking/
- vllm/v1/worker/gpu/
autorun_on_main: true
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
# - label: LM Eval Large Models (4xA100)
# key: lm-eval-large-models-4xa100
# device: a100
@@ -0,0 +1,22 @@
model_name: "Qwen/Qwen3.5-4B"
# full GSM8K set is 1319 questions
num_questions: 1319
num_fewshot: 5
max_concurrency: 128
max_tokens: 1024
use_chat_completions: true
temperature: 0.7
seed: 42
# Calibrated by running non-watermaked 13 times
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
accuracy_threshold: 0.885
tolerance: 0.015
server_args: >-
--language-model-only
--max-model-len 8192
--max-num-seqs 128
--gpu-memory-utilization 0.4
--default-chat-template-kwargs '{"enable_thinking": false}'
--watermark-config '{"key": 42, "context_width": 4}'
@@ -0,0 +1 @@
Qwen3.5-4B-watermark.yaml
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
from vllm.v1.watermarking import GumbelWatermarkDetector
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_llm_generated_sequence_is_watermarked(vllm_runner):
"""Engine-level watermarking must produce tokens the detector recognizes."""
watermark_config = {"key": 42, "context_width": 4}
detector = GumbelWatermarkDetector(**watermark_config)
runner = vllm_runner(
"facebook/opt-125m",
watermark_config=watermark_config,
seed=0,
enforce_eager=True,
max_model_len=512,
gpu_memory_utilization=0.2,
)
request = "Tell me a story about an explorer who discovers a mysterious island."
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
with runner:
output_use_wm = runner.llm.generate(request, params_use_wm)
output_no_wm = runner.llm.generate(request, params_no_wm)
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
assert result_use_wm.is_watermarked, result_use_wm
recorded_p_value = 9.83e-20
assert result_use_wm.p_value < recorded_p_value * 10 # tolerance
assert not result_no_wm.is_watermarked, result_no_wm