[watermarking hardening] Add e2e test and basic GSM8K quality tests (#56309)

Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tomas Ruiz
2026-09-14 11:29:13 -04:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1713a09491
commit b7e8dd8f37
4 changed files with 81 additions and 0 deletions
@@ -0,0 +1,22 @@
model_name: "Qwen/Qwen3.5-4B"
# full GSM8K set is 1319 questions
num_questions: 1319
num_fewshot: 5
max_concurrency: 128
max_tokens: 1024
use_chat_completions: true
temperature: 0.7
seed: 42
# Calibrated by running non-watermaked 13 times
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
accuracy_threshold: 0.885
tolerance: 0.015
server_args: >-
--language-model-only
--max-model-len 8192
--max-num-seqs 128
--gpu-memory-utilization 0.4
--default-chat-template-kwargs '{"enable_thinking": false}'
--watermark-config '{"key": 42, "context_width": 4}'
@@ -0,0 +1 @@
Qwen3.5-4B-watermark.yaml
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
from vllm.v1.watermarking import GumbelWatermarkDetector
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_llm_generated_sequence_is_watermarked(vllm_runner):
"""Engine-level watermarking must produce tokens the detector recognizes."""
watermark_config = {"key": 42, "context_width": 4}
detector = GumbelWatermarkDetector(**watermark_config)
runner = vllm_runner(
"facebook/opt-125m",
watermark_config=watermark_config,
seed=0,
enforce_eager=True,
max_model_len=512,
gpu_memory_utilization=0.2,
)
request = "Tell me a story about an explorer who discovers a mysterious island."
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
with runner:
output_use_wm = runner.llm.generate(request, params_use_wm)
output_no_wm = runner.llm.generate(request, params_no_wm)
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
assert result_use_wm.is_watermarked, result_use_wm
recorded_p_value = 9.83e-20
assert result_use_wm.p_value < recorded_p_value * 10 # tolerance
assert not result_no_wm.is_watermarked, result_no_wm