[watermarking hardening] Add e2e test and basic GSM8K quality tests (#56309)
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1713a09491
commit
b7e8dd8f37
@@ -30,6 +30,22 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking"
|
||||
device: h200_35gb
|
||||
key: lm-eval-watermarking
|
||||
# Recorded runs took 80–140 seconds
|
||||
timeout_in_minutes: 10
|
||||
source_file_dependencies:
|
||||
- .buildkite/test_areas/lm_eval.yaml
|
||||
- tests/evals/gsm8k/
|
||||
- vllm/config/watermarking.py
|
||||
- vllm/sampling_params.py
|
||||
- vllm/v1/watermarking/
|
||||
- vllm/v1/worker/gpu/
|
||||
autorun_on_main: true
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
|
||||
|
||||
# - label: LM Eval Large Models (4xA100)
|
||||
# key: lm-eval-large-models-4xa100
|
||||
# device: a100
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
model_name: "Qwen/Qwen3.5-4B"
|
||||
# full GSM8K set is 1319 questions
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 128
|
||||
max_tokens: 1024
|
||||
use_chat_completions: true
|
||||
temperature: 0.7
|
||||
seed: 42
|
||||
|
||||
# Calibrated by running non-watermaked 13 times
|
||||
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
|
||||
accuracy_threshold: 0.885
|
||||
tolerance: 0.015
|
||||
|
||||
server_args: >-
|
||||
--language-model-only
|
||||
--max-model-len 8192
|
||||
--max-num-seqs 128
|
||||
--gpu-memory-utilization 0.4
|
||||
--default-chat-template-kwargs '{"enable_thinking": false}'
|
||||
--watermark-config '{"key": 42, "context_width": 4}'
|
||||
@@ -0,0 +1 @@
|
||||
Qwen3.5-4B-watermark.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking import GumbelWatermarkDetector
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_llm_generated_sequence_is_watermarked(vllm_runner):
|
||||
"""Engine-level watermarking must produce tokens the detector recognizes."""
|
||||
watermark_config = {"key": 42, "context_width": 4}
|
||||
detector = GumbelWatermarkDetector(**watermark_config)
|
||||
|
||||
runner = vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
watermark_config=watermark_config,
|
||||
seed=0,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.2,
|
||||
)
|
||||
request = "Tell me a story about an explorer who discovers a mysterious island."
|
||||
params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True)
|
||||
params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False)
|
||||
|
||||
with runner:
|
||||
output_use_wm = runner.llm.generate(request, params_use_wm)
|
||||
output_no_wm = runner.llm.generate(request, params_no_wm)
|
||||
|
||||
result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids))
|
||||
result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids))
|
||||
|
||||
assert result_use_wm.is_watermarked, result_use_wm
|
||||
recorded_p_value = 9.83e-20
|
||||
assert result_use_wm.p_value < recorded_p_value * 10 # tolerance
|
||||
|
||||
assert not result_no_wm.is_watermarked, result_no_wm
|
||||
Reference in New Issue
Block a user