diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 3c65611966..d9afefc82e 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -30,6 +30,22 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py +- label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking" + device: h200_35gb + key: lm-eval-watermarking + # Recorded runs took 80–140 seconds + timeout_in_minutes: 10 + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - vllm/config/watermarking.py + - vllm/sampling_params.py + - vllm/v1/watermarking/ + - vllm/v1/worker/gpu/ + autorun_on_main: true + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt + # - label: LM Eval Large Models (4xA100) # key: lm-eval-large-models-4xa100 # device: a100 diff --git a/tests/evals/gsm8k/configs/Qwen3.5-4B-watermark.yaml b/tests/evals/gsm8k/configs/Qwen3.5-4B-watermark.yaml new file mode 100644 index 0000000000..bdd42a0f1a --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3.5-4B-watermark.yaml @@ -0,0 +1,22 @@ +model_name: "Qwen/Qwen3.5-4B" +# full GSM8K set is 1319 questions +num_questions: 1319 +num_fewshot: 5 +max_concurrency: 128 +max_tokens: 1024 +use_chat_completions: true +temperature: 0.7 +seed: 42 + +# Calibrated by running non-watermaked 13 times +# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations) +accuracy_threshold: 0.885 +tolerance: 0.015 + +server_args: >- + --language-model-only + --max-model-len 8192 + --max-num-seqs 128 + --gpu-memory-utilization 0.4 + --default-chat-template-kwargs '{"enable_thinking": false}' + --watermark-config '{"key": 42, "context_width": 4}' diff --git a/tests/evals/gsm8k/configs/models-watermark.txt b/tests/evals/gsm8k/configs/models-watermark.txt new file mode 100644 index 0000000000..a761b41c1b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-watermark.txt @@ -0,0 +1 @@ +Qwen3.5-4B-watermark.yaml diff --git a/tests/watermarking/test_watermarking_e2e.py b/tests/watermarking/test_watermarking_e2e.py new file mode 100644 index 0000000000..c73d48f6b6 --- /dev/null +++ b/tests/watermarking/test_watermarking_e2e.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm import SamplingParams +from vllm.platforms import current_platform +from vllm.v1.watermarking import GumbelWatermarkDetector + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_llm_generated_sequence_is_watermarked(vllm_runner): + """Engine-level watermarking must produce tokens the detector recognizes.""" + watermark_config = {"key": 42, "context_width": 4} + detector = GumbelWatermarkDetector(**watermark_config) + + runner = vllm_runner( + "facebook/opt-125m", + watermark_config=watermark_config, + seed=0, + enforce_eager=True, + max_model_len=512, + gpu_memory_utilization=0.2, + ) + request = "Tell me a story about an explorer who discovers a mysterious island." + params_use_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=True) + params_no_wm = SamplingParams(temperature=1.0, max_tokens=256, watermarking=False) + + with runner: + output_use_wm = runner.llm.generate(request, params_use_wm) + output_no_wm = runner.llm.generate(request, params_no_wm) + + result_use_wm = detector.detect(list(output_use_wm[0].outputs[0].token_ids)) + result_no_wm = detector.detect(list(output_no_wm[0].outputs[0].token_ids)) + + assert result_use_wm.is_watermarked, result_use_wm + recorded_p_value = 9.83e-20 + assert result_use_wm.p_value < recorded_p_value * 10 # tolerance + + assert not result_no_wm.is_watermarked, result_no_wm