[watermarking hardening] Add e2e test and basic GSM8K quality tests (#56309)

Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tomas Ruiz
2026-09-14 11:29:13 -04:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1713a09491
commit b7e8dd8f37
4 changed files with 81 additions and 0 deletions
@@ -0,0 +1,22 @@
model_name: "Qwen/Qwen3.5-4B"
# full GSM8K set is 1319 questions
num_questions: 1319
num_fewshot: 5
max_concurrency: 128
max_tokens: 1024
use_chat_completions: true
temperature: 0.7
seed: 42
# Calibrated by running non-watermaked 13 times
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
accuracy_threshold: 0.885
tolerance: 0.015
server_args: >-
--language-model-only
--max-model-len 8192
--max-num-seqs 128
--gpu-memory-utilization 0.4
--default-chat-template-kwargs '{"enable_thinking": false}'
--watermark-config '{"key": 42, "context_width": 4}'
@@ -0,0 +1 @@
Qwen3.5-4B-watermark.yaml