[watermark] compatibility validation (#56801)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
This commit is contained in:
Raphaël Rialland
2026-10-05 14:37:13 -04:00
committed by GitHub
co-authored by Simon Veitner
parent 88f3c601b3
commit c32495d58f
45 changed files with 1670 additions and 101 deletions
@@ -2117,3 +2117,30 @@ class TestMidConversationToolChanges:
ValidationError, match='only allowed in messages with role "system"'
):
request_cls(model="test-model", max_tokens=128, **fields)
class TestWatermarking:
def test_defaults_to_unspecified(self):
request = _make_request([{"role": "user", "content": "hi"}])
assert _convert(request).watermarking is None
def test_forwards_explicit_enable(self):
request = _make_request([{"role": "user", "content": "hi"}], watermarking=True)
assert _convert(request).watermarking is True
def test_forwards_the_opt_out(self):
request = _make_request([{"role": "user", "content": "hi"}], watermarking=False)
assert _convert(request).watermarking is False
@pytest.mark.parametrize("watermarking", [None, True, False])
def test_reaches_sampling_params(self, watermarking):
request = _make_request(
[{"role": "user", "content": "hi"}], watermarking=watermarking
)
params = _convert(request).to_sampling_params(16, {})
assert params.watermarking is watermarking
@@ -2510,3 +2510,24 @@ class TestCreateErrorResponse:
assert err.error.message == "oops"
assert err.error.code == 400
assert err.error.type == "bad_request"
def test_watermarking_defaults_to_unspecified():
assert _convert(_make_request()).watermarking is None
def test_watermarking_explicit_enable_is_forwarded():
assert _convert(_make_request(watermarking=True)).watermarking is True
def test_watermarking_opt_out_is_forwarded():
assert _convert(_make_request(watermarking=False)).watermarking is False
@pytest.mark.parametrize("watermarking", [None, True, False])
def test_watermarking_reaches_sampling_params(watermarking):
request = _make_request(watermarking=watermarking)
params = _convert(request).to_sampling_params(16, {})
assert params.watermarking is watermarking
@@ -12,6 +12,7 @@ Tests cover:
import math
from dataclasses import dataclass, field
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
@@ -302,8 +303,10 @@ class TestGeneration:
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
mock_output = _create_mock_request_output(mock_logprobs)
state = SimpleNamespace(sampling_params=None)
async def mock_generate(*args, **kwargs):
state.sampling_params = args[1]
yield mock_output
mock_engine.generate = mock_generate
@@ -317,6 +320,8 @@ class TestGeneration:
result = await serving.create_generative_scoring(request, None)
assert isinstance(result, GenerativeScoringResponse)
assert state.sampling_params is not None
assert state.sampling_params.watermarking is False
assert len(result.data) == 2
for item_result in result.data:
assert 0.0 <= item_result.score <= 1.0
@@ -154,6 +154,40 @@ async def client(server):
yield async_client
@pytest.fixture(scope="module")
def watermarked_server():
args = [
"--dtype",
"half",
"--enable-auto-tool-choice",
"--structured-outputs-config.backend",
"xgrammar",
"--tool-call-parser",
"hermes",
"--reasoning-parser",
"qwen3",
"--gpu-memory-utilization",
"0.4",
"--enforce-eager",
"--watermark-config",
'{"algorithm":"gumbel","key":42}',
] + ROCM_EXTRA_ARGS
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(params=["server", "watermarked_server"], ids=["plain", "watermarked"])
def tool_use_server(request: pytest.FixtureRequest):
return request.getfixturevalue(request.param)
@pytest_asyncio.fixture
async def tool_use_client(tool_use_server):
async with tool_use_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("stream", [True, False])
@@ -167,12 +201,13 @@ async def client(server):
)
@pytest.mark.parametrize("enable_thinking", [True, False])
async def test_function_tool_use(
client: openai.AsyncOpenAI,
tool_use_client: openai.AsyncOpenAI,
model_name: str,
stream: bool,
tool_choice: str | dict,
enable_thinking: bool,
):
client = tool_use_client
if not stream:
# Non-streaming test
chat_completion = await client.chat.completions.create(
+224 -7
View File
@@ -1,32 +1,249 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from io import BytesIO
import pytest
from fastapi import UploadFile
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.speech_to_text.transcription.protocol import (
TranscriptionRequest,
)
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
from vllm.sampling_params import StructuredOutputsParams
def test_chat_request_disables_watermarking():
def test_chat_request_preserves_watermarking_opt_out():
request = ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}], watermarking=False
)
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
assert not params.watermarking
assert params.watermarking is False
def test_completion_request_disables_watermarking():
def test_completion_request_preserves_watermarking_opt_out():
request = CompletionRequest(prompt="hello", watermarking=False)
params = request.to_sampling_params(max_tokens=1)
assert not params.watermarking
assert params.watermarking is False
def test_responses_request_disables_watermarking():
def test_responses_request_preserves_watermarking_opt_out():
request = ResponsesRequest(input="hello", watermarking=False)
params = request.to_sampling_params(default_max_tokens=1)
assert not params.watermarking
assert params.watermarking is False
def test_batch_chat_request_preserves_watermarking_opt_out():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]], watermarking=False
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert params.watermarking is False
@pytest.mark.parametrize(
"api_request,conversion,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}],
use_beam_search=True,
watermarking=False,
),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=False),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_beam_search_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_beam_search_params",
{"default_max_tokens": 1},
),
],
)
def test_request_preserves_watermarking_opt_out(api_request, conversion, kwargs):
params = getattr(api_request, conversion)(**kwargs)
assert params.watermarking is False
@pytest.mark.parametrize("request_cls", [TranscriptionRequest, TranslationRequest])
def test_speech_requests_leave_watermarking_unspecified(request_cls):
request = request_cls(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
temperature=0.8,
)
sampling_params = request.to_sampling_params(default_max_tokens=1)
beam_params = request.to_beam_search_params(default_max_tokens=1)
assert request.watermarking is None
assert sampling_params.watermarking is None
assert beam_params.watermarking is None
@pytest.mark.parametrize(
"api_request,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "choose A or B"}],
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"max_tokens": 8, "default_sampling_params": {}},
),
(
CompletionRequest(
prompt="choose A or B",
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"max_tokens": 8},
),
(
ResponsesRequest(
input="choose A or B",
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"default_max_tokens": 8},
),
],
)
def test_request_preserves_unspecified_watermarking_with_structured_outputs(
api_request, kwargs
):
params = api_request.to_sampling_params(**kwargs)
assert params.watermarking is None
assert params.structured_outputs is not None
assert params.structured_outputs.choice == ["A", "B"]
@pytest.mark.parametrize(
"api_request,conversion,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}], watermarking=True
),
"to_sampling_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}],
use_beam_search=True,
watermarking=True,
),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
CompletionRequest(prompt="hello", watermarking=True),
"to_sampling_params",
{"max_tokens": 1},
),
(
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=True),
"to_beam_search_params",
{"max_tokens": 1},
),
(
ResponsesRequest(input="hello", watermarking=True),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=True,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=True,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
],
)
def test_request_preserves_explicit_watermarking(api_request, conversion, kwargs):
params = getattr(api_request, conversion)(**kwargs)
assert params.watermarking is True
def test_batch_chat_request_preserves_explicit_watermarking():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]], watermarking=True
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert converted.watermarking is True
assert params.watermarking is True
def test_batch_chat_request_inherits_watermarking():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]]
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert converted.watermarking is None
assert params.watermarking is None
@@ -4,6 +4,7 @@
import asyncio
import json
import warnings
from types import SimpleNamespace
import numpy as np
import pybase64 as base64
@@ -12,6 +13,7 @@ import websockets
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.assets.audio import AudioAsset
from vllm.entrypoints.speech_to_text.realtime.connection import RealtimeConnection
from vllm.multimodal.media.audio import load_audio
MISTRAL_FORMAT_ARGS = [
@@ -26,6 +28,46 @@ MISTRAL_FORMAT_ARGS = [
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
def test_realtime_generation_preserves_watermarking():
transcription_outputs = []
class EngineClient:
def generate(self, **kwargs):
async def outputs():
output = SimpleNamespace(
outputs=[],
prompt_token_ids=[],
is_watermarked=kwargs["sampling_params"].watermarking,
)
transcription_outputs.append(output)
yield output
return outputs()
async def streaming_input():
chunk: np.ndarray
for chunk in ():
yield chunk
async def send(_event):
return None
connection = object.__new__(RealtimeConnection)
connection.connection_id = "test"
connection.serving = SimpleNamespace(
model_cls=SimpleNamespace(realtime_max_tokens=1),
engine_client=EngineClient(),
)
connection.audio_queue = asyncio.Queue()
connection._is_connected = True
connection.send = send
asyncio.run(connection._run_generation(streaming_input(), asyncio.Queue()))
assert len(transcription_outputs) == 1
assert transcription_outputs[0].is_watermarked is None
def _get_websocket_url(server: RemoteOpenAIServer) -> str:
"""Convert HTTP URL to WebSocket URL for realtime endpoint."""
http_url = server.url_root
@@ -198,4 +198,6 @@ async def test_language_detection_cancel_aborts_engine_request():
with pytest.raises(asyncio.CancelledError):
await task
sampling_params = engine_client.generate.call_args.args[1]
assert sampling_params.watermarking is False
engine_client.abort.assert_awaited_once_with(request_id)