[watermark] compatibility validation (#56801)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Simon Veitner <sveitner@redhat.com>
This commit is contained in:
co-authored by
Simon Veitner
parent
88f3c601b3
commit
c32495d58f
@@ -2117,3 +2117,30 @@ class TestMidConversationToolChanges:
|
||||
ValidationError, match='only allowed in messages with role "system"'
|
||||
):
|
||||
request_cls(model="test-model", max_tokens=128, **fields)
|
||||
|
||||
|
||||
class TestWatermarking:
|
||||
def test_defaults_to_unspecified(self):
|
||||
request = _make_request([{"role": "user", "content": "hi"}])
|
||||
|
||||
assert _convert(request).watermarking is None
|
||||
|
||||
def test_forwards_explicit_enable(self):
|
||||
request = _make_request([{"role": "user", "content": "hi"}], watermarking=True)
|
||||
|
||||
assert _convert(request).watermarking is True
|
||||
|
||||
def test_forwards_the_opt_out(self):
|
||||
request = _make_request([{"role": "user", "content": "hi"}], watermarking=False)
|
||||
|
||||
assert _convert(request).watermarking is False
|
||||
|
||||
@pytest.mark.parametrize("watermarking", [None, True, False])
|
||||
def test_reaches_sampling_params(self, watermarking):
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "hi"}], watermarking=watermarking
|
||||
)
|
||||
|
||||
params = _convert(request).to_sampling_params(16, {})
|
||||
|
||||
assert params.watermarking is watermarking
|
||||
|
||||
@@ -2510,3 +2510,24 @@ class TestCreateErrorResponse:
|
||||
assert err.error.message == "oops"
|
||||
assert err.error.code == 400
|
||||
assert err.error.type == "bad_request"
|
||||
|
||||
|
||||
def test_watermarking_defaults_to_unspecified():
|
||||
assert _convert(_make_request()).watermarking is None
|
||||
|
||||
|
||||
def test_watermarking_explicit_enable_is_forwarded():
|
||||
assert _convert(_make_request(watermarking=True)).watermarking is True
|
||||
|
||||
|
||||
def test_watermarking_opt_out_is_forwarded():
|
||||
assert _convert(_make_request(watermarking=False)).watermarking is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("watermarking", [None, True, False])
|
||||
def test_watermarking_reaches_sampling_params(watermarking):
|
||||
request = _make_request(watermarking=watermarking)
|
||||
|
||||
params = _convert(request).to_sampling_params(16, {})
|
||||
|
||||
assert params.watermarking is watermarking
|
||||
|
||||
@@ -12,6 +12,7 @@ Tests cover:
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -302,8 +303,10 @@ class TestGeneration:
|
||||
|
||||
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
|
||||
mock_output = _create_mock_request_output(mock_logprobs)
|
||||
state = SimpleNamespace(sampling_params=None)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
state.sampling_params = args[1]
|
||||
yield mock_output
|
||||
|
||||
mock_engine.generate = mock_generate
|
||||
@@ -317,6 +320,8 @@ class TestGeneration:
|
||||
result = await serving.create_generative_scoring(request, None)
|
||||
|
||||
assert isinstance(result, GenerativeScoringResponse)
|
||||
assert state.sampling_params is not None
|
||||
assert state.sampling_params.watermarking is False
|
||||
assert len(result.data) == 2
|
||||
for item_result in result.data:
|
||||
assert 0.0 <= item_result.score <= 1.0
|
||||
|
||||
@@ -154,6 +154,40 @@ async def client(server):
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def watermarked_server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"half",
|
||||
"--enable-auto-tool-choice",
|
||||
"--structured-outputs-config.backend",
|
||||
"xgrammar",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--enforce-eager",
|
||||
"--watermark-config",
|
||||
'{"algorithm":"gumbel","key":42}',
|
||||
] + ROCM_EXTRA_ARGS
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(params=["server", "watermarked_server"], ids=["plain", "watermarked"])
|
||||
def tool_use_server(request: pytest.FixtureRequest):
|
||||
return request.getfixturevalue(request.param)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def tool_use_client(tool_use_server):
|
||||
async with tool_use_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("stream", [True, False])
|
||||
@@ -167,12 +201,13 @@ async def client(server):
|
||||
)
|
||||
@pytest.mark.parametrize("enable_thinking", [True, False])
|
||||
async def test_function_tool_use(
|
||||
client: openai.AsyncOpenAI,
|
||||
tool_use_client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
stream: bool,
|
||||
tool_choice: str | dict,
|
||||
enable_thinking: bool,
|
||||
):
|
||||
client = tool_use_client
|
||||
if not stream:
|
||||
# Non-streaming test
|
||||
chat_completion = await client.chat.completions.create(
|
||||
|
||||
@@ -1,32 +1,249 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from fastapi import UploadFile
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
|
||||
def test_chat_request_disables_watermarking():
|
||||
def test_chat_request_preserves_watermarking_opt_out():
|
||||
request = ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}], watermarking=False
|
||||
)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert not params.watermarking
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_completion_request_disables_watermarking():
|
||||
def test_completion_request_preserves_watermarking_opt_out():
|
||||
request = CompletionRequest(prompt="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1)
|
||||
|
||||
assert not params.watermarking
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_responses_request_disables_watermarking():
|
||||
def test_responses_request_preserves_watermarking_opt_out():
|
||||
request = ResponsesRequest(input="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(default_max_tokens=1)
|
||||
|
||||
assert not params.watermarking
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_batch_chat_request_preserves_watermarking_opt_out():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]], watermarking=False
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,conversion,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
use_beam_search=True,
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=False),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_watermarking_opt_out(api_request, conversion, kwargs):
|
||||
params = getattr(api_request, conversion)(**kwargs)
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_cls", [TranscriptionRequest, TranslationRequest])
|
||||
def test_speech_requests_leave_watermarking_unspecified(request_cls):
|
||||
request = request_cls(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
temperature=0.8,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(default_max_tokens=1)
|
||||
beam_params = request.to_beam_search_params(default_max_tokens=1)
|
||||
|
||||
assert request.watermarking is None
|
||||
assert sampling_params.watermarking is None
|
||||
assert beam_params.watermarking is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "choose A or B"}],
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"max_tokens": 8, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(
|
||||
prompt="choose A or B",
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"max_tokens": 8},
|
||||
),
|
||||
(
|
||||
ResponsesRequest(
|
||||
input="choose A or B",
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"default_max_tokens": 8},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_unspecified_watermarking_with_structured_outputs(
|
||||
api_request, kwargs
|
||||
):
|
||||
params = api_request.to_sampling_params(**kwargs)
|
||||
|
||||
assert params.watermarking is None
|
||||
assert params.structured_outputs is not None
|
||||
assert params.structured_outputs.choice == ["A", "B"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,conversion,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}], watermarking=True
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
use_beam_search=True,
|
||||
watermarking=True,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", watermarking=True),
|
||||
"to_sampling_params",
|
||||
{"max_tokens": 1},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=True),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1},
|
||||
),
|
||||
(
|
||||
ResponsesRequest(input="hello", watermarking=True),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=True,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=True,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_explicit_watermarking(api_request, conversion, kwargs):
|
||||
params = getattr(api_request, conversion)(**kwargs)
|
||||
|
||||
assert params.watermarking is True
|
||||
|
||||
|
||||
def test_batch_chat_request_preserves_explicit_watermarking():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]], watermarking=True
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert converted.watermarking is True
|
||||
assert params.watermarking is True
|
||||
|
||||
|
||||
def test_batch_chat_request_inherits_watermarking():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]]
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert converted.watermarking is None
|
||||
assert params.watermarking is None
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
@@ -12,6 +13,7 @@ import websockets
|
||||
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.entrypoints.speech_to_text.realtime.connection import RealtimeConnection
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
@@ -26,6 +28,46 @@ MISTRAL_FORMAT_ARGS = [
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
|
||||
|
||||
def test_realtime_generation_preserves_watermarking():
|
||||
transcription_outputs = []
|
||||
|
||||
class EngineClient:
|
||||
def generate(self, **kwargs):
|
||||
async def outputs():
|
||||
output = SimpleNamespace(
|
||||
outputs=[],
|
||||
prompt_token_ids=[],
|
||||
is_watermarked=kwargs["sampling_params"].watermarking,
|
||||
)
|
||||
transcription_outputs.append(output)
|
||||
yield output
|
||||
|
||||
return outputs()
|
||||
|
||||
async def streaming_input():
|
||||
chunk: np.ndarray
|
||||
for chunk in ():
|
||||
yield chunk
|
||||
|
||||
async def send(_event):
|
||||
return None
|
||||
|
||||
connection = object.__new__(RealtimeConnection)
|
||||
connection.connection_id = "test"
|
||||
connection.serving = SimpleNamespace(
|
||||
model_cls=SimpleNamespace(realtime_max_tokens=1),
|
||||
engine_client=EngineClient(),
|
||||
)
|
||||
connection.audio_queue = asyncio.Queue()
|
||||
connection._is_connected = True
|
||||
connection.send = send
|
||||
|
||||
asyncio.run(connection._run_generation(streaming_input(), asyncio.Queue()))
|
||||
|
||||
assert len(transcription_outputs) == 1
|
||||
assert transcription_outputs[0].is_watermarked is None
|
||||
|
||||
|
||||
def _get_websocket_url(server: RemoteOpenAIServer) -> str:
|
||||
"""Convert HTTP URL to WebSocket URL for realtime endpoint."""
|
||||
http_url = server.url_root
|
||||
|
||||
@@ -198,4 +198,6 @@ async def test_language_detection_cancel_aborts_engine_request():
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
sampling_params = engine_client.generate.call_args.args[1]
|
||||
assert sampling_params.watermarking is False
|
||||
engine_client.abort.assert_awaited_once_with(request_id)
|
||||
|
||||
Reference in New Issue
Block a user