[Rust Frontend] Forward per-request watermarking controls (#56338)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -65,6 +65,7 @@ pub fn to_text_request(
|
||||
|
||||
let mut sampling_params =
|
||||
build_sampling_params(req.temperature, sampling, decoding, stopping, response)?;
|
||||
sampling_params.watermarking = req.watermarking.unwrap_or(true);
|
||||
|
||||
// Thread KVCacheParameters → SamplingParams fields.
|
||||
if let Some(kv) = kv {
|
||||
@@ -521,6 +522,7 @@ impl ResponseOpts {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use prost::Message as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params,
|
||||
@@ -539,6 +541,26 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let request = pb::GenerateRequest {
|
||||
watermarking,
|
||||
..base_request()
|
||||
};
|
||||
let encoded = request.encode_to_vec();
|
||||
for stream in [false, true] {
|
||||
let decoded = pb::GenerateRequest::decode(encoded.as_slice()).unwrap();
|
||||
let text = to_text_request(decoded, stream, &["test-model".to_string()])
|
||||
.expect("convert request");
|
||||
assert_eq!(
|
||||
text.sampling_params.watermarking,
|
||||
watermarking.unwrap_or(true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temperature_propagates_from_top_level_request_field() {
|
||||
let req = pb::GenerateRequest {
|
||||
|
||||
@@ -164,6 +164,30 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_preserves_watermarking_defaults_and_opt_out() {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let mut body = json!({
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {}
|
||||
});
|
||||
if let Some(watermarking) = watermarking {
|
||||
body["sampling_params"]["watermarking"] = json!(watermarking);
|
||||
}
|
||||
let prepared = prepare_generate_request(
|
||||
serde_json::from_value(body).expect("parse request"),
|
||||
&served(&["test-model"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare request");
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.watermarking,
|
||||
watermarking.unwrap_or(true)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_forwards_thinking_token_budget() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -150,6 +150,7 @@ pub(super) fn prepare_chat_request(
|
||||
messages,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
watermarking: request.watermarking,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
|
||||
@@ -116,6 +116,10 @@ pub struct ChatCompletionRequest {
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
#[serde(default = "default_true")]
|
||||
pub watermarking: bool,
|
||||
|
||||
/// Use beam search instead of sampling
|
||||
#[serde(default)]
|
||||
pub use_beam_search: bool,
|
||||
@@ -282,6 +286,7 @@ impl Default for ChatCompletionRequest {
|
||||
include_reasoning: true,
|
||||
parallel_tool_calls: None,
|
||||
user: None,
|
||||
watermarking: true,
|
||||
use_beam_search: false,
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
|
||||
@@ -115,6 +115,7 @@ pub(super) fn prepare_completion_request(
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
watermarking: request.watermarking,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
|
||||
@@ -90,6 +90,10 @@ pub struct CompletionRequest {
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
#[serde(default = "default_true")]
|
||||
pub watermarking: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
|
||||
@@ -2736,6 +2736,49 @@ async fn non_stream_chat_returns_json_response() {
|
||||
assert!(!message.contains_key("tool_calls"), "{json}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn openai_watermarking_defaults_and_opt_out_reach_engine_core() {
|
||||
for endpoint in ["/v1/chat/completions", "/v1/completions"] {
|
||||
for stream in [false, true] {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let (mut app, engine_task) = test_app_with_backend_and_engine_request_check(
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
move |request| {
|
||||
let params = request.sampling_params.as_ref().expect("sampling params");
|
||||
assert_eq!(params.watermarking, watermarking.unwrap_or(true));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut body = json!({"stream": stream, "max_tokens": 2});
|
||||
if endpoint == "/v1/chat/completions" {
|
||||
body["messages"] = json!([{"role": "user", "content": "hi"}]);
|
||||
} else {
|
||||
body["prompt"] = json!("hi");
|
||||
}
|
||||
if let Some(watermarking) = watermarking {
|
||||
body["watermarking"] = json!(watermarking);
|
||||
}
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(endpoint)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
to_bytes(response.into_body(), usize::MAX).await.expect("drain response");
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_image_url_reaches_engine_mm_features() {
|
||||
|
||||
Reference in New Issue
Block a user