From 1713a094915d643c4ffaa25f421353108ee2049f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Mon, 14 Sep 2026 17:17:02 +0200 Subject: [PATCH] [Rust Frontend] Forward per-request watermarking controls (#56338) Co-authored-by: Codex Signed-off-by: NickLucche --- docs/features/watermarking.md | 7 ++- rust/proto/inference.proto | 3 ++ .../src/protocol/sampling.rs | 9 ++++ .../engine-core-client/src/tests/client.rs | 2 + .../src/tests/python_compat.py | 2 + rust/src/server/src/grpc/convert.rs | 22 ++++++++++ .../src/routes/inference/generate/convert.rs | 24 +++++++++++ .../routes/openai/chat_completions/convert.rs | 1 + .../routes/openai/chat_completions/types.rs | 5 +++ .../src/routes/openai/completions/convert.rs | 1 + .../src/routes/openai/completions/types.rs | 4 ++ rust/src/server/src/routes/tests.rs | 43 +++++++++++++++++++ rust/src/text/src/lower.rs | 8 ++++ rust/src/text/src/request.rs | 3 ++ 14 files changed, 133 insertions(+), 1 deletion(-) diff --git a/docs/features/watermarking.md b/docs/features/watermarking.md index f9ceda0547..54c7e58ce7 100644 --- a/docs/features/watermarking.md +++ b/docs/features/watermarking.md @@ -29,7 +29,12 @@ from vllm import SamplingParams sampling_params = SamplingParams(watermarking=False) ``` -The OpenAI-compatible APIs accept the same `watermarking: false` request field. +The Python OpenAI-compatible APIs and the Rust chat/completions APIs accept the +same `watermarking: false` request field. The Rust token API accepts it in +`sampling_params`, and the Rust gRPC `GenerateRequest` accepts +the optional `watermarking` field. Omitting the field keeps watermarking enabled +when the engine has a watermark configuration. + Deployments that require watermarking must restrict this field to trusted callers, or strip and validate it at the ingress boundary, so untrusted clients cannot opt out. diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 858d8a7a7e..46313f915a 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -53,6 +53,9 @@ message GenerateRequest { // Loaded LoRA name; empty selects the base model. string lora_name = 15; + + // Apply the engine's configured watermark. Defaults to true when omitted. + optional bool watermarking = 16; } message RandomSampling { diff --git a/rust/src/engine-core-client/src/protocol/sampling.rs b/rust/src/engine-core-client/src/protocol/sampling.rs index 19c579ac33..92f354be6f 100644 --- a/rust/src/engine-core-client/src/protocol/sampling.rs +++ b/rust/src/engine-core-client/src/protocol/sampling.rs @@ -20,6 +20,10 @@ fn default_temperature() -> f32 { 1.0 } +fn default_watermarking() -> bool { + true +} + fn default_max_tokens() -> u32 { 16 } @@ -68,6 +72,9 @@ pub struct EngineCoreSamplingParams { /// greedy sampling. #[serde(default = "default_temperature")] pub temperature: f32, + /// Whether to apply the engine's configured watermark to this request. + #[serde(default = "default_watermarking")] + pub watermarking: bool, /// Cumulative probability threshold for nucleus sampling. #[serde(default = "default_top_p")] pub top_p: f32, @@ -155,6 +162,7 @@ impl EngineCoreSamplingParams { pub fn for_test() -> Self { Self { temperature: 1.0, + watermarking: true, top_p: 1.0, top_k: 0, seed: None, @@ -229,6 +237,7 @@ mod tests { // Omitted fields -> Python defaults. assert_eq!(sampling.temperature, 1.0); + assert!(sampling.watermarking); assert_eq!(sampling.top_p, 1.0); assert_eq!(sampling.top_k, 0); assert_eq!(sampling.seed, None); diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 13525375f8..27fbbe9abd 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -151,6 +151,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest { prompt_token_ids: Some(vec![11, 22]), sampling_params: Some(EngineCoreSamplingParams { temperature: 0.8, + watermarking: false, top_p: 0.9, top_k: 8, max_tokens: 32, @@ -2662,6 +2663,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { sampling, EngineCoreSamplingParams { temperature: 1.0, + watermarking: true, top_p: 1.0, top_k: 0, seed: None, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index fd6cb0c86b..20cec81808 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -34,6 +34,7 @@ class FinishReason(IntEnum): # Mirror of real SamplingParams; omit_defaults makes fixtures match real maps. class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True): temperature: float = 1.0 + watermarking: bool = True top_p: float = 1.0 top_k: int = 0 seed: int | None = None @@ -123,6 +124,7 @@ request = EngineCoreRequest( mm_features=None, sampling_params=EngineCoreSamplingParams( temperature=0.8, + watermarking=False, top_p=0.9, top_k=8, seed=None, diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index aea04495eb..5f6ce5d42b 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -65,6 +65,7 @@ pub fn to_text_request( let mut sampling_params = build_sampling_params(req.temperature, sampling, decoding, stopping, response)?; + sampling_params.watermarking = req.watermarking.unwrap_or(true); // Thread KVCacheParameters → SamplingParams fields. if let Some(kv) = kv { @@ -521,6 +522,7 @@ impl ResponseOpts { #[cfg(test)] mod tests { + use prost::Message as _; use vllm_engine_core_client::protocol::output::StopReason; use vllm_text::{ FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params, @@ -539,6 +541,26 @@ mod tests { } } + #[test] + fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() { + for watermarking in [None, Some(true), Some(false)] { + let request = pb::GenerateRequest { + watermarking, + ..base_request() + }; + let encoded = request.encode_to_vec(); + for stream in [false, true] { + let decoded = pb::GenerateRequest::decode(encoded.as_slice()).unwrap(); + let text = to_text_request(decoded, stream, &["test-model".to_string()]) + .expect("convert request"); + assert_eq!( + text.sampling_params.watermarking, + watermarking.unwrap_or(true) + ); + } + } + } + #[test] fn temperature_propagates_from_top_level_request_field() { let req = pb::GenerateRequest { diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index cb06cfe33c..08c4ce18be 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -164,6 +164,30 @@ mod tests { ); } + #[test] + fn prepare_generate_request_preserves_watermarking_defaults_and_opt_out() { + for watermarking in [None, Some(true), Some(false)] { + let mut body = json!({ + "token_ids": [11, 22], + "sampling_params": {} + }); + if let Some(watermarking) = watermarking { + body["sampling_params"]["watermarking"] = json!(watermarking); + } + let prepared = prepare_generate_request( + serde_json::from_value(body).expect("parse request"), + &served(&["test-model"]), + ResolvedRequestContext::default(), + None, + ) + .expect("prepare request"); + assert_eq!( + prepared.text_request.sampling_params.watermarking, + watermarking.unwrap_or(true) + ); + } + } + #[test] fn prepare_generate_request_forwards_thinking_token_budget() { let request: GenerateRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 2c64ffbef4..51b0fb68ad 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -150,6 +150,7 @@ pub(super) fn prepare_chat_request( messages, sampling_params: SamplingParams { temperature: request.temperature, + watermarking: request.watermarking, top_p: request.top_p, top_k: request.top_k, seed: request.seed, diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index e4da86c22d..a371462694 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -116,6 +116,10 @@ pub struct ChatCompletionRequest { pub user: Option, // -------- vLLM Sampling Parameters -------- + /// Whether to apply the engine's configured watermark to this request. + #[serde(default = "default_true")] + pub watermarking: bool, + /// Use beam search instead of sampling #[serde(default)] pub use_beam_search: bool, @@ -282,6 +286,7 @@ impl Default for ChatCompletionRequest { include_reasoning: true, parallel_tool_calls: None, user: None, + watermarking: true, use_beam_search: false, top_k: None, min_p: None, diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index d944dca657..a4e8b6d3e4 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -115,6 +115,7 @@ pub(super) fn prepare_completion_request( mm_features: None, sampling_params: SamplingParams { temperature: request.temperature, + watermarking: request.watermarking, top_p: request.top_p, top_k: request.top_k, seed: request.seed, diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index 8d93a89ed4..2a69013706 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -90,6 +90,10 @@ pub struct CompletionRequest { pub user: Option, // -------- vLLM Sampling Parameters -------- + /// Whether to apply the engine's configured watermark to this request. + #[serde(default = "default_true")] + pub watermarking: bool, + /// Options for streaming response pub stream_options: Option, diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 6fbe22e501..4731958e2f 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -2736,6 +2736,49 @@ async fn non_stream_chat_returns_json_response() { assert!(!message.contains_key("tool_calls"), "{json}"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn openai_watermarking_defaults_and_opt_out_reach_engine_core() { + for endpoint in ["/v1/chat/completions", "/v1/completions"] { + for stream in [false, true] { + for watermarking in [None, Some(true), Some(false)] { + let (mut app, engine_task) = test_app_with_backend_and_engine_request_check( + Arc::new(FakeChatBackend::new()), + move |request| { + let params = request.sampling_params.as_ref().expect("sampling params"); + assert_eq!(params.watermarking, watermarking.unwrap_or(true)); + }, + ) + .await; + + let mut body = json!({"stream": stream, "max_tokens": 2}); + if endpoint == "/v1/chat/completions" { + body["messages"] = json!([{"role": "user", "content": "hi"}]); + } else { + body["prompt"] = json!("hi"); + } + if let Some(watermarking) = watermarking { + body["watermarking"] = json!(watermarking); + } + let response = app + .call( + Request::builder() + .method("POST") + .uri(endpoint) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + to_bytes(response.into_body(), usize::MAX).await.expect("drain response"); + engine_task.await.expect("mock engine task"); + } + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn non_stream_chat_image_url_reaches_engine_mm_features() { diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index bff4a6d48b..466b9d5b31 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -92,6 +92,7 @@ pub fn lower_sampling_params( ) -> Result { let SamplingParams { temperature, + watermarking, top_p, top_k, seed, @@ -164,6 +165,7 @@ pub fn lower_sampling_params( let params = EngineCoreSamplingParams { temperature, + watermarking, top_p, top_k, seed, @@ -632,6 +634,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 1.0, + watermarking: true, top_p: 1.0, top_k: 0, seed: None, @@ -686,6 +689,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 1.0, + watermarking: true, top_p: 1.0, top_k: 0, seed: None, @@ -854,6 +858,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 0.6, + watermarking: true, top_p: 0.95, top_k: 20, seed: None, @@ -918,6 +923,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 1.0, + watermarking: true, top_p: 1.0, top_k: 0, seed: None, @@ -990,6 +996,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 0.2, + watermarking: true, top_p: 0.3, top_k: 4, seed: None, @@ -1240,6 +1247,7 @@ mod tests { expect_test::expect![[r#" EngineCoreSamplingParams { temperature: 0.8, + watermarking: true, top_p: 0.9, top_k: 12, seed: None, diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 182e20c660..f0dd2fb1f8 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -69,6 +69,8 @@ pub struct SamplingParams { /// Controls randomness. Lower values are more deterministic; zero means /// greedy sampling. `None` means no explicit user override. pub temperature: Option, + /// Whether to apply the engine's configured watermark to this request. + pub watermarking: bool, /// Cumulative probability threshold for nucleus sampling. pub top_p: Option, /// Maximum number of top tokens to consider. `Some(0)` means all tokens. @@ -144,6 +146,7 @@ impl Default for SamplingParams { fn default() -> Self { Self { temperature: None, + watermarking: true, top_p: None, top_k: None, seed: None,