[Rust Frontend] Forward per-request watermarking controls (#56338)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -29,7 +29,12 @@ from vllm import SamplingParams
|
||||
sampling_params = SamplingParams(watermarking=False)
|
||||
```
|
||||
|
||||
The OpenAI-compatible APIs accept the same `watermarking: false` request field.
|
||||
The Python OpenAI-compatible APIs and the Rust chat/completions APIs accept the
|
||||
same `watermarking: false` request field. The Rust token API accepts it in
|
||||
`sampling_params`, and the Rust gRPC `GenerateRequest` accepts
|
||||
the optional `watermarking` field. Omitting the field keeps watermarking enabled
|
||||
when the engine has a watermark configuration.
|
||||
|
||||
Deployments that require watermarking must restrict this field to trusted
|
||||
callers, or strip and validate it at the ingress boundary, so untrusted clients
|
||||
cannot opt out.
|
||||
|
||||
@@ -53,6 +53,9 @@ message GenerateRequest {
|
||||
|
||||
// Loaded LoRA name; empty selects the base model.
|
||||
string lora_name = 15;
|
||||
|
||||
// Apply the engine's configured watermark. Defaults to true when omitted.
|
||||
optional bool watermarking = 16;
|
||||
}
|
||||
|
||||
message RandomSampling {
|
||||
|
||||
@@ -20,6 +20,10 @@ fn default_temperature() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_watermarking() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
16
|
||||
}
|
||||
@@ -68,6 +72,9 @@ pub struct EngineCoreSamplingParams {
|
||||
/// greedy sampling.
|
||||
#[serde(default = "default_temperature")]
|
||||
pub temperature: f32,
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
#[serde(default = "default_watermarking")]
|
||||
pub watermarking: bool,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
#[serde(default = "default_top_p")]
|
||||
pub top_p: f32,
|
||||
@@ -155,6 +162,7 @@ impl EngineCoreSamplingParams {
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
temperature: 1.0,
|
||||
watermarking: true,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
@@ -229,6 +237,7 @@ mod tests {
|
||||
|
||||
// Omitted fields -> Python defaults.
|
||||
assert_eq!(sampling.temperature, 1.0);
|
||||
assert!(sampling.watermarking);
|
||||
assert_eq!(sampling.top_p, 1.0);
|
||||
assert_eq!(sampling.top_k, 0);
|
||||
assert_eq!(sampling.seed, None);
|
||||
|
||||
@@ -151,6 +151,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
|
||||
prompt_token_ids: Some(vec![11, 22]),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
temperature: 0.8,
|
||||
watermarking: false,
|
||||
top_p: 0.9,
|
||||
top_k: 8,
|
||||
max_tokens: 32,
|
||||
@@ -2662,6 +2663,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
sampling,
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
watermarking: true,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
|
||||
@@ -34,6 +34,7 @@ class FinishReason(IntEnum):
|
||||
# Mirror of real SamplingParams; omit_defaults makes fixtures match real maps.
|
||||
class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
|
||||
temperature: float = 1.0
|
||||
watermarking: bool = True
|
||||
top_p: float = 1.0
|
||||
top_k: int = 0
|
||||
seed: int | None = None
|
||||
@@ -123,6 +124,7 @@ request = EngineCoreRequest(
|
||||
mm_features=None,
|
||||
sampling_params=EngineCoreSamplingParams(
|
||||
temperature=0.8,
|
||||
watermarking=False,
|
||||
top_p=0.9,
|
||||
top_k=8,
|
||||
seed=None,
|
||||
|
||||
@@ -65,6 +65,7 @@ pub fn to_text_request(
|
||||
|
||||
let mut sampling_params =
|
||||
build_sampling_params(req.temperature, sampling, decoding, stopping, response)?;
|
||||
sampling_params.watermarking = req.watermarking.unwrap_or(true);
|
||||
|
||||
// Thread KVCacheParameters → SamplingParams fields.
|
||||
if let Some(kv) = kv {
|
||||
@@ -521,6 +522,7 @@ impl ResponseOpts {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use prost::Message as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params,
|
||||
@@ -539,6 +541,26 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let request = pb::GenerateRequest {
|
||||
watermarking,
|
||||
..base_request()
|
||||
};
|
||||
let encoded = request.encode_to_vec();
|
||||
for stream in [false, true] {
|
||||
let decoded = pb::GenerateRequest::decode(encoded.as_slice()).unwrap();
|
||||
let text = to_text_request(decoded, stream, &["test-model".to_string()])
|
||||
.expect("convert request");
|
||||
assert_eq!(
|
||||
text.sampling_params.watermarking,
|
||||
watermarking.unwrap_or(true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temperature_propagates_from_top_level_request_field() {
|
||||
let req = pb::GenerateRequest {
|
||||
|
||||
@@ -164,6 +164,30 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_preserves_watermarking_defaults_and_opt_out() {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let mut body = json!({
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {}
|
||||
});
|
||||
if let Some(watermarking) = watermarking {
|
||||
body["sampling_params"]["watermarking"] = json!(watermarking);
|
||||
}
|
||||
let prepared = prepare_generate_request(
|
||||
serde_json::from_value(body).expect("parse request"),
|
||||
&served(&["test-model"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare request");
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.watermarking,
|
||||
watermarking.unwrap_or(true)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_forwards_thinking_token_budget() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -150,6 +150,7 @@ pub(super) fn prepare_chat_request(
|
||||
messages,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
watermarking: request.watermarking,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
|
||||
@@ -116,6 +116,10 @@ pub struct ChatCompletionRequest {
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
#[serde(default = "default_true")]
|
||||
pub watermarking: bool,
|
||||
|
||||
/// Use beam search instead of sampling
|
||||
#[serde(default)]
|
||||
pub use_beam_search: bool,
|
||||
@@ -282,6 +286,7 @@ impl Default for ChatCompletionRequest {
|
||||
include_reasoning: true,
|
||||
parallel_tool_calls: None,
|
||||
user: None,
|
||||
watermarking: true,
|
||||
use_beam_search: false,
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
|
||||
@@ -115,6 +115,7 @@ pub(super) fn prepare_completion_request(
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams {
|
||||
temperature: request.temperature,
|
||||
watermarking: request.watermarking,
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
|
||||
@@ -90,6 +90,10 @@ pub struct CompletionRequest {
|
||||
pub user: Option<String>,
|
||||
|
||||
// -------- vLLM Sampling Parameters --------
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
#[serde(default = "default_true")]
|
||||
pub watermarking: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
|
||||
@@ -2736,6 +2736,49 @@ async fn non_stream_chat_returns_json_response() {
|
||||
assert!(!message.contains_key("tool_calls"), "{json}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn openai_watermarking_defaults_and_opt_out_reach_engine_core() {
|
||||
for endpoint in ["/v1/chat/completions", "/v1/completions"] {
|
||||
for stream in [false, true] {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
let (mut app, engine_task) = test_app_with_backend_and_engine_request_check(
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
move |request| {
|
||||
let params = request.sampling_params.as_ref().expect("sampling params");
|
||||
assert_eq!(params.watermarking, watermarking.unwrap_or(true));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut body = json!({"stream": stream, "max_tokens": 2});
|
||||
if endpoint == "/v1/chat/completions" {
|
||||
body["messages"] = json!([{"role": "user", "content": "hi"}]);
|
||||
} else {
|
||||
body["prompt"] = json!("hi");
|
||||
}
|
||||
if let Some(watermarking) = watermarking {
|
||||
body["watermarking"] = json!(watermarking);
|
||||
}
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(endpoint)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
to_bytes(response.into_body(), usize::MAX).await.expect("drain response");
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_image_url_reaches_engine_mm_features() {
|
||||
|
||||
@@ -92,6 +92,7 @@ pub fn lower_sampling_params(
|
||||
) -> Result<EngineCoreSamplingParams> {
|
||||
let SamplingParams {
|
||||
temperature,
|
||||
watermarking,
|
||||
top_p,
|
||||
top_k,
|
||||
seed,
|
||||
@@ -164,6 +165,7 @@ pub fn lower_sampling_params(
|
||||
|
||||
let params = EngineCoreSamplingParams {
|
||||
temperature,
|
||||
watermarking,
|
||||
top_p,
|
||||
top_k,
|
||||
seed,
|
||||
@@ -632,6 +634,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
watermarking: true,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
@@ -686,6 +689,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
watermarking: true,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
@@ -854,6 +858,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.6,
|
||||
watermarking: true,
|
||||
top_p: 0.95,
|
||||
top_k: 20,
|
||||
seed: None,
|
||||
@@ -918,6 +923,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
watermarking: true,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
@@ -990,6 +996,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.2,
|
||||
watermarking: true,
|
||||
top_p: 0.3,
|
||||
top_k: 4,
|
||||
seed: None,
|
||||
@@ -1240,6 +1247,7 @@ mod tests {
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.8,
|
||||
watermarking: true,
|
||||
top_p: 0.9,
|
||||
top_k: 12,
|
||||
seed: None,
|
||||
|
||||
@@ -69,6 +69,8 @@ pub struct SamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling. `None` means no explicit user override.
|
||||
pub temperature: Option<f32>,
|
||||
/// Whether to apply the engine's configured watermark to this request.
|
||||
pub watermarking: bool,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
pub top_p: Option<f32>,
|
||||
/// Maximum number of top tokens to consider. `Some(0)` means all tokens.
|
||||
@@ -144,6 +146,7 @@ impl Default for SamplingParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
temperature: None,
|
||||
watermarking: true,
|
||||
top_p: None,
|
||||
top_k: None,
|
||||
seed: None,
|
||||
|
||||
Reference in New Issue
Block a user