[Rust Frontend] Coerce completion max_tokens: null to default (#47166)
Signed-off-by: Blas Rodriguez Irizar <rodrigblas@gmail.com>
This commit is contained in:
@@ -199,6 +199,7 @@ mod tests {
|
||||
use super::prepare_completion_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::types::CompletionRequest;
|
||||
use crate::routes::openai::utils::types::Normalizable;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
fn request_context(headers: &HeaderMap, request_id: Option<&str>) -> ResolvedRequestContext {
|
||||
@@ -249,6 +250,28 @@ mod tests {
|
||||
assert!(request.ignore_eos);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_coerces_null_max_tokens_to_default() {
|
||||
// An absent `max_tokens` already gets the serde default.
|
||||
let absent: CompletionRequest =
|
||||
serde_json::from_value(base_request_json()).expect("parse request");
|
||||
assert_eq!(absent.max_tokens, Some(16));
|
||||
|
||||
// An explicit `null` deserializes to `None`, bypassing the default;
|
||||
// `normalize` must coerce it back to match Python vLLM.
|
||||
let mut request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": true,
|
||||
"max_tokens": null
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert_eq!(request.max_tokens, None);
|
||||
|
||||
request.normalize();
|
||||
assert_eq!(request.max_tokens, Some(16));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_maps_sampling_fields() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -179,7 +179,17 @@ pub struct CompletionRequest {
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl Normalizable for CompletionRequest {}
|
||||
impl Normalizable for CompletionRequest {
|
||||
/// Normalize the request by applying defaults.
|
||||
fn normalize(&mut self) {
|
||||
// An explicit `"max_tokens": null` deserializes to `None`, bypassing the
|
||||
// serde field default. Coerce it back to the default so it behaves like
|
||||
// an absent field, matching Python vLLM's `normalize_null_max_tokens`.
|
||||
if self.max_tokens.is_none() {
|
||||
self.max_tokens = default_completion_max_tokens();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionResponse` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
|
||||
Reference in New Issue
Block a user