[Rust Frontend] Align OpenAI request and response edge cases (#53218)

Co-authored-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-27 19:23:38 +00:00
committed by GitHub
co-authored by Nick Hill
parent 4aab2b0ebe
commit 9db222c6be
16 changed files with 419 additions and 69 deletions
+9 -12
View File
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
use crate::error::{Error, Result};
use crate::normalize_top_k;
/// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
#[derive(Debug, Default, Deserialize)]
@@ -118,23 +119,19 @@ pub(super) struct GenerationConfig {
pub max_new_tokens: Option<u32>,
}
/// Deserialize vLLM-compatible `top_k` values from generation configs.
/// Deserialize a generation-config `top_k` into the text sampling model.
///
/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
/// are preserved.
/// Null, `-1`, and `0` all disable top-k sampling; positive limits are
/// preserved.
fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
where
D: serde::Deserializer<'de>,
{
match Option::<i64>::deserialize(deserializer)? {
None | Some(-1) | Some(0) => Ok(None),
Some(value) if value > 0 => {
u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
}
Some(value) => Err(serde::de::Error::custom(format!(
"top_k must be -1, 0, or a positive integer, got {value}"
))),
}
Option::<i64>::deserialize(deserializer)?
.map(normalize_top_k)
.transpose()
.map(Option::flatten)
.map_err(serde::de::Error::custom)
}
/// HF generation configs allow either one EOS id or a list of EOS ids.
+1 -1
View File
@@ -21,7 +21,7 @@ pub use output::{
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
};
pub use request::{Prompt, SamplingParams, TextRequest};
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
use trait_set::trait_set;
use vllm_engine_core_client::EngineCoreClient;
pub use vllm_llm::FinishReason;
+14
View File
@@ -434,6 +434,20 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_preserves_zero_min_tokens() {
let params = lower_sampling_params_with_limits(
SamplingParams {
min_tokens: Some(0),
..SamplingParams::default()
},
sample_sampling_limits(),
)
.expect("lower zero min_tokens");
assert_eq!(params.min_tokens, 0);
}
#[test]
fn lower_sampling_params_validates_repetition_detection() {
let lower = |repetition_detection| {
+18
View File
@@ -15,6 +15,24 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
use crate::error::{Error, Result};
use crate::output::TextDecodeOptions;
/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
///
/// vLLM uses both `-1` and `0` to disable top-k sampling. The text sampling
/// model represents that state as `None` and preserves positive limits as
/// `Some(k)`. Values below `-1` and values above `u32::MAX` are rejected.
///
/// Callers that need to distinguish an omitted value from an explicit disable
/// should preserve that request-level distinction around this normalization.
pub fn normalize_top_k(value: i64) -> std::result::Result<Option<u32>, String> {
match value {
-1 | 0 => Ok(None),
value if value > 0 => u32::try_from(value).map(Some).map_err(|error| error.to_string()),
value => Err(format!(
"top_k must be -1, 0, or a positive integer, got {value}"
)),
}
}
/// One raw text-generation prompt.
///
/// This supports either ordinary text that still needs tokenization or