[Rust Frontend] Align OpenAI request and response edge cases (#53218)
Co-authored-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::normalize_top_k;
|
||||
|
||||
/// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -118,23 +119,19 @@ pub(super) struct GenerationConfig {
|
||||
pub max_new_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Deserialize vLLM-compatible `top_k` values from generation configs.
|
||||
/// Deserialize a generation-config `top_k` into the text sampling model.
|
||||
///
|
||||
/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
|
||||
/// are preserved.
|
||||
/// Null, `-1`, and `0` all disable top-k sampling; positive limits are
|
||||
/// preserved.
|
||||
fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
match Option::<i64>::deserialize(deserializer)? {
|
||||
None | Some(-1) | Some(0) => Ok(None),
|
||||
Some(value) if value > 0 => {
|
||||
u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
|
||||
}
|
||||
Some(value) => Err(serde::de::Error::custom(format!(
|
||||
"top_k must be -1, 0, or a positive integer, got {value}"
|
||||
))),
|
||||
}
|
||||
Option::<i64>::deserialize(deserializer)?
|
||||
.map(normalize_top_k)
|
||||
.transpose()
|
||||
.map(Option::flatten)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
/// HF generation configs allow either one EOS id or a list of EOS ids.
|
||||
|
||||
@@ -21,7 +21,7 @@ pub use output::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
|
||||
};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
|
||||
use trait_set::trait_set;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
pub use vllm_llm::FinishReason;
|
||||
|
||||
@@ -434,6 +434,20 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_preserves_zero_min_tokens() {
|
||||
let params = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
min_tokens: Some(0),
|
||||
..SamplingParams::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.expect("lower zero min_tokens");
|
||||
|
||||
assert_eq!(params.min_tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_repetition_detection() {
|
||||
let lower = |repetition_detection| {
|
||||
|
||||
@@ -15,6 +15,24 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
|
||||
use crate::error::{Error, Result};
|
||||
use crate::output::TextDecodeOptions;
|
||||
|
||||
/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
|
||||
///
|
||||
/// vLLM uses both `-1` and `0` to disable top-k sampling. The text sampling
|
||||
/// model represents that state as `None` and preserves positive limits as
|
||||
/// `Some(k)`. Values below `-1` and values above `u32::MAX` are rejected.
|
||||
///
|
||||
/// Callers that need to distinguish an omitted value from an explicit disable
|
||||
/// should preserve that request-level distinction around this normalization.
|
||||
pub fn normalize_top_k(value: i64) -> std::result::Result<Option<u32>, String> {
|
||||
match value {
|
||||
-1 | 0 => Ok(None),
|
||||
value if value > 0 => u32::try_from(value).map(Some).map_err(|error| error.to_string()),
|
||||
value => Err(format!(
|
||||
"top_k must be -1, 0, or a positive integer, got {value}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// One raw text-generation prompt.
|
||||
///
|
||||
/// This supports either ordinary text that still needs tokenization or
|
||||
|
||||
Reference in New Issue
Block a user