[Rust Frontend] Add serde defaults for omit_defaults fields in EngineCoreSamplingParams (#45848)

Signed-off-by: Will Eaton <weaton@redhat.com>
This commit is contained in:
Will Eaton
2026-06-17 06:40:49 +00:00
committed by GitHub
parent 295232a26a
commit 17bc144556
3 changed files with 121 additions and 13 deletions
+70 -11
View File
@@ -44,6 +44,14 @@ fn default_repetition_penalty() -> f32 {
1.0
}
fn default_temperature() -> f32 {
1.0
}
fn default_max_tokens() -> u32 {
16
}
mod classified_outputs;
pub mod dtype;
pub mod handshake;
@@ -246,24 +254,28 @@ pub struct StructuredOutputsParams {
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
// default-valued keys; default the whole struct. Per-field fns cover the
// non-zero defaults.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
#[serde(default)]
pub struct EngineCoreSamplingParams {
/// Controls randomness. Lower values are more deterministic; zero means
/// greedy sampling.
#[serde(default = "default_temperature")]
pub temperature: f32,
/// Cumulative probability threshold for nucleus sampling.
#[serde(default = "default_top_p")]
pub top_p: f32,
/// Maximum number of top tokens to consider. `0` means all tokens.
#[serde(default)]
pub top_k: u32,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate per output sequence.
#[serde(default = "default_max_tokens")]
pub max_tokens: u32,
/// Minimum number of tokens to generate before EOS or stop-token handling.
#[serde(default)]
pub min_tokens: u32,
/// Number of log probabilities to return per generated token.
///
@@ -274,7 +286,6 @@ pub struct EngineCoreSamplingParams {
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling.
#[serde(default)]
pub min_p: f32,
/// Frequency penalty applied by the sampler.
pub frequency_penalty: f32,
@@ -301,16 +312,13 @@ pub struct EngineCoreSamplingParams {
pub all_stop_token_ids: BTreeSet<u32>,
/// Logit biases to apply during sampling.
/// Keys are token IDs
#[serde(default)]
pub logit_bias: Option<HashMap<u32, f32>>,
/// Restrict output to these token IDs only.
#[serde(default)]
pub allowed_token_ids: Option<Vec<u32>>,
/// Tokenized bad words to avoid during generation.
#[serde(default, rename = "_bad_words_token_ids")]
#[serde(rename = "_bad_words_token_ids")]
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
/// Parameters for configuring structured outputs (guided decoding).
#[serde(default)]
pub structured_outputs: Option<StructuredOutputsParams>,
/// Specific token IDs for which log probabilities should be returned at
/// each position.
@@ -318,15 +326,12 @@ pub struct EngineCoreSamplingParams {
/// When set, the engine returns logprobs for exactly these tokens in
/// addition to the sampled/scored token. Mutually exclusive with the
/// `logprobs` count field in practice.
#[serde(default)]
pub logprob_token_ids: Option<Vec<u32>>,
/// If `Some(true)`, the request will not attempt to read from the prefix
/// cache; newly computed blocks may still populate the cache. `None`
/// defers to engine-core defaults.
#[serde(default)]
pub skip_reading_prefix_cache: Option<bool>,
/// Additional request parameters for custom extensions (from `vllm_xargs`).
#[serde(default)]
pub extra_args: Option<HashMap<String, serde_json::Value>>,
}
@@ -640,4 +645,58 @@ mod tests {
let value = serde_json::to_value(params).unwrap();
assert_eq!(value["_backend"], "guidance");
}
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
/// must fall back to defaults. `python_compat` can't catch this since Rust
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
#[test]
fn decodes_sampling_params_with_omitted_defaults() {
let sampling_params = Value::Map(vec![
(
Value::from("stop_token_ids"),
Value::Array(vec![Value::from(151643u32)]),
),
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
]);
let request = Value::Array(vec![
Value::from("req-omit-defaults"),
Value::Array(vec![
Value::from(1u32),
Value::from(2u32),
Value::from(3u32),
]),
Value::Nil,
sampling_params,
Value::Nil,
Value::from(1.0f64),
]);
let mut bytes = Vec::new();
rmpv::encode::write_value(&mut bytes, &request).unwrap();
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
.expect("a real omit_defaults request must decode (regression: missing field)");
assert_eq!(decoded.request_id, "req-omit-defaults");
let sampling = decoded.sampling_params.expect("sampling params present");
assert_eq!(sampling.stop_token_ids, vec![151643]);
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
// Omitted fields -> Python defaults.
assert_eq!(sampling.temperature, 1.0);
assert_eq!(sampling.top_p, 1.0);
assert_eq!(sampling.top_k, 0);
assert_eq!(sampling.seed, None);
assert_eq!(sampling.max_tokens, 16);
assert_eq!(sampling.min_tokens, 0);
assert_eq!(sampling.min_p, 0.0);
assert_eq!(sampling.frequency_penalty, 0.0);
assert_eq!(sampling.presence_penalty, 0.0);
assert_eq!(sampling.repetition_penalty, 1.0);
assert_eq!(sampling.logprobs, None);
assert_eq!(sampling.prompt_logprobs, None);
assert_eq!(sampling.eos_token_id, None);
assert!(sampling.all_stop_token_ids.is_empty());
}
}
@@ -2438,6 +2438,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let stdout = String::from_utf8(output.stdout).unwrap();
let mut lines = stdout.lines();
let request_hex = lines.next().expect("missing request fixture line");
let defaults_request_hex = lines.next().expect("missing defaults request fixture line");
let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line");
let outputs_hex = lines.next().expect("missing outputs fixture line");
let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line");
@@ -2455,6 +2456,42 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let expected_request = sample_request();
assert_eq!(decoded_request, expected_request);
// All-default sampling params -> empty map; must decode to Python defaults.
let defaults_request_bytes = hex::decode(defaults_request_hex).unwrap();
let decoded_defaults: EngineCoreRequest =
rmp_serde::from_slice(&defaults_request_bytes).unwrap();
assert_eq!(decoded_defaults.request_id, "req-defaults");
let sampling = decoded_defaults
.sampling_params
.expect("defaults request carries sampling params");
assert_eq!(
sampling,
EngineCoreSamplingParams {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 16,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: Vec::new(),
eos_token_id: None,
all_stop_token_ids: BTreeSet::new(),
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
},
);
let decoded_multimodal_request: EngineCoreRequest =
rmp_serde::from_slice(&multimodal_request_bytes).unwrap();
assert_eq!(decoded_multimodal_request, sample_multimodal_request());
@@ -31,12 +31,13 @@ class FinishReason(IntEnum):
REPETITION = 4
class EngineCoreSamplingParams(msgspec.Struct, dict=True):
# Mirror of real SamplingParams; omit_defaults makes fixtures match real maps.
class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
temperature: float = 1.0
top_p: float = 1.0
top_k: int = 0
seed: int | None = None
max_tokens: int = 65536
max_tokens: int = 16
min_tokens: int = 0
min_p: float = 0.0
frequency_penalty: float = 0.0
@@ -135,6 +136,16 @@ request = EngineCoreRequest(
client_index=0,
)
# All defaults -> empty map. Regression guard for the sparse-map decode.
defaults_request = EngineCoreRequest(
request_id="req-defaults",
prompt_token_ids=[5, 6, 7],
mm_features=None,
sampling_params=EngineCoreSamplingParams(),
pooling_params=None,
arrival_time=1.0,
)
multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
multimodal_features = [
{
@@ -361,6 +372,7 @@ ready_response = EngineCoreReadyResponse(
)
print(msgspec.msgpack.encode(request).hex())
print(msgspec.msgpack.encode(defaults_request).hex())
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
print(msgspec.msgpack.encode(outputs).hex())
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))