diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 6fae982d2e..5e340b9117 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -44,6 +44,14 @@ fn default_repetition_penalty() -> f32 { 1.0 } +fn default_temperature() -> f32 { + 1.0 +} + +fn default_max_tokens() -> u32 { + 16 +} + mod classified_outputs; pub mod dtype; pub mod handshake; @@ -246,24 +254,28 @@ pub struct StructuredOutputsParams { /// /// Original Python definition: /// +// Python's SamplingParams is `omit_defaults=True`, so msgpack drops +// default-valued keys; default the whole struct. Per-field fns cover the +// non-zero defaults. #[serde_with::skip_serializing_none] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)] +#[serde(default)] pub struct EngineCoreSamplingParams { /// Controls randomness. Lower values are more deterministic; zero means /// greedy sampling. + #[serde(default = "default_temperature")] pub temperature: f32, /// Cumulative probability threshold for nucleus sampling. #[serde(default = "default_top_p")] pub top_p: f32, /// Maximum number of top tokens to consider. `0` means all tokens. - #[serde(default)] pub top_k: u32, /// Random seed used by the sampler when present. pub seed: Option, /// Maximum number of tokens to generate per output sequence. + #[serde(default = "default_max_tokens")] pub max_tokens: u32, /// Minimum number of tokens to generate before EOS or stop-token handling. - #[serde(default)] pub min_tokens: u32, /// Number of log probabilities to return per generated token. /// @@ -274,7 +286,6 @@ pub struct EngineCoreSamplingParams { /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, /// Minimum probability threshold for token sampling. - #[serde(default)] pub min_p: f32, /// Frequency penalty applied by the sampler. pub frequency_penalty: f32, @@ -301,16 +312,13 @@ pub struct EngineCoreSamplingParams { pub all_stop_token_ids: BTreeSet, /// Logit biases to apply during sampling. /// Keys are token IDs - #[serde(default)] pub logit_bias: Option>, /// Restrict output to these token IDs only. - #[serde(default)] pub allowed_token_ids: Option>, /// Tokenized bad words to avoid during generation. - #[serde(default, rename = "_bad_words_token_ids")] + #[serde(rename = "_bad_words_token_ids")] pub bad_words_token_ids: Option>>, /// Parameters for configuring structured outputs (guided decoding). - #[serde(default)] pub structured_outputs: Option, /// Specific token IDs for which log probabilities should be returned at /// each position. @@ -318,15 +326,12 @@ pub struct EngineCoreSamplingParams { /// When set, the engine returns logprobs for exactly these tokens in /// addition to the sampled/scored token. Mutually exclusive with the /// `logprobs` count field in practice. - #[serde(default)] pub logprob_token_ids: Option>, /// If `Some(true)`, the request will not attempt to read from the prefix /// cache; newly computed blocks may still populate the cache. `None` /// defers to engine-core defaults. - #[serde(default)] pub skip_reading_prefix_cache: Option, /// Additional request parameters for custom extensions (from `vllm_xargs`). - #[serde(default)] pub extra_args: Option>, } @@ -640,4 +645,58 @@ mod tests { let value = serde_json::to_value(params).unwrap(); assert_eq!(value["_backend"], "guidance"); } + + /// A real `sampling_params` is a sparse `omit_defaults` map; absent fields + /// must fall back to defaults. `python_compat` can't catch this since Rust + /// encodes full maps (see `engine_core_request_serializes_as_full_array`). + #[test] + fn decodes_sampling_params_with_omitted_defaults() { + let sampling_params = Value::Map(vec![ + ( + Value::from("stop_token_ids"), + Value::Array(vec![Value::from(151643u32)]), + ), + (Value::from("skip_reading_prefix_cache"), Value::from(false)), + ]); + let request = Value::Array(vec![ + Value::from("req-omit-defaults"), + Value::Array(vec![ + Value::from(1u32), + Value::from(2u32), + Value::from(3u32), + ]), + Value::Nil, + sampling_params, + Value::Nil, + Value::from(1.0f64), + ]); + + let mut bytes = Vec::new(); + rmpv::encode::write_value(&mut bytes, &request).unwrap(); + + let decoded: EngineCoreRequest = decode_msgpack(&bytes) + .expect("a real omit_defaults request must decode (regression: missing field)"); + + assert_eq!(decoded.request_id, "req-omit-defaults"); + let sampling = decoded.sampling_params.expect("sampling params present"); + + assert_eq!(sampling.stop_token_ids, vec![151643]); + assert_eq!(sampling.skip_reading_prefix_cache, Some(false)); + + // Omitted fields -> Python defaults. + assert_eq!(sampling.temperature, 1.0); + assert_eq!(sampling.top_p, 1.0); + assert_eq!(sampling.top_k, 0); + assert_eq!(sampling.seed, None); + assert_eq!(sampling.max_tokens, 16); + assert_eq!(sampling.min_tokens, 0); + assert_eq!(sampling.min_p, 0.0); + assert_eq!(sampling.frequency_penalty, 0.0); + assert_eq!(sampling.presence_penalty, 0.0); + assert_eq!(sampling.repetition_penalty, 1.0); + assert_eq!(sampling.logprobs, None); + assert_eq!(sampling.prompt_logprobs, None); + assert_eq!(sampling.eos_token_id, None); + assert!(sampling.all_stop_token_ids.is_empty()); + } } diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 83e6cb7ec2..7d24299dea 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2438,6 +2438,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { let stdout = String::from_utf8(output.stdout).unwrap(); let mut lines = stdout.lines(); let request_hex = lines.next().expect("missing request fixture line"); + let defaults_request_hex = lines.next().expect("missing defaults request fixture line"); let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line"); let outputs_hex = lines.next().expect("missing outputs fixture line"); let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line"); @@ -2455,6 +2456,42 @@ fn python_msgpack_fixtures_match_rust_encoding() { let expected_request = sample_request(); assert_eq!(decoded_request, expected_request); + // All-default sampling params -> empty map; must decode to Python defaults. + let defaults_request_bytes = hex::decode(defaults_request_hex).unwrap(); + let decoded_defaults: EngineCoreRequest = + rmp_serde::from_slice(&defaults_request_bytes).unwrap(); + assert_eq!(decoded_defaults.request_id, "req-defaults"); + let sampling = decoded_defaults + .sampling_params + .expect("defaults request carries sampling params"); + assert_eq!( + sampling, + EngineCoreSamplingParams { + temperature: 1.0, + top_p: 1.0, + top_k: 0, + seed: None, + max_tokens: 16, + min_tokens: 0, + logprobs: None, + prompt_logprobs: None, + min_p: 0.0, + frequency_penalty: 0.0, + presence_penalty: 0.0, + repetition_penalty: 1.0, + stop_token_ids: Vec::new(), + eos_token_id: None, + all_stop_token_ids: BTreeSet::new(), + logit_bias: None, + allowed_token_ids: None, + bad_words_token_ids: None, + structured_outputs: None, + logprob_token_ids: None, + skip_reading_prefix_cache: None, + extra_args: None, + }, + ); + let decoded_multimodal_request: EngineCoreRequest = rmp_serde::from_slice(&multimodal_request_bytes).unwrap(); assert_eq!(decoded_multimodal_request, sample_multimodal_request()); diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 89179b3fbf..8398c874da 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -31,12 +31,13 @@ class FinishReason(IntEnum): REPETITION = 4 -class EngineCoreSamplingParams(msgspec.Struct, dict=True): +# Mirror of real SamplingParams; omit_defaults makes fixtures match real maps. +class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True): temperature: float = 1.0 top_p: float = 1.0 top_k: int = 0 seed: int | None = None - max_tokens: int = 65536 + max_tokens: int = 16 min_tokens: int = 0 min_p: float = 0.0 frequency_penalty: float = 0.0 @@ -135,6 +136,16 @@ request = EngineCoreRequest( client_index=0, ) +# All defaults -> empty map. Regression guard for the sparse-map decode. +defaults_request = EngineCoreRequest( + request_id="req-defaults", + prompt_token_ids=[5, 6, 7], + mm_features=None, + sampling_params=EngineCoreSamplingParams(), + pooling_params=None, + arrival_time=1.0, +) + multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32) multimodal_features = [ { @@ -361,6 +372,7 @@ ready_response = EngineCoreReadyResponse( ) print(msgspec.msgpack.encode(request).hex()) +print(msgspec.msgpack.encode(defaults_request).hex()) print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex()) print(msgspec.msgpack.encode(outputs).hex()) print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))