[Rust Frontend] Add serde defaults for omit_defaults fields in EngineCoreSamplingParams (#45848)
Signed-off-by: Will Eaton <weaton@redhat.com>
This commit is contained in:
@@ -44,6 +44,14 @@ fn default_repetition_penalty() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_temperature() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
16
|
||||
}
|
||||
|
||||
mod classified_outputs;
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
@@ -246,24 +254,28 @@ pub struct StructuredOutputsParams {
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
|
||||
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
|
||||
// default-valued keys; default the whole struct. Per-field fns cover the
|
||||
// non-zero defaults.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
|
||||
#[serde(default)]
|
||||
pub struct EngineCoreSamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling.
|
||||
#[serde(default = "default_temperature")]
|
||||
pub temperature: f32,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
#[serde(default = "default_top_p")]
|
||||
pub top_p: f32,
|
||||
/// Maximum number of top tokens to consider. `0` means all tokens.
|
||||
#[serde(default)]
|
||||
pub top_k: u32,
|
||||
/// Random seed used by the sampler when present.
|
||||
pub seed: Option<i64>,
|
||||
/// Maximum number of tokens to generate per output sequence.
|
||||
#[serde(default = "default_max_tokens")]
|
||||
pub max_tokens: u32,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
#[serde(default)]
|
||||
pub min_tokens: u32,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
@@ -274,7 +286,6 @@ pub struct EngineCoreSamplingParams {
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
#[serde(default)]
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
pub frequency_penalty: f32,
|
||||
@@ -301,16 +312,13 @@ pub struct EngineCoreSamplingParams {
|
||||
pub all_stop_token_ids: BTreeSet<u32>,
|
||||
/// Logit biases to apply during sampling.
|
||||
/// Keys are token IDs
|
||||
#[serde(default)]
|
||||
pub logit_bias: Option<HashMap<u32, f32>>,
|
||||
/// Restrict output to these token IDs only.
|
||||
#[serde(default)]
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
/// Tokenized bad words to avoid during generation.
|
||||
#[serde(default, rename = "_bad_words_token_ids")]
|
||||
#[serde(rename = "_bad_words_token_ids")]
|
||||
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
#[serde(default)]
|
||||
pub structured_outputs: Option<StructuredOutputsParams>,
|
||||
/// Specific token IDs for which log probabilities should be returned at
|
||||
/// each position.
|
||||
@@ -318,15 +326,12 @@ pub struct EngineCoreSamplingParams {
|
||||
/// When set, the engine returns logprobs for exactly these tokens in
|
||||
/// addition to the sampled/scored token. Mutually exclusive with the
|
||||
/// `logprobs` count field in practice.
|
||||
#[serde(default)]
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
/// If `Some(true)`, the request will not attempt to read from the prefix
|
||||
/// cache; newly computed blocks may still populate the cache. `None`
|
||||
/// defers to engine-core defaults.
|
||||
#[serde(default)]
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions (from `vllm_xargs`).
|
||||
#[serde(default)]
|
||||
pub extra_args: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
@@ -640,4 +645,58 @@ mod tests {
|
||||
let value = serde_json::to_value(params).unwrap();
|
||||
assert_eq!(value["_backend"], "guidance");
|
||||
}
|
||||
|
||||
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
|
||||
/// must fall back to defaults. `python_compat` can't catch this since Rust
|
||||
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
|
||||
#[test]
|
||||
fn decodes_sampling_params_with_omitted_defaults() {
|
||||
let sampling_params = Value::Map(vec![
|
||||
(
|
||||
Value::from("stop_token_ids"),
|
||||
Value::Array(vec![Value::from(151643u32)]),
|
||||
),
|
||||
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
|
||||
]);
|
||||
let request = Value::Array(vec![
|
||||
Value::from("req-omit-defaults"),
|
||||
Value::Array(vec![
|
||||
Value::from(1u32),
|
||||
Value::from(2u32),
|
||||
Value::from(3u32),
|
||||
]),
|
||||
Value::Nil,
|
||||
sampling_params,
|
||||
Value::Nil,
|
||||
Value::from(1.0f64),
|
||||
]);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
rmpv::encode::write_value(&mut bytes, &request).unwrap();
|
||||
|
||||
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
|
||||
.expect("a real omit_defaults request must decode (regression: missing field)");
|
||||
|
||||
assert_eq!(decoded.request_id, "req-omit-defaults");
|
||||
let sampling = decoded.sampling_params.expect("sampling params present");
|
||||
|
||||
assert_eq!(sampling.stop_token_ids, vec![151643]);
|
||||
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
|
||||
|
||||
// Omitted fields -> Python defaults.
|
||||
assert_eq!(sampling.temperature, 1.0);
|
||||
assert_eq!(sampling.top_p, 1.0);
|
||||
assert_eq!(sampling.top_k, 0);
|
||||
assert_eq!(sampling.seed, None);
|
||||
assert_eq!(sampling.max_tokens, 16);
|
||||
assert_eq!(sampling.min_tokens, 0);
|
||||
assert_eq!(sampling.min_p, 0.0);
|
||||
assert_eq!(sampling.frequency_penalty, 0.0);
|
||||
assert_eq!(sampling.presence_penalty, 0.0);
|
||||
assert_eq!(sampling.repetition_penalty, 1.0);
|
||||
assert_eq!(sampling.logprobs, None);
|
||||
assert_eq!(sampling.prompt_logprobs, None);
|
||||
assert_eq!(sampling.eos_token_id, None);
|
||||
assert!(sampling.all_stop_token_ids.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2438,6 +2438,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
let mut lines = stdout.lines();
|
||||
let request_hex = lines.next().expect("missing request fixture line");
|
||||
let defaults_request_hex = lines.next().expect("missing defaults request fixture line");
|
||||
let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line");
|
||||
let outputs_hex = lines.next().expect("missing outputs fixture line");
|
||||
let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line");
|
||||
@@ -2455,6 +2456,42 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let expected_request = sample_request();
|
||||
assert_eq!(decoded_request, expected_request);
|
||||
|
||||
// All-default sampling params -> empty map; must decode to Python defaults.
|
||||
let defaults_request_bytes = hex::decode(defaults_request_hex).unwrap();
|
||||
let decoded_defaults: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&defaults_request_bytes).unwrap();
|
||||
assert_eq!(decoded_defaults.request_id, "req-defaults");
|
||||
let sampling = decoded_defaults
|
||||
.sampling_params
|
||||
.expect("defaults request carries sampling params");
|
||||
assert_eq!(
|
||||
sampling,
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 16,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: Vec::new(),
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: BTreeSet::new(),
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
},
|
||||
);
|
||||
|
||||
let decoded_multimodal_request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&multimodal_request_bytes).unwrap();
|
||||
assert_eq!(decoded_multimodal_request, sample_multimodal_request());
|
||||
|
||||
@@ -31,12 +31,13 @@ class FinishReason(IntEnum):
|
||||
REPETITION = 4
|
||||
|
||||
|
||||
class EngineCoreSamplingParams(msgspec.Struct, dict=True):
|
||||
# Mirror of real SamplingParams; omit_defaults makes fixtures match real maps.
|
||||
class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
|
||||
temperature: float = 1.0
|
||||
top_p: float = 1.0
|
||||
top_k: int = 0
|
||||
seed: int | None = None
|
||||
max_tokens: int = 65536
|
||||
max_tokens: int = 16
|
||||
min_tokens: int = 0
|
||||
min_p: float = 0.0
|
||||
frequency_penalty: float = 0.0
|
||||
@@ -135,6 +136,16 @@ request = EngineCoreRequest(
|
||||
client_index=0,
|
||||
)
|
||||
|
||||
# All defaults -> empty map. Regression guard for the sparse-map decode.
|
||||
defaults_request = EngineCoreRequest(
|
||||
request_id="req-defaults",
|
||||
prompt_token_ids=[5, 6, 7],
|
||||
mm_features=None,
|
||||
sampling_params=EngineCoreSamplingParams(),
|
||||
pooling_params=None,
|
||||
arrival_time=1.0,
|
||||
)
|
||||
|
||||
multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
|
||||
multimodal_features = [
|
||||
{
|
||||
@@ -361,6 +372,7 @@ ready_response = EngineCoreReadyResponse(
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
print(msgspec.msgpack.encode(defaults_request).hex())
|
||||
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
|
||||
print(msgspec.msgpack.encode(outputs).hex())
|
||||
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))
|
||||
|
||||
Reference in New Issue
Block a user