[Rust Frontend] Split engine core DTOs into separate modules (#47265)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-01 15:28:21 +01:00
committed by GitHub
parent 13c49f9845
commit c638f9216a
54 changed files with 806 additions and 789 deletions
+2 -1
View File
@@ -11,10 +11,11 @@ use tracing::{debug, info, trace};
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::dtype::ModelDtype;
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::lora::LoraRequest;
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
use crate::transport::{self, ConnectedEngine};
@@ -16,12 +16,11 @@ use crate::client::stream::EngineCoreStreamOutput;
use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::request::EngineCoreRequestType;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::protocol::{
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
encode_msgpack,
};
use crate::transport::{ConnectedEngine, EngineId};
use crate::{Error, Result, transport};
@@ -7,9 +7,9 @@ use tracing::trace;
use crate::EngineId;
use crate::client::stream::EngineCoreStreamOutput;
use crate::error::{Error, Result};
use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::protocol::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
use crate::transport::ConnectedEngine;
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -452,7 +452,7 @@ mod tests {
EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
};
use crate::mock_engine::default_ready_response;
use crate::protocol::{
use crate::protocol::output::{
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
};
use crate::transport::ConnectedEngine;
@@ -10,7 +10,7 @@ use tracing::{debug, error, warn};
use crate::client::AbortRequest;
use crate::client::state::OutputReceiver;
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
use crate::{AbortCause, Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -10,10 +10,9 @@ use zeromq::{XPubSocket, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::{
ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
encode_msgpack,
};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs};
use crate::protocol::request::EngineCoreRequestType;
/// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
/// engine-facing coordinator socket.
@@ -8,8 +8,9 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
use crate::EngineId;
use crate::error::{Error, Result, bail_unexpected_handshake_message};
use crate::protocol::dtype::ModelDtype;
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
use crate::protocol::{decode_msgpack, encode_msgpack};
/// Default model length advertised by reusable mock engine helpers.
pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
@@ -2,7 +2,8 @@ use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::protocol::{ModelDtype, OpaqueValue};
use crate::protocol::OpaqueValue;
use crate::protocol::dtype::ModelDtype;
/// Decoded engine startup-handshake payload sent on the handshake socket.
///
@@ -9,8 +9,7 @@ use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Deserializer, Serialize};
use self::wire::*;
use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
use crate::error::{Error, Result, bail_ext_value_decode};
use crate::protocol::tensor::{WireArrayData, WireNdArray};
/// One token candidate and its logprob metadata for a single sequence position.
@@ -160,7 +159,7 @@ impl Serialize for MaybeWireLogprobs {
impl MaybeWireLogprobs {
/// Resolve the wire representation into decoded logprobs by looking up aux
/// frames and decoding raw views as needed.
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
pub(super) fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
where
Frame: AsRef<[u8]>,
{
@@ -171,37 +170,6 @@ impl MaybeWireLogprobs {
}
}
impl EngineCoreOutputs {
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
for output in &mut self.outputs {
output.resolve_in_place(frames)?;
}
Ok(())
}
}
impl EngineCoreOutput {
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
self.new_logprobs = (self.new_logprobs.take())
.map(|value| value.resolve(frames, "new_logprobs"))
.transpose()?;
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
.transpose()?;
Ok(())
}
}
impl WireLogprobs {
/// Convert semantic per-position logprobs into the Python wire tuple shape.
///
@@ -315,16 +283,3 @@ impl WireLogprobs {
Ok(Logprobs { positions })
}
}
/// Decode one ordinary or multipart engine-core output message into the strong
/// typed public protocol shape.
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
where
Frame: AsRef<[u8]>,
{
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
outputs.resolve_in_place(frames)?;
Ok(outputs)
}
@@ -3,8 +3,8 @@ use std::collections::BTreeSet;
use bytes::Bytes;
use rmpv::Value;
use super::{Logprobs, PositionLogprobs, TokenLogprob, decode_engine_core_outputs};
use crate::protocol::EngineCoreFinishReason;
use super::{Logprobs, PositionLogprobs, TokenLogprob};
use crate::protocol::output::{EngineCoreFinishReason, decode_engine_core_outputs};
fn encode_value(value: &Value) -> Vec<u8> {
let mut out = Vec::new();
+6 -636
View File
@@ -1,28 +1,11 @@
use std::any::type_name;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::Cursor;
use bytes::Bytes;
use rmpv::Value;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_repr::{Deserialize_repr, Serialize_repr};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use crate::error::{Error, Result};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::MmFeatures;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::utility::UtilityOutput;
// TODO: This module currently mixes reusable frontend-facing semantic types
// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
// cleaned-up frontend sampling types) with engine-core-specific wire DTOs and
// handshake/control messages. While the Rust frontend is still evolving
// quickly, keep them co-located here for iteration speed. Once the higher-level
// API boundary stabilizes, move the truly reusable semantic types into a
// lower-level common crate and keep the engine transport/wire messages here.
/// Dynamic msgpack value used for schema positions that are preserved but not
/// yet strongly typed in the early-stage Rust client.
@@ -36,499 +19,18 @@ fn is_false(v: &bool) -> bool {
!v
}
fn default_top_p() -> f32 {
1.0
}
fn default_repetition_penalty() -> f32 {
1.0
}
fn default_temperature() -> f32 {
1.0
}
fn default_max_tokens() -> u32 {
16
}
mod classified_outputs;
pub mod dtype;
pub mod handshake;
pub mod logprobs;
pub mod lora;
pub mod multimodal;
pub mod output;
pub mod request;
pub mod sampling;
pub mod stats;
pub mod structured_outputs;
pub mod tensor;
pub mod utility;
pub use classified_outputs::{
ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
};
pub use dtype::ModelDtype;
pub use logprobs::decode_engine_core_outputs;
/// Request types are encoded as single-byte protocol constants so they can be
/// sent over the ZMQ socket without an extra encoding step.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum EngineCoreRequestType {
Add = 0,
Abort = 1,
StartDpWave = 2,
Utility = 3,
}
impl EngineCoreRequestType {
/// Decode the single-byte request type frame used on the engine input
/// socket. Returns `None` for unrecognized values.
pub fn from_frame(frame: &[u8]) -> Option<Self> {
let [value] = frame else {
return None;
};
match value {
0 => Some(Self::Add),
1 => Some(Self::Abort),
2 => Some(Self::StartDpWave),
3 => Some(Self::Utility),
_ => None,
}
}
/// Encode the request type as the single-byte frame used on the engine
/// input socket.
pub fn to_frame(self) -> Bytes {
Bytes::from_static(match self {
Self::Add => b"\x00",
Self::Abort => b"\x01",
Self::StartDpWave => b"\x02",
Self::Utility => b"\x03",
})
}
}
/// Reason a request finished: stop, length, abort, error, or repetition.
///
/// This mirrors the Python enum and uses integer encoding for compact wire
/// representation.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreFinishReason {
/// A stop string was emitted.
Stop = 0,
/// `max_tokens` or `max_model_len` was reached.
Length = 1,
/// The request was aborted by the client.
Abort = 2,
/// A retryable request-level internal error occurred.
Error = 3,
/// A repetitive token pattern was detected.
Repetition = 4,
}
/// Event types emitted by engine-core for one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreEventType {
Queued = 1,
Scheduled = 2,
Preempted = 3,
}
/// A timestamped engine-core event associated with one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EngineCoreEvent {
pub r#type: EngineCoreEventType,
pub timestamp: f64,
}
/// Controls how intermediate outputs are returned to the frontend.
///
/// `Cumulative = 0` is intentionally not supported in Rust frontend.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L146-L152>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum RequestOutputKind {
/// Return only token deltas in each update.
#[default]
Delta = 1,
/// Suppress intermediate updates and return only the final output.
FinalOnly = 2,
}
/// Structured-output backend selected for EngineCore grammar compilation.
///
/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
/// validation. The Rust frontend currently always lowers structured-output
/// requests to guidance, while ignoring any user-supplied `_backend` value.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum StructuredOutputBackend {
Xgrammar,
#[default]
Guidance,
Outlines,
LmFormatEnforcer,
}
/// The stop reason associated with a finished output.
///
/// Python models this as the union-typed `stop_reason: int | str | None`
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
///
/// Original Python field:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum StopReason {
TokenId(u32),
Text(String),
}
/// Parameters for configuring structured outputs (guided decoding).
///
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
/// `json_object`, or `structural_tag`) should be set. The engine-core
/// backend selects the appropriate grammar compiler based on which field
/// is present.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct StructuredOutputsParams {
/// JSON schema (as a dict/object or JSON string) constraining the output.
pub json: Option<serde_json::Value>,
/// Regular expression the output must match.
pub regex: Option<String>,
/// List of allowed output strings (the model must produce one of these).
pub choice: Option<Vec<String>>,
/// Context-free grammar (in EBNF-like notation) the output must conform to.
pub grammar: Option<String>,
/// When `true`, output must be valid JSON (free-form, no schema).
pub json_object: Option<bool>,
/// Disable any additional whitespace in guided JSON output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_any_whitespace: bool,
/// Disable `additionalProperties` in JSON schema output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_additional_properties: bool,
/// Custom whitespace pattern for guided JSON output.
pub whitespace_pattern: Option<String>,
/// Structural tag configuration (JSON-encoded string).
pub structural_tag: Option<String>,
/// Structured-output backend, mirroring Python's internal `_backend`.
///
/// User-supplied values are ignored during deserialization. This matches
/// Python's request boundary, where `_backend` is set by validation rather
/// than accepted as a request-level backend selector.
#[serde(
default,
rename = "_backend",
deserialize_with = "serde_with::rust::deserialize_ignore_any"
)]
pub backend: StructuredOutputBackend,
}
/// Engine-core-facing sampling parameters for text generation.
///
/// This is the normalized southbound subset used by the Rust frontend when it
/// talks to Python engine-core over the wire. User-facing request semantics
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
/// intentionally handled by higher layers before values reach this DTO.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
// default-valued keys; default the whole struct. Per-field fns cover the
// non-zero defaults.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
#[serde(default)]
pub struct EngineCoreSamplingParams {
/// Controls randomness. Lower values are more deterministic; zero means
/// greedy sampling.
#[serde(default = "default_temperature")]
pub temperature: f32,
/// Cumulative probability threshold for nucleus sampling.
#[serde(default = "default_top_p")]
pub top_p: f32,
/// Maximum number of top tokens to consider. `0` means all tokens.
pub top_k: u32,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate per output sequence.
#[serde(default = "default_max_tokens")]
pub max_tokens: u32,
/// Minimum number of tokens to generate before EOS or stop-token handling.
pub min_tokens: u32,
/// Maximum number of reasoning ("thinking") tokens to emit before the
/// reasoning section is force-closed. `None` means unlimited; the
/// user-facing `-1` sentinel is normalized to `None` by the frontend before
/// reaching this DTO, so only non-negative values are sent. Enforced
/// engine-side (and only when a reasoning parser is configured).
pub thinking_token_budget: Option<u64>,
/// Number of log probabilities to return per generated token.
///
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
pub logprobs: Option<i32>,
/// Number of log probabilities to return per prompt token.
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
pub frequency_penalty: f32,
/// Presence penalty applied by the sampler.
pub presence_penalty: f32,
/// Repetition penalty applied by the sampler.
#[serde(default = "default_repetition_penalty")]
pub repetition_penalty: f32,
/// Token IDs that stop generation.
pub stop_token_ids: Vec<u32>,
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
///
/// This mirrors Python's internal `_eos_token_id` field and is derived by
/// the frontend from tokenizer/model metadata rather than supplied directly
/// by end users.
#[serde(rename = "_eos_token_id")]
pub eos_token_id: Option<u32>,
/// Complete stop-token set used by engine-core for `min_tokens` masking.
///
/// This mirrors Python's internal `_all_stop_token_ids` field and should
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
/// IDs.
#[serde(rename = "_all_stop_token_ids")]
pub all_stop_token_ids: BTreeSet<u32>,
/// Logit biases to apply during sampling.
/// Keys are token IDs
pub logit_bias: Option<HashMap<u32, f32>>,
/// Restrict output to these token IDs only.
pub allowed_token_ids: Option<Vec<u32>>,
/// Tokenized bad words to avoid during generation.
#[serde(rename = "_bad_words_token_ids")]
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
/// Parameters for configuring structured outputs (guided decoding).
pub structured_outputs: Option<StructuredOutputsParams>,
/// Specific token IDs for which log probabilities should be returned at
/// each position.
///
/// When set, the engine returns logprobs for exactly these tokens in
/// addition to the sampled/scored token. Mutually exclusive with the
/// `logprobs` count field in practice.
pub logprob_token_ids: Option<Vec<u32>>,
/// If `Some(true)`, the request will not attempt to read from the prefix
/// cache; newly computed blocks may still populate the cache. `None`
/// defers to engine-core defaults.
pub skip_reading_prefix_cache: Option<bool>,
/// Additional request parameters for custom extensions (from `vllm_xargs`).
pub extra_args: Option<HashMap<String, serde_json::Value>>,
}
impl EngineCoreSamplingParams {
/// Constructs a default sampling params for testing purposes only.
pub fn for_test() -> Self {
Self {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 65536,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: Vec::new(),
eos_token_id: None,
all_stop_token_ids: BTreeSet::new(),
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
}
}
/// Extra kwargs consumed by engine-side reasoning parsers.
///
/// Original Python construction point:
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ReasoningParserKwargs {
/// Effective kwargs visible to the chat template for this request.
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
}
/// Engine-core add-request payload sent from frontend to engine.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreRequest {
pub request_id: String,
pub prompt_token_ids: Option<Vec<u32>>,
/// Multimodal features attached to the request.
pub mm_features: Option<MmFeatures>,
pub sampling_params: Option<EngineCoreSamplingParams>,
/// Pooling parameters are preserved in the schema but not yet strongly
/// typed.
pub pooling_params: Option<OpaqueValue>,
pub arrival_time: f64,
#[serde(default)]
pub lora_request: Option<lora::LoraRequest>,
#[serde(default)]
pub cache_salt: Option<String>,
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Unsupported in the first-stage Rust client because Python uses a custom
/// tensor/aux-frame encoding path for this field.
#[serde(default)]
pub prompt_embeds: Option<OpaqueValue>,
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
/// `prompt_embeds` content parts). `Some(true)` means real token id;
/// `Some(false)` means the position uses a pre-computed entry from
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
#[serde(default)]
pub prompt_is_token_ids: Option<Vec<bool>>,
/// Index of the client, used to ensure outputs are sent back to the same
/// client when scaling out the frontend.
#[serde(default)]
pub client_index: u32,
/// In DP mode, indicates which wave this request is expected to belong to.
#[serde(default)]
pub current_wave: u32,
#[serde(default)]
pub priority: i32,
#[serde(default)]
pub trace_headers: Option<BTreeMap<String, String>>,
#[serde(default)]
pub resumable: bool,
/// Original user-provided request ID, used for output reporting and aborts.
#[serde(default)]
pub external_req_id: Option<String>,
#[serde(default)]
pub reasoning_ended: Option<bool>,
/// Reasoning-parser kwargs forwarded from the frontend to the
/// structured-output backend.
#[serde(default)]
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
/// If `true`, the request should be added to the scheduler's waiting queue
/// and immediately aborted, so connector-side cleanup runs via the
/// standard `request_finished` hook.
#[serde(default)]
pub abort_immediately: bool,
}
impl EngineCoreRequest {
/// Validate fields intentionally not supported in the first-stage client.
pub fn validate(&self) -> Result<()> {
if self.prompt_embeds.is_some() {
return Err(Error::UnsupportedField {
context: "EngineCoreRequest",
field: "prompt_embeds",
});
}
Ok(())
}
}
/// Engine-core output for a single request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutput {
pub request_id: String,
pub new_token_ids: Vec<u32>,
/// Decoded sample logprobs for the newly generated positions in this
/// output.
#[serde(default)]
pub new_logprobs: Option<MaybeWireLogprobs>,
/// Decoded prompt logprobs for the scored prompt positions emitted in this
/// output.
#[serde(default)]
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
#[serde(default)]
pub pooling_output: Option<OpaqueValue>,
#[serde(default)]
pub finish_reason: Option<EngineCoreFinishReason>,
#[serde(default)]
pub stop_reason: Option<StopReason>,
#[serde(default)]
pub events: Option<Vec<EngineCoreEvent>>,
#[serde(default)]
pub kv_transfer_params: Option<serde_json::Value>,
#[serde(default)]
pub trace_headers: Option<OpaqueValue>,
/// Breakdown of the scheduled prefill computation, set on the first output
/// of a newly scheduled prefill and elided for subsequent decode outputs.
#[serde(default)]
pub prefill_stats: Option<PrefillStats>,
#[serde(default)]
pub routed_experts: Option<OpaqueValue>,
/// Number of NaNs seen in logits. Values above zero indicate corruption.
#[serde(default)]
pub num_nans_in_logits: u32,
}
impl EngineCoreOutput {
/// Returns whether this output is terminal for the request.
pub fn finished(&self) -> bool {
self.finish_reason.is_some()
}
}
/// Batch of engine-core outputs returned to a frontend client.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutputs {
#[serde(default)]
pub engine_index: u32,
/// Outputs grouped for this client in the current engine tick.
#[serde(default)]
pub outputs: Vec<EngineCoreOutput>,
#[serde(default)]
pub scheduler_stats: Option<Box<SchedulerStats>>,
#[serde(default)]
pub timestamp: f64,
#[serde(default)]
pub utility_output: Option<UtilityOutput>,
#[serde(default)]
pub finished_requests: Option<BTreeSet<String>>,
/// In DP mode, signals that the current wave finished and engines are
/// paused.
#[serde(default)]
pub wave_complete: Option<u32>,
/// In DP mode, signals that a request arrived for an old wave and the next
/// wave needs to start in other engines.
#[serde(default)]
pub start_wave: Option<u32>,
}
/// Encode a Rust value into msgpack using the protocol crate's serde model.
pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
@@ -564,81 +66,17 @@ where
})
}
/// Decode a msgpack payload into a dynamic value for diagnostics and tests.
pub fn decode_value(bytes: &[u8]) -> Result<Value> {
Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use std::collections::BTreeMap;
use super::*;
#[test]
fn engine_core_request_serializes_as_full_array() {
let request = EngineCoreRequest {
request_id: "req-1".to_string(),
prompt_token_ids: Some(vec![1, 2, 3]),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens: 8,
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 1234.5,
client_index: 7,
..EngineCoreRequest::default()
};
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected array, got {other:?}"),
};
assert_eq!(array.len(), 20);
assert_eq!(array[0], Value::from("req-1"));
assert_eq!(array[2], Value::Nil);
assert_eq!(array[4], Value::Nil);
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
}
#[test]
fn engine_core_outputs_roundtrip_finished_fields() {
let outputs = EngineCoreOutputs {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![42],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: Some(EngineCoreFinishReason::Length),
stop_reason: Some(StopReason::Text("stop".to_string())),
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
};
let encoded = encode_msgpack(&outputs).unwrap();
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
assert_eq!(decoded.outputs.len(), 1);
assert_eq!(
decoded.outputs[0].finish_reason,
Some(EngineCoreFinishReason::Length)
);
assert_eq!(
decoded.finished_requests,
Some(BTreeSet::from(["req-1".to_string()]))
);
}
#[test]
fn decode_msgpack_includes_type_name_and_value_fallback() {
let error = decode_msgpack::<u64>(
@@ -648,72 +86,4 @@ mod tests {
expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
}
#[test]
fn structured_outputs_backend_ignores_deserialized_value() {
let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
"json_object": true,
"_backend": "xgrammar",
}))
.unwrap();
assert_eq!(params.backend, StructuredOutputBackend::Guidance);
let value = serde_json::to_value(params).unwrap();
assert_eq!(value["_backend"], "guidance");
}
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
/// must fall back to defaults. `python_compat` can't catch this since Rust
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
#[test]
fn decodes_sampling_params_with_omitted_defaults() {
let sampling_params = Value::Map(vec![
(
Value::from("stop_token_ids"),
Value::Array(vec![Value::from(151643u32)]),
),
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
]);
let request = Value::Array(vec![
Value::from("req-omit-defaults"),
Value::Array(vec![
Value::from(1u32),
Value::from(2u32),
Value::from(3u32),
]),
Value::Nil,
sampling_params,
Value::Nil,
Value::from(1.0f64),
]);
let mut bytes = Vec::new();
rmpv::encode::write_value(&mut bytes, &request).unwrap();
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
.expect("a real omit_defaults request must decode (regression: missing field)");
assert_eq!(decoded.request_id, "req-omit-defaults");
let sampling = decoded.sampling_params.expect("sampling params present");
assert_eq!(sampling.stop_token_ids, vec![151643]);
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
// Omitted fields -> Python defaults.
assert_eq!(sampling.temperature, 1.0);
assert_eq!(sampling.top_p, 1.0);
assert_eq!(sampling.top_k, 0);
assert_eq!(sampling.seed, None);
assert_eq!(sampling.max_tokens, 16);
assert_eq!(sampling.min_tokens, 0);
assert_eq!(sampling.min_p, 0.0);
assert_eq!(sampling.frequency_penalty, 0.0);
assert_eq!(sampling.presence_penalty, 0.0);
assert_eq!(sampling.repetition_penalty, 1.0);
assert_eq!(sampling.logprobs, None);
assert_eq!(sampling.prompt_logprobs, None);
assert_eq!(sampling.eos_token_id, None);
assert!(sampling.all_stop_token_ids.is_empty());
}
}
@@ -1,10 +1,164 @@
use std::collections::BTreeSet;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_repr::{Deserialize_repr, Serialize_repr};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use super::utility::UtilityOutput;
use super::{EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::stats::SchedulerStats;
use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::{OpaqueValue, decode_msgpack};
/// The stop reason associated with a finished output.
///
/// Python models this as the union-typed `stop_reason: int | str | None`
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
///
/// Original Python field:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum StopReason {
TokenId(u32),
Text(String),
}
/// Reason a request finished: stop, length, abort, error, or repetition.
///
/// This mirrors the Python enum and uses integer encoding for compact wire
/// representation.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreFinishReason {
/// A stop string was emitted.
Stop = 0,
/// `max_tokens` or `max_model_len` was reached.
Length = 1,
/// The request was aborted by the client.
Abort = 2,
/// A retryable request-level internal error occurred.
Error = 3,
/// A repetitive token pattern was detected.
Repetition = 4,
}
/// Event types emitted by engine-core for one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreEventType {
Queued = 1,
Scheduled = 2,
Preempted = 3,
}
/// A timestamped engine-core event associated with one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EngineCoreEvent {
pub r#type: EngineCoreEventType,
pub timestamp: f64,
}
/// Engine-core output for a single request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutput {
pub request_id: String,
pub new_token_ids: Vec<u32>,
/// Decoded sample logprobs for the newly generated positions in this
/// output.
#[serde(default)]
pub new_logprobs: Option<MaybeWireLogprobs>,
/// Decoded prompt logprobs for the scored prompt positions emitted in this
/// output.
#[serde(default)]
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
#[serde(default)]
pub pooling_output: Option<OpaqueValue>,
#[serde(default)]
pub finish_reason: Option<EngineCoreFinishReason>,
#[serde(default)]
pub stop_reason: Option<StopReason>,
#[serde(default)]
pub events: Option<Vec<EngineCoreEvent>>,
#[serde(default)]
pub kv_transfer_params: Option<serde_json::Value>,
#[serde(default)]
pub trace_headers: Option<OpaqueValue>,
/// Breakdown of the scheduled prefill computation, set on the first output
/// of a newly scheduled prefill and elided for subsequent decode outputs.
#[serde(default)]
pub prefill_stats: Option<PrefillStats>,
#[serde(default)]
pub routed_experts: Option<OpaqueValue>,
/// Number of NaNs seen in logits. Values above zero indicate corruption.
#[serde(default)]
pub num_nans_in_logits: u32,
}
impl EngineCoreOutput {
/// Returns whether this output is terminal for the request.
pub fn finished(&self) -> bool {
self.finish_reason.is_some()
}
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
self.new_logprobs = (self.new_logprobs.take())
.map(|value| value.resolve(frames, "new_logprobs"))
.transpose()?;
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
.transpose()?;
Ok(())
}
}
/// Batch of engine-core outputs returned to a frontend client.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutputs {
#[serde(default)]
pub engine_index: u32,
/// Outputs grouped for this client in the current engine tick.
#[serde(default)]
pub outputs: Vec<EngineCoreOutput>,
#[serde(default)]
pub scheduler_stats: Option<Box<SchedulerStats>>,
#[serde(default)]
pub timestamp: f64,
#[serde(default)]
pub utility_output: Option<UtilityOutput>,
#[serde(default)]
pub finished_requests: Option<BTreeSet<String>>,
/// In DP mode, signals that the current wave finished and engines are
/// paused.
#[serde(default)]
pub wave_complete: Option<u32>,
/// In DP mode, signals that a request arrived for an old wave and the next
/// wave needs to start in other engines.
#[serde(default)]
pub start_wave: Option<u32>,
}
/// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -49,6 +203,18 @@ pub enum ClassifiedEngineCoreOutputs {
}
impl EngineCoreOutputs {
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
for output in &mut self.outputs {
output.resolve_in_place(frames)?;
}
Ok(())
}
/// Classify the raw wire message into a more semantic Rust enum.
pub fn classify(self) -> ClassifiedEngineCoreOutputs {
let has_request_payload = !self.outputs.is_empty()
@@ -92,12 +258,62 @@ impl EngineCoreOutputs {
}
}
/// Decode one ordinary or multipart engine-core output message into the strong
/// typed public protocol shape.
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
where
Frame: AsRef<[u8]>,
{
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
outputs.resolve_in_place(frames)?;
Ok(outputs)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use crate::protocol::EngineCoreOutput;
use crate::protocol::output::EngineCoreOutput;
use crate::protocol::{decode_msgpack, encode_msgpack};
#[test]
fn engine_core_outputs_roundtrip_finished_fields() {
let outputs = EngineCoreOutputs {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![42],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: Some(EngineCoreFinishReason::Length),
stop_reason: Some(StopReason::Text("stop".to_string())),
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
};
let encoded = encode_msgpack(&outputs).unwrap();
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
assert_eq!(decoded.outputs.len(), 1);
assert_eq!(
decoded.outputs[0].finish_reason,
Some(EngineCoreFinishReason::Length)
);
assert_eq!(
decoded.finished_requests,
Some(BTreeSet::from(["req-1".to_string()]))
);
}
#[test]
fn engine_core_outputs_classify_request_batch() {
@@ -0,0 +1,175 @@
use std::collections::{BTreeMap, HashMap};
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::protocol::multimodal::MmFeatures;
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::{OpaqueValue, lora};
use crate::{Error, Result};
/// Request types are encoded as single-byte protocol constants so they can be
/// sent over the ZMQ socket without an extra encoding step.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum EngineCoreRequestType {
Add = 0,
Abort = 1,
StartDpWave = 2,
Utility = 3,
}
impl EngineCoreRequestType {
/// Decode the single-byte request type frame used on the engine input
/// socket. Returns `None` for unrecognized values.
pub fn from_frame(frame: &[u8]) -> Option<Self> {
let [value] = frame else {
return None;
};
match value {
0 => Some(Self::Add),
1 => Some(Self::Abort),
2 => Some(Self::StartDpWave),
3 => Some(Self::Utility),
_ => None,
}
}
/// Encode the request type as the single-byte frame used on the engine
/// input socket.
pub fn to_frame(self) -> Bytes {
Bytes::from_static(match self {
Self::Add => b"\x00",
Self::Abort => b"\x01",
Self::StartDpWave => b"\x02",
Self::Utility => b"\x03",
})
}
}
/// Extra kwargs consumed by engine-side reasoning parsers.
///
/// Original Python construction point:
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ReasoningParserKwargs {
/// Effective kwargs visible to the chat template for this request.
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
}
/// Engine-core add-request payload sent from frontend to engine.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreRequest {
pub request_id: String,
pub prompt_token_ids: Option<Vec<u32>>,
/// Multimodal features attached to the request.
pub mm_features: Option<MmFeatures>,
pub sampling_params: Option<EngineCoreSamplingParams>,
/// Pooling parameters are preserved in the schema but not yet strongly
/// typed.
pub pooling_params: Option<OpaqueValue>,
pub arrival_time: f64,
#[serde(default)]
pub lora_request: Option<lora::LoraRequest>,
#[serde(default)]
pub cache_salt: Option<String>,
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Unsupported in the first-stage Rust client because Python uses a custom
/// tensor/aux-frame encoding path for this field.
#[serde(default)]
pub prompt_embeds: Option<OpaqueValue>,
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
/// `prompt_embeds` content parts). `Some(true)` means real token id;
/// `Some(false)` means the position uses a pre-computed entry from
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
#[serde(default)]
pub prompt_is_token_ids: Option<Vec<bool>>,
/// Index of the client, used to ensure outputs are sent back to the same
/// client when scaling out the frontend.
#[serde(default)]
pub client_index: u32,
/// In DP mode, indicates which wave this request is expected to belong to.
#[serde(default)]
pub current_wave: u32,
#[serde(default)]
pub priority: i32,
#[serde(default)]
pub trace_headers: Option<BTreeMap<String, String>>,
#[serde(default)]
pub resumable: bool,
/// Original user-provided request ID, used for output reporting and aborts.
#[serde(default)]
pub external_req_id: Option<String>,
#[serde(default)]
pub reasoning_ended: Option<bool>,
/// Reasoning-parser kwargs forwarded from the frontend to the
/// structured-output backend.
#[serde(default)]
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
/// If `true`, the request should be added to the scheduler's waiting queue
/// and immediately aborted, so connector-side cleanup runs via the
/// standard `request_finished` hook.
#[serde(default)]
pub abort_immediately: bool,
}
impl EngineCoreRequest {
/// Validate fields intentionally not supported in the first-stage client.
pub fn validate(&self) -> Result<()> {
if self.prompt_embeds.is_some() {
return Err(Error::UnsupportedField {
context: "EngineCoreRequest",
field: "prompt_embeds",
});
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use rmpv::Value;
use super::*;
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::{decode_value, encode_msgpack};
#[test]
fn engine_core_request_serializes_as_full_array() {
let request = EngineCoreRequest {
request_id: "req-1".to_string(),
prompt_token_ids: Some(vec![1, 2, 3]),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens: 8,
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 1234.5,
client_index: 7,
..EngineCoreRequest::default()
};
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected array, got {other:?}"),
};
assert_eq!(array.len(), 20);
assert_eq!(array[0], Value::from("req-1"));
assert_eq!(array[2], Value::Nil);
assert_eq!(array[4], Value::Nil);
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
}
}
@@ -0,0 +1,211 @@
use std::collections::{BTreeSet, HashMap};
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use crate::protocol::structured_outputs::StructuredOutputsParams;
fn default_top_p() -> f32 {
1.0
}
fn default_repetition_penalty() -> f32 {
1.0
}
fn default_temperature() -> f32 {
1.0
}
fn default_max_tokens() -> u32 {
16
}
/// Engine-core-facing sampling parameters for text generation.
///
/// This is the normalized southbound subset used by the Rust frontend when it
/// talks to Python engine-core over the wire. User-facing request semantics
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
/// intentionally handled by higher layers before values reach this DTO.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
// default-valued keys; default the whole struct. Per-field fns cover the
// non-zero defaults.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
#[serde(default)]
pub struct EngineCoreSamplingParams {
/// Controls randomness. Lower values are more deterministic; zero means
/// greedy sampling.
#[serde(default = "default_temperature")]
pub temperature: f32,
/// Cumulative probability threshold for nucleus sampling.
#[serde(default = "default_top_p")]
pub top_p: f32,
/// Maximum number of top tokens to consider. `0` means all tokens.
pub top_k: u32,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate per output sequence.
#[serde(default = "default_max_tokens")]
pub max_tokens: u32,
/// Minimum number of tokens to generate before EOS or stop-token handling.
pub min_tokens: u32,
/// Maximum number of reasoning ("thinking") tokens to emit before the
/// reasoning section is force-closed. `None` means unlimited; the
/// user-facing `-1` sentinel is normalized to `None` by the frontend before
/// reaching this DTO, so only non-negative values are sent. Enforced
/// engine-side (and only when a reasoning parser is configured).
pub thinking_token_budget: Option<u64>,
/// Number of log probabilities to return per generated token.
///
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
pub logprobs: Option<i32>,
/// Number of log probabilities to return per prompt token.
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
pub frequency_penalty: f32,
/// Presence penalty applied by the sampler.
pub presence_penalty: f32,
/// Repetition penalty applied by the sampler.
#[serde(default = "default_repetition_penalty")]
pub repetition_penalty: f32,
/// Token IDs that stop generation.
pub stop_token_ids: Vec<u32>,
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
///
/// This mirrors Python's internal `_eos_token_id` field and is derived by
/// the frontend from tokenizer/model metadata rather than supplied directly
/// by end users.
#[serde(rename = "_eos_token_id")]
pub eos_token_id: Option<u32>,
/// Complete stop-token set used by engine-core for `min_tokens` masking.
///
/// This mirrors Python's internal `_all_stop_token_ids` field and should
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
/// IDs.
#[serde(rename = "_all_stop_token_ids")]
pub all_stop_token_ids: BTreeSet<u32>,
/// Logit biases to apply during sampling.
/// Keys are token IDs
pub logit_bias: Option<HashMap<u32, f32>>,
/// Restrict output to these token IDs only.
pub allowed_token_ids: Option<Vec<u32>>,
/// Tokenized bad words to avoid during generation.
#[serde(rename = "_bad_words_token_ids")]
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
/// Parameters for configuring structured outputs (guided decoding).
pub structured_outputs: Option<StructuredOutputsParams>,
/// Specific token IDs for which log probabilities should be returned at
/// each position.
///
/// When set, the engine returns logprobs for exactly these tokens in
/// addition to the sampled/scored token. Mutually exclusive with the
/// `logprobs` count field in practice.
pub logprob_token_ids: Option<Vec<u32>>,
/// If `Some(true)`, the request will not attempt to read from the prefix
/// cache; newly computed blocks may still populate the cache. `None`
/// defers to engine-core defaults.
pub skip_reading_prefix_cache: Option<bool>,
/// Additional request parameters for custom extensions (from `vllm_xargs`).
pub extra_args: Option<HashMap<String, serde_json::Value>>,
}
impl EngineCoreSamplingParams {
/// Constructs a default sampling params for testing purposes only.
pub fn for_test() -> Self {
Self {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 65536,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: Vec::new(),
eos_token_id: None,
all_stop_token_ids: BTreeSet::new(),
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
}
}
#[cfg(test)]
mod tests {
use rmpv::Value;
use crate::protocol::decode_msgpack;
use crate::protocol::request::EngineCoreRequest;
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
/// must fall back to defaults. `python_compat` can't catch this since Rust
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
#[test]
fn decodes_sampling_params_with_omitted_defaults() {
let sampling_params = Value::Map(vec![
(
Value::from("stop_token_ids"),
Value::Array(vec![Value::from(151643u32)]),
),
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
]);
let request = Value::Array(vec![
Value::from("req-omit-defaults"),
Value::Array(vec![
Value::from(1u32),
Value::from(2u32),
Value::from(3u32),
]),
Value::Nil,
sampling_params,
Value::Nil,
Value::from(1.0f64),
]);
let mut bytes = Vec::new();
rmpv::encode::write_value(&mut bytes, &request).unwrap();
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
.expect("a real omit_defaults request must decode (regression: missing field)");
assert_eq!(decoded.request_id, "req-omit-defaults");
let sampling = decoded.sampling_params.expect("sampling params present");
assert_eq!(sampling.stop_token_ids, vec![151643]);
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
// Omitted fields -> Python defaults.
assert_eq!(sampling.temperature, 1.0);
assert_eq!(sampling.top_p, 1.0);
assert_eq!(sampling.top_k, 0);
assert_eq!(sampling.seed, None);
assert_eq!(sampling.max_tokens, 16);
assert_eq!(sampling.min_tokens, 0);
assert_eq!(sampling.min_p, 0.0);
assert_eq!(sampling.frequency_penalty, 0.0);
assert_eq!(sampling.presence_penalty, 0.0);
assert_eq!(sampling.repetition_penalty, 1.0);
assert_eq!(sampling.logprobs, None);
assert_eq!(sampling.prompt_logprobs, None);
assert_eq!(sampling.eos_token_id, None);
assert!(sampling.all_stop_token_ids.is_empty());
}
}
@@ -0,0 +1,81 @@
use serde::{Deserialize, Serialize};
/// Structured-output backend selected for EngineCore grammar compilation.
///
/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
/// validation. The Rust frontend currently always lowers structured-output
/// requests to guidance, while ignoring any user-supplied `_backend` value.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum StructuredOutputBackend {
Xgrammar,
#[default]
Guidance,
Outlines,
LmFormatEnforcer,
}
/// Parameters for configuring structured outputs (guided decoding).
///
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
/// `json_object`, or `structural_tag`) should be set. The engine-core
/// backend selects the appropriate grammar compiler based on which field
/// is present.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct StructuredOutputsParams {
/// JSON schema (as a dict/object or JSON string) constraining the output.
pub json: Option<serde_json::Value>,
/// Regular expression the output must match.
pub regex: Option<String>,
/// List of allowed output strings (the model must produce one of these).
pub choice: Option<Vec<String>>,
/// Context-free grammar (in EBNF-like notation) the output must conform to.
pub grammar: Option<String>,
/// When `true`, output must be valid JSON (free-form, no schema).
pub json_object: Option<bool>,
/// Disable any additional whitespace in guided JSON output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_any_whitespace: bool,
/// Disable `additionalProperties` in JSON schema output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_additional_properties: bool,
/// Custom whitespace pattern for guided JSON output.
pub whitespace_pattern: Option<String>,
/// Structural tag configuration (JSON-encoded string).
pub structural_tag: Option<String>,
/// Structured-output backend, mirroring Python's internal `_backend`.
///
/// User-supplied values are ignored during deserialization. This matches
/// Python's request boundary, where `_backend` is set by validation rather
/// than accepted as a request-level backend selector.
#[serde(
default,
rename = "_backend",
deserialize_with = "serde_with::rust::deserialize_ignore_any"
)]
pub backend: StructuredOutputBackend,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn structured_outputs_backend_ignores_deserialized_value() {
let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
"json_object": true,
"_backend": "xgrammar",
}))
.unwrap();
assert_eq!(params.backend, StructuredOutputBackend::Guidance);
let value = serde_json::to_value(params).unwrap();
assert_eq!(value["_backend"], "guidance");
}
}
@@ -22,13 +22,14 @@ use crate::protocol::multimodal::{
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
SliceSpec,
};
use crate::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, decode_engine_core_outputs,
};
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::tensor::WireTensor;
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use crate::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
EngineCoreRequestType, EngineCoreSamplingParams, decode_engine_core_outputs,
};
use crate::test_utils::{
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
setup_mock_engine_with_init, spawn_mock_engine_task,
+2 -3
View File
@@ -18,9 +18,8 @@ use crate::error::{Error, Result, bail_unexpected_handshake_message};
use crate::protocol::handshake::{
EngineCoreReadyResponse, HandshakeAddresses, HandshakeInitMessage, ReadyMessage,
};
use crate::protocol::{
EngineCoreOutputs, decode_engine_core_outputs, decode_msgpack, encode_msgpack,
};
use crate::protocol::output::{EngineCoreOutputs, decode_engine_core_outputs};
use crate::protocol::{decode_msgpack, encode_msgpack};
/// Dedicated single-frame sentinel emitted by Python `EngineCoreProc` when the
/// engine dies.