[Rust Frontend] Split engine core DTOs into separate modules (#47265)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -5,9 +5,9 @@ use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::{
|
||||
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
|
||||
@@ -11,10 +11,11 @@ use tracing::{debug, info, trace};
|
||||
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::dtype::ModelDtype;
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
|
||||
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
|
||||
use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
|
||||
@@ -16,12 +16,11 @@ use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs};
|
||||
use crate::protocol::request::EngineCoreRequestType;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::protocol::{
|
||||
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
|
||||
encode_msgpack,
|
||||
};
|
||||
use crate::transport::{ConnectedEngine, EngineId};
|
||||
use crate::{Error, Result, transport};
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@ use tracing::trace;
|
||||
use crate::EngineId;
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::protocol::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
|
||||
@@ -452,7 +452,7 @@ mod tests {
|
||||
EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
|
||||
};
|
||||
use crate::mock_engine::default_ready_response;
|
||||
use crate::protocol::{
|
||||
use crate::protocol::output::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
@@ -10,7 +10,7 @@ use tracing::{debug, error, warn};
|
||||
|
||||
use crate::client::AbortRequest;
|
||||
use crate::client::state::OutputReceiver;
|
||||
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::{AbortCause, Error, Result};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -10,10 +10,9 @@ use zeromq::{XPubSocket, ZmqMessage};
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::{
|
||||
ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
|
||||
encode_msgpack,
|
||||
};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs};
|
||||
use crate::protocol::request::EngineCoreRequestType;
|
||||
|
||||
/// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
|
||||
/// engine-facing coordinator socket.
|
||||
|
||||
@@ -8,8 +8,9 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::error::{Error, Result, bail_unexpected_handshake_message};
|
||||
use crate::protocol::dtype::ModelDtype;
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
|
||||
use crate::protocol::{decode_msgpack, encode_msgpack};
|
||||
|
||||
/// Default model length advertised by reusable mock engine helpers.
|
||||
pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
|
||||
|
||||
@@ -2,7 +2,8 @@ use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::{ModelDtype, OpaqueValue};
|
||||
use crate::protocol::OpaqueValue;
|
||||
use crate::protocol::dtype::ModelDtype;
|
||||
|
||||
/// Decoded engine startup-handshake payload sent on the handshake socket.
|
||||
///
|
||||
|
||||
@@ -9,8 +9,7 @@ use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use self::wire::*;
|
||||
use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
|
||||
use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
|
||||
use crate::error::{Error, Result, bail_ext_value_decode};
|
||||
use crate::protocol::tensor::{WireArrayData, WireNdArray};
|
||||
|
||||
/// One token candidate and its logprob metadata for a single sequence position.
|
||||
@@ -160,7 +159,7 @@ impl Serialize for MaybeWireLogprobs {
|
||||
impl MaybeWireLogprobs {
|
||||
/// Resolve the wire representation into decoded logprobs by looking up aux
|
||||
/// frames and decoding raw views as needed.
|
||||
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
|
||||
pub(super) fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
@@ -171,37 +170,6 @@ impl MaybeWireLogprobs {
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineCoreOutputs {
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
for output in &mut self.outputs {
|
||||
output.resolve_in_place(frames)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
self.new_logprobs = (self.new_logprobs.take())
|
||||
.map(|value| value.resolve(frames, "new_logprobs"))
|
||||
.transpose()?;
|
||||
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
|
||||
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl WireLogprobs {
|
||||
/// Convert semantic per-position logprobs into the Python wire tuple shape.
|
||||
///
|
||||
@@ -315,16 +283,3 @@ impl WireLogprobs {
|
||||
Ok(Logprobs { positions })
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one ordinary or multipart engine-core output message into the strong
|
||||
/// typed public protocol shape.
|
||||
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
|
||||
|
||||
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
|
||||
outputs.resolve_in_place(frames)?;
|
||||
Ok(outputs)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ use std::collections::BTreeSet;
|
||||
use bytes::Bytes;
|
||||
use rmpv::Value;
|
||||
|
||||
use super::{Logprobs, PositionLogprobs, TokenLogprob, decode_engine_core_outputs};
|
||||
use crate::protocol::EngineCoreFinishReason;
|
||||
use super::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
use crate::protocol::output::{EngineCoreFinishReason, decode_engine_core_outputs};
|
||||
|
||||
fn encode_value(value: &Value) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
|
||||
@@ -1,28 +1,11 @@
|
||||
use std::any::type_name;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::io::Cursor;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rmpv::Value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::MmFeatures;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
|
||||
// TODO: This module currently mixes reusable frontend-facing semantic types
|
||||
// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
|
||||
// cleaned-up frontend sampling types) with engine-core-specific wire DTOs and
|
||||
// handshake/control messages. While the Rust frontend is still evolving
|
||||
// quickly, keep them co-located here for iteration speed. Once the higher-level
|
||||
// API boundary stabilizes, move the truly reusable semantic types into a
|
||||
// lower-level common crate and keep the engine transport/wire messages here.
|
||||
|
||||
/// Dynamic msgpack value used for schema positions that are preserved but not
|
||||
/// yet strongly typed in the early-stage Rust client.
|
||||
@@ -36,499 +19,18 @@ fn is_false(v: &bool) -> bool {
|
||||
!v
|
||||
}
|
||||
|
||||
fn default_top_p() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_repetition_penalty() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_temperature() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
16
|
||||
}
|
||||
|
||||
mod classified_outputs;
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
pub mod logprobs;
|
||||
pub mod lora;
|
||||
pub mod multimodal;
|
||||
pub mod output;
|
||||
pub mod request;
|
||||
pub mod sampling;
|
||||
pub mod stats;
|
||||
pub mod structured_outputs;
|
||||
pub mod tensor;
|
||||
pub mod utility;
|
||||
pub use classified_outputs::{
|
||||
ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
|
||||
};
|
||||
pub use dtype::ModelDtype;
|
||||
pub use logprobs::decode_engine_core_outputs;
|
||||
|
||||
/// Request types are encoded as single-byte protocol constants so they can be
|
||||
/// sent over the ZMQ socket without an extra encoding step.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreRequestType {
|
||||
Add = 0,
|
||||
Abort = 1,
|
||||
StartDpWave = 2,
|
||||
Utility = 3,
|
||||
}
|
||||
|
||||
impl EngineCoreRequestType {
|
||||
/// Decode the single-byte request type frame used on the engine input
|
||||
/// socket. Returns `None` for unrecognized values.
|
||||
pub fn from_frame(frame: &[u8]) -> Option<Self> {
|
||||
let [value] = frame else {
|
||||
return None;
|
||||
};
|
||||
|
||||
match value {
|
||||
0 => Some(Self::Add),
|
||||
1 => Some(Self::Abort),
|
||||
2 => Some(Self::StartDpWave),
|
||||
3 => Some(Self::Utility),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode the request type as the single-byte frame used on the engine
|
||||
/// input socket.
|
||||
pub fn to_frame(self) -> Bytes {
|
||||
Bytes::from_static(match self {
|
||||
Self::Add => b"\x00",
|
||||
Self::Abort => b"\x01",
|
||||
Self::StartDpWave => b"\x02",
|
||||
Self::Utility => b"\x03",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Reason a request finished: stop, length, abort, error, or repetition.
|
||||
///
|
||||
/// This mirrors the Python enum and uses integer encoding for compact wire
|
||||
/// representation.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreFinishReason {
|
||||
/// A stop string was emitted.
|
||||
Stop = 0,
|
||||
/// `max_tokens` or `max_model_len` was reached.
|
||||
Length = 1,
|
||||
/// The request was aborted by the client.
|
||||
Abort = 2,
|
||||
/// A retryable request-level internal error occurred.
|
||||
Error = 3,
|
||||
/// A repetitive token pattern was detected.
|
||||
Repetition = 4,
|
||||
}
|
||||
|
||||
/// Event types emitted by engine-core for one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreEventType {
|
||||
Queued = 1,
|
||||
Scheduled = 2,
|
||||
Preempted = 3,
|
||||
}
|
||||
|
||||
/// A timestamped engine-core event associated with one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EngineCoreEvent {
|
||||
pub r#type: EngineCoreEventType,
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
/// Controls how intermediate outputs are returned to the frontend.
|
||||
///
|
||||
/// `Cumulative = 0` is intentionally not supported in Rust frontend.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L146-L152>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum RequestOutputKind {
|
||||
/// Return only token deltas in each update.
|
||||
#[default]
|
||||
Delta = 1,
|
||||
/// Suppress intermediate updates and return only the final output.
|
||||
FinalOnly = 2,
|
||||
}
|
||||
|
||||
/// Structured-output backend selected for EngineCore grammar compilation.
|
||||
///
|
||||
/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
|
||||
/// validation. The Rust frontend currently always lowers structured-output
|
||||
/// requests to guidance, while ignoring any user-supplied `_backend` value.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum StructuredOutputBackend {
|
||||
Xgrammar,
|
||||
#[default]
|
||||
Guidance,
|
||||
Outlines,
|
||||
LmFormatEnforcer,
|
||||
}
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
///
|
||||
/// Python models this as the union-typed `stop_reason: int | str | None`
|
||||
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
|
||||
///
|
||||
/// Original Python field:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StopReason {
|
||||
TokenId(u32),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
///
|
||||
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
|
||||
/// `json_object`, or `structural_tag`) should be set. The engine-core
|
||||
/// backend selects the appropriate grammar compiler based on which field
|
||||
/// is present.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct StructuredOutputsParams {
|
||||
/// JSON schema (as a dict/object or JSON string) constraining the output.
|
||||
pub json: Option<serde_json::Value>,
|
||||
/// Regular expression the output must match.
|
||||
pub regex: Option<String>,
|
||||
/// List of allowed output strings (the model must produce one of these).
|
||||
pub choice: Option<Vec<String>>,
|
||||
/// Context-free grammar (in EBNF-like notation) the output must conform to.
|
||||
pub grammar: Option<String>,
|
||||
/// When `true`, output must be valid JSON (free-form, no schema).
|
||||
pub json_object: Option<bool>,
|
||||
/// Disable any additional whitespace in guided JSON output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_any_whitespace: bool,
|
||||
/// Disable `additionalProperties` in JSON schema output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_additional_properties: bool,
|
||||
/// Custom whitespace pattern for guided JSON output.
|
||||
pub whitespace_pattern: Option<String>,
|
||||
/// Structural tag configuration (JSON-encoded string).
|
||||
pub structural_tag: Option<String>,
|
||||
/// Structured-output backend, mirroring Python's internal `_backend`.
|
||||
///
|
||||
/// User-supplied values are ignored during deserialization. This matches
|
||||
/// Python's request boundary, where `_backend` is set by validation rather
|
||||
/// than accepted as a request-level backend selector.
|
||||
#[serde(
|
||||
default,
|
||||
rename = "_backend",
|
||||
deserialize_with = "serde_with::rust::deserialize_ignore_any"
|
||||
)]
|
||||
pub backend: StructuredOutputBackend,
|
||||
}
|
||||
|
||||
/// Engine-core-facing sampling parameters for text generation.
|
||||
///
|
||||
/// This is the normalized southbound subset used by the Rust frontend when it
|
||||
/// talks to Python engine-core over the wire. User-facing request semantics
|
||||
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
|
||||
/// intentionally handled by higher layers before values reach this DTO.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
|
||||
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
|
||||
// default-valued keys; default the whole struct. Per-field fns cover the
|
||||
// non-zero defaults.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
|
||||
#[serde(default)]
|
||||
pub struct EngineCoreSamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling.
|
||||
#[serde(default = "default_temperature")]
|
||||
pub temperature: f32,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
#[serde(default = "default_top_p")]
|
||||
pub top_p: f32,
|
||||
/// Maximum number of top tokens to consider. `0` means all tokens.
|
||||
pub top_k: u32,
|
||||
/// Random seed used by the sampler when present.
|
||||
pub seed: Option<i64>,
|
||||
/// Maximum number of tokens to generate per output sequence.
|
||||
#[serde(default = "default_max_tokens")]
|
||||
pub max_tokens: u32,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
pub min_tokens: u32,
|
||||
/// Maximum number of reasoning ("thinking") tokens to emit before the
|
||||
/// reasoning section is force-closed. `None` means unlimited; the
|
||||
/// user-facing `-1` sentinel is normalized to `None` by the frontend before
|
||||
/// reaching this DTO, so only non-negative values are sent. Enforced
|
||||
/// engine-side (and only when a reasoning parser is configured).
|
||||
pub thinking_token_budget: Option<u64>,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
|
||||
pub logprobs: Option<i32>,
|
||||
/// Number of log probabilities to return per prompt token.
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
pub frequency_penalty: f32,
|
||||
/// Presence penalty applied by the sampler.
|
||||
pub presence_penalty: f32,
|
||||
/// Repetition penalty applied by the sampler.
|
||||
#[serde(default = "default_repetition_penalty")]
|
||||
pub repetition_penalty: f32,
|
||||
/// Token IDs that stop generation.
|
||||
pub stop_token_ids: Vec<u32>,
|
||||
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
|
||||
///
|
||||
/// This mirrors Python's internal `_eos_token_id` field and is derived by
|
||||
/// the frontend from tokenizer/model metadata rather than supplied directly
|
||||
/// by end users.
|
||||
#[serde(rename = "_eos_token_id")]
|
||||
pub eos_token_id: Option<u32>,
|
||||
/// Complete stop-token set used by engine-core for `min_tokens` masking.
|
||||
///
|
||||
/// This mirrors Python's internal `_all_stop_token_ids` field and should
|
||||
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
|
||||
/// IDs.
|
||||
#[serde(rename = "_all_stop_token_ids")]
|
||||
pub all_stop_token_ids: BTreeSet<u32>,
|
||||
/// Logit biases to apply during sampling.
|
||||
/// Keys are token IDs
|
||||
pub logit_bias: Option<HashMap<u32, f32>>,
|
||||
/// Restrict output to these token IDs only.
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
/// Tokenized bad words to avoid during generation.
|
||||
#[serde(rename = "_bad_words_token_ids")]
|
||||
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
pub structured_outputs: Option<StructuredOutputsParams>,
|
||||
/// Specific token IDs for which log probabilities should be returned at
|
||||
/// each position.
|
||||
///
|
||||
/// When set, the engine returns logprobs for exactly these tokens in
|
||||
/// addition to the sampled/scored token. Mutually exclusive with the
|
||||
/// `logprobs` count field in practice.
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
/// If `Some(true)`, the request will not attempt to read from the prefix
|
||||
/// cache; newly computed blocks may still populate the cache. `None`
|
||||
/// defers to engine-core defaults.
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions (from `vllm_xargs`).
|
||||
pub extra_args: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
impl EngineCoreSamplingParams {
|
||||
/// Constructs a default sampling params for testing purposes only.
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 65536,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: Vec::new(),
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: BTreeSet::new(),
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extra kwargs consumed by engine-side reasoning parsers.
|
||||
///
|
||||
/// Original Python construction point:
|
||||
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReasoningParserKwargs {
|
||||
/// Effective kwargs visible to the chat template for this request.
|
||||
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Engine-core add-request payload sent from frontend to engine.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreRequest {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
/// Multimodal features attached to the request.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
pub sampling_params: Option<EngineCoreSamplingParams>,
|
||||
/// Pooling parameters are preserved in the schema but not yet strongly
|
||||
/// typed.
|
||||
pub pooling_params: Option<OpaqueValue>,
|
||||
pub arrival_time: f64,
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<lora::LoraRequest>,
|
||||
#[serde(default)]
|
||||
pub cache_salt: Option<String>,
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Unsupported in the first-stage Rust client because Python uses a custom
|
||||
/// tensor/aux-frame encoding path for this field.
|
||||
#[serde(default)]
|
||||
pub prompt_embeds: Option<OpaqueValue>,
|
||||
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
|
||||
/// `prompt_embeds` content parts). `Some(true)` means real token id;
|
||||
/// `Some(false)` means the position uses a pre-computed entry from
|
||||
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
|
||||
#[serde(default)]
|
||||
pub prompt_is_token_ids: Option<Vec<bool>>,
|
||||
/// Index of the client, used to ensure outputs are sent back to the same
|
||||
/// client when scaling out the frontend.
|
||||
#[serde(default)]
|
||||
pub client_index: u32,
|
||||
/// In DP mode, indicates which wave this request is expected to belong to.
|
||||
#[serde(default)]
|
||||
pub current_wave: u32,
|
||||
#[serde(default)]
|
||||
pub priority: i32,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<BTreeMap<String, String>>,
|
||||
#[serde(default)]
|
||||
pub resumable: bool,
|
||||
/// Original user-provided request ID, used for output reporting and aborts.
|
||||
#[serde(default)]
|
||||
pub external_req_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Reasoning-parser kwargs forwarded from the frontend to the
|
||||
/// structured-output backend.
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
/// If `true`, the request should be added to the scheduler's waiting queue
|
||||
/// and immediately aborted, so connector-side cleanup runs via the
|
||||
/// standard `request_finished` hook.
|
||||
#[serde(default)]
|
||||
pub abort_immediately: bool,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
/// Validate fields intentionally not supported in the first-stage client.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.prompt_embeds.is_some() {
|
||||
return Err(Error::UnsupportedField {
|
||||
context: "EngineCoreRequest",
|
||||
field: "prompt_embeds",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Engine-core output for a single request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutput {
|
||||
pub request_id: String,
|
||||
pub new_token_ids: Vec<u32>,
|
||||
/// Decoded sample logprobs for the newly generated positions in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_logprobs: Option<MaybeWireLogprobs>,
|
||||
/// Decoded prompt logprobs for the scored prompt positions emitted in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
|
||||
#[serde(default)]
|
||||
pub pooling_output: Option<OpaqueValue>,
|
||||
#[serde(default)]
|
||||
pub finish_reason: Option<EngineCoreFinishReason>,
|
||||
#[serde(default)]
|
||||
pub stop_reason: Option<StopReason>,
|
||||
#[serde(default)]
|
||||
pub events: Option<Vec<EngineCoreEvent>>,
|
||||
#[serde(default)]
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<OpaqueValue>,
|
||||
/// Breakdown of the scheduled prefill computation, set on the first output
|
||||
/// of a newly scheduled prefill and elided for subsequent decode outputs.
|
||||
#[serde(default)]
|
||||
pub prefill_stats: Option<PrefillStats>,
|
||||
#[serde(default)]
|
||||
pub routed_experts: Option<OpaqueValue>,
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
/// Returns whether this output is terminal for the request.
|
||||
pub fn finished(&self) -> bool {
|
||||
self.finish_reason.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch of engine-core outputs returned to a frontend client.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutputs {
|
||||
#[serde(default)]
|
||||
pub engine_index: u32,
|
||||
/// Outputs grouped for this client in the current engine tick.
|
||||
#[serde(default)]
|
||||
pub outputs: Vec<EngineCoreOutput>,
|
||||
#[serde(default)]
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
#[serde(default)]
|
||||
pub timestamp: f64,
|
||||
#[serde(default)]
|
||||
pub utility_output: Option<UtilityOutput>,
|
||||
#[serde(default)]
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
/// In DP mode, signals that the current wave finished and engines are
|
||||
/// paused.
|
||||
#[serde(default)]
|
||||
pub wave_complete: Option<u32>,
|
||||
/// In DP mode, signals that a request arrived for an old wave and the next
|
||||
/// wave needs to start in other engines.
|
||||
#[serde(default)]
|
||||
pub start_wave: Option<u32>,
|
||||
}
|
||||
|
||||
/// Encode a Rust value into msgpack using the protocol crate's serde model.
|
||||
pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
|
||||
@@ -564,81 +66,17 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a msgpack payload into a dynamic value for diagnostics and tests.
|
||||
pub fn decode_value(bytes: &[u8]) -> Result<Value> {
|
||||
Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_serializes_as_full_array() {
|
||||
let request = EngineCoreRequest {
|
||||
request_id: "req-1".to_string(),
|
||||
prompt_token_ids: Some(vec![1, 2, 3]),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens: 8,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&request).unwrap();
|
||||
let value = decode_value(&encoded).unwrap();
|
||||
let array = match value {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 20);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_roundtrip_finished_fields() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![42],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
stop_reason: Some(StopReason::Text("stop".to_string())),
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&outputs).unwrap();
|
||||
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded.outputs.len(), 1);
|
||||
assert_eq!(
|
||||
decoded.outputs[0].finish_reason,
|
||||
Some(EngineCoreFinishReason::Length)
|
||||
);
|
||||
assert_eq!(
|
||||
decoded.finished_requests,
|
||||
Some(BTreeSet::from(["req-1".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_msgpack_includes_type_name_and_value_fallback() {
|
||||
let error = decode_msgpack::<u64>(
|
||||
@@ -648,72 +86,4 @@ mod tests {
|
||||
|
||||
expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_outputs_backend_ignores_deserialized_value() {
|
||||
let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
|
||||
"json_object": true,
|
||||
"_backend": "xgrammar",
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(params.backend, StructuredOutputBackend::Guidance);
|
||||
|
||||
let value = serde_json::to_value(params).unwrap();
|
||||
assert_eq!(value["_backend"], "guidance");
|
||||
}
|
||||
|
||||
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
|
||||
/// must fall back to defaults. `python_compat` can't catch this since Rust
|
||||
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
|
||||
#[test]
|
||||
fn decodes_sampling_params_with_omitted_defaults() {
|
||||
let sampling_params = Value::Map(vec![
|
||||
(
|
||||
Value::from("stop_token_ids"),
|
||||
Value::Array(vec![Value::from(151643u32)]),
|
||||
),
|
||||
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
|
||||
]);
|
||||
let request = Value::Array(vec![
|
||||
Value::from("req-omit-defaults"),
|
||||
Value::Array(vec![
|
||||
Value::from(1u32),
|
||||
Value::from(2u32),
|
||||
Value::from(3u32),
|
||||
]),
|
||||
Value::Nil,
|
||||
sampling_params,
|
||||
Value::Nil,
|
||||
Value::from(1.0f64),
|
||||
]);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
rmpv::encode::write_value(&mut bytes, &request).unwrap();
|
||||
|
||||
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
|
||||
.expect("a real omit_defaults request must decode (regression: missing field)");
|
||||
|
||||
assert_eq!(decoded.request_id, "req-omit-defaults");
|
||||
let sampling = decoded.sampling_params.expect("sampling params present");
|
||||
|
||||
assert_eq!(sampling.stop_token_ids, vec![151643]);
|
||||
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
|
||||
|
||||
// Omitted fields -> Python defaults.
|
||||
assert_eq!(sampling.temperature, 1.0);
|
||||
assert_eq!(sampling.top_p, 1.0);
|
||||
assert_eq!(sampling.top_k, 0);
|
||||
assert_eq!(sampling.seed, None);
|
||||
assert_eq!(sampling.max_tokens, 16);
|
||||
assert_eq!(sampling.min_tokens, 0);
|
||||
assert_eq!(sampling.min_p, 0.0);
|
||||
assert_eq!(sampling.frequency_penalty, 0.0);
|
||||
assert_eq!(sampling.presence_penalty, 0.0);
|
||||
assert_eq!(sampling.repetition_penalty, 1.0);
|
||||
assert_eq!(sampling.logprobs, None);
|
||||
assert_eq!(sampling.prompt_logprobs, None);
|
||||
assert_eq!(sampling.eos_token_id, None);
|
||||
assert!(sampling.all_stop_token_ids.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
+219
-3
@@ -1,10 +1,164 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use super::utility::UtilityOutput;
|
||||
use super::{EngineCoreOutput, EngineCoreOutputs};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
///
|
||||
/// Python models this as the union-typed `stop_reason: int | str | None`
|
||||
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
|
||||
///
|
||||
/// Original Python field:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StopReason {
|
||||
TokenId(u32),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
/// Reason a request finished: stop, length, abort, error, or repetition.
|
||||
///
|
||||
/// This mirrors the Python enum and uses integer encoding for compact wire
|
||||
/// representation.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreFinishReason {
|
||||
/// A stop string was emitted.
|
||||
Stop = 0,
|
||||
/// `max_tokens` or `max_model_len` was reached.
|
||||
Length = 1,
|
||||
/// The request was aborted by the client.
|
||||
Abort = 2,
|
||||
/// A retryable request-level internal error occurred.
|
||||
Error = 3,
|
||||
/// A repetitive token pattern was detected.
|
||||
Repetition = 4,
|
||||
}
|
||||
|
||||
/// Event types emitted by engine-core for one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreEventType {
|
||||
Queued = 1,
|
||||
Scheduled = 2,
|
||||
Preempted = 3,
|
||||
}
|
||||
|
||||
/// A timestamped engine-core event associated with one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EngineCoreEvent {
|
||||
pub r#type: EngineCoreEventType,
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
/// Engine-core output for a single request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutput {
|
||||
pub request_id: String,
|
||||
pub new_token_ids: Vec<u32>,
|
||||
/// Decoded sample logprobs for the newly generated positions in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_logprobs: Option<MaybeWireLogprobs>,
|
||||
/// Decoded prompt logprobs for the scored prompt positions emitted in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
|
||||
#[serde(default)]
|
||||
pub pooling_output: Option<OpaqueValue>,
|
||||
#[serde(default)]
|
||||
pub finish_reason: Option<EngineCoreFinishReason>,
|
||||
#[serde(default)]
|
||||
pub stop_reason: Option<StopReason>,
|
||||
#[serde(default)]
|
||||
pub events: Option<Vec<EngineCoreEvent>>,
|
||||
#[serde(default)]
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<OpaqueValue>,
|
||||
/// Breakdown of the scheduled prefill computation, set on the first output
|
||||
/// of a newly scheduled prefill and elided for subsequent decode outputs.
|
||||
#[serde(default)]
|
||||
pub prefill_stats: Option<PrefillStats>,
|
||||
#[serde(default)]
|
||||
pub routed_experts: Option<OpaqueValue>,
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
/// Returns whether this output is terminal for the request.
|
||||
pub fn finished(&self) -> bool {
|
||||
self.finish_reason.is_some()
|
||||
}
|
||||
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
self.new_logprobs = (self.new_logprobs.take())
|
||||
.map(|value| value.resolve(frames, "new_logprobs"))
|
||||
.transpose()?;
|
||||
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
|
||||
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch of engine-core outputs returned to a frontend client.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutputs {
|
||||
#[serde(default)]
|
||||
pub engine_index: u32,
|
||||
/// Outputs grouped for this client in the current engine tick.
|
||||
#[serde(default)]
|
||||
pub outputs: Vec<EngineCoreOutput>,
|
||||
#[serde(default)]
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
#[serde(default)]
|
||||
pub timestamp: f64,
|
||||
#[serde(default)]
|
||||
pub utility_output: Option<UtilityOutput>,
|
||||
#[serde(default)]
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
/// In DP mode, signals that the current wave finished and engines are
|
||||
/// paused.
|
||||
#[serde(default)]
|
||||
pub wave_complete: Option<u32>,
|
||||
/// In DP mode, signals that a request arrived for an old wave and the next
|
||||
/// wave needs to start in other engines.
|
||||
#[serde(default)]
|
||||
pub start_wave: Option<u32>,
|
||||
}
|
||||
|
||||
/// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -49,6 +203,18 @@ pub enum ClassifiedEngineCoreOutputs {
|
||||
}
|
||||
|
||||
impl EngineCoreOutputs {
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
for output in &mut self.outputs {
|
||||
output.resolve_in_place(frames)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Classify the raw wire message into a more semantic Rust enum.
|
||||
pub fn classify(self) -> ClassifiedEngineCoreOutputs {
|
||||
let has_request_payload = !self.outputs.is_empty()
|
||||
@@ -92,12 +258,62 @@ impl EngineCoreOutputs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one ordinary or multipart engine-core output message into the strong
|
||||
/// typed public protocol shape.
|
||||
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
|
||||
|
||||
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
|
||||
outputs.resolve_in_place(frames)?;
|
||||
Ok(outputs)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::EngineCoreOutput;
|
||||
use crate::protocol::output::EngineCoreOutput;
|
||||
use crate::protocol::{decode_msgpack, encode_msgpack};
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_roundtrip_finished_fields() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![42],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
stop_reason: Some(StopReason::Text("stop".to_string())),
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&outputs).unwrap();
|
||||
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded.outputs.len(), 1);
|
||||
assert_eq!(
|
||||
decoded.outputs[0].finish_reason,
|
||||
Some(EngineCoreFinishReason::Length)
|
||||
);
|
||||
assert_eq!(
|
||||
decoded.finished_requests,
|
||||
Some(BTreeSet::from(["req-1".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_request_batch() {
|
||||
@@ -0,0 +1,175 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::protocol::multimodal::MmFeatures;
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::{OpaqueValue, lora};
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// Request types are encoded as single-byte protocol constants so they can be
|
||||
/// sent over the ZMQ socket without an extra encoding step.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreRequestType {
|
||||
Add = 0,
|
||||
Abort = 1,
|
||||
StartDpWave = 2,
|
||||
Utility = 3,
|
||||
}
|
||||
|
||||
impl EngineCoreRequestType {
|
||||
/// Decode the single-byte request type frame used on the engine input
|
||||
/// socket. Returns `None` for unrecognized values.
|
||||
pub fn from_frame(frame: &[u8]) -> Option<Self> {
|
||||
let [value] = frame else {
|
||||
return None;
|
||||
};
|
||||
|
||||
match value {
|
||||
0 => Some(Self::Add),
|
||||
1 => Some(Self::Abort),
|
||||
2 => Some(Self::StartDpWave),
|
||||
3 => Some(Self::Utility),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode the request type as the single-byte frame used on the engine
|
||||
/// input socket.
|
||||
pub fn to_frame(self) -> Bytes {
|
||||
Bytes::from_static(match self {
|
||||
Self::Add => b"\x00",
|
||||
Self::Abort => b"\x01",
|
||||
Self::StartDpWave => b"\x02",
|
||||
Self::Utility => b"\x03",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Extra kwargs consumed by engine-side reasoning parsers.
|
||||
///
|
||||
/// Original Python construction point:
|
||||
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReasoningParserKwargs {
|
||||
/// Effective kwargs visible to the chat template for this request.
|
||||
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Engine-core add-request payload sent from frontend to engine.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreRequest {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
/// Multimodal features attached to the request.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
pub sampling_params: Option<EngineCoreSamplingParams>,
|
||||
/// Pooling parameters are preserved in the schema but not yet strongly
|
||||
/// typed.
|
||||
pub pooling_params: Option<OpaqueValue>,
|
||||
pub arrival_time: f64,
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<lora::LoraRequest>,
|
||||
#[serde(default)]
|
||||
pub cache_salt: Option<String>,
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Unsupported in the first-stage Rust client because Python uses a custom
|
||||
/// tensor/aux-frame encoding path for this field.
|
||||
#[serde(default)]
|
||||
pub prompt_embeds: Option<OpaqueValue>,
|
||||
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
|
||||
/// `prompt_embeds` content parts). `Some(true)` means real token id;
|
||||
/// `Some(false)` means the position uses a pre-computed entry from
|
||||
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
|
||||
#[serde(default)]
|
||||
pub prompt_is_token_ids: Option<Vec<bool>>,
|
||||
/// Index of the client, used to ensure outputs are sent back to the same
|
||||
/// client when scaling out the frontend.
|
||||
#[serde(default)]
|
||||
pub client_index: u32,
|
||||
/// In DP mode, indicates which wave this request is expected to belong to.
|
||||
#[serde(default)]
|
||||
pub current_wave: u32,
|
||||
#[serde(default)]
|
||||
pub priority: i32,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<BTreeMap<String, String>>,
|
||||
#[serde(default)]
|
||||
pub resumable: bool,
|
||||
/// Original user-provided request ID, used for output reporting and aborts.
|
||||
#[serde(default)]
|
||||
pub external_req_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Reasoning-parser kwargs forwarded from the frontend to the
|
||||
/// structured-output backend.
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
/// If `true`, the request should be added to the scheduler's waiting queue
|
||||
/// and immediately aborted, so connector-side cleanup runs via the
|
||||
/// standard `request_finished` hook.
|
||||
#[serde(default)]
|
||||
pub abort_immediately: bool,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
/// Validate fields intentionally not supported in the first-stage client.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.prompt_embeds.is_some() {
|
||||
return Err(Error::UnsupportedField {
|
||||
context: "EngineCoreRequest",
|
||||
field: "prompt_embeds",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rmpv::Value;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::{decode_value, encode_msgpack};
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_serializes_as_full_array() {
|
||||
let request = EngineCoreRequest {
|
||||
request_id: "req-1".to_string(),
|
||||
prompt_token_ids: Some(vec![1, 2, 3]),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens: 8,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&request).unwrap();
|
||||
let value = decode_value(&encoded).unwrap();
|
||||
let array = match value {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 20);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
|
||||
use crate::protocol::structured_outputs::StructuredOutputsParams;
|
||||
|
||||
fn default_top_p() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_repetition_penalty() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_temperature() -> f32 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
16
|
||||
}
|
||||
|
||||
/// Engine-core-facing sampling parameters for text generation.
|
||||
///
|
||||
/// This is the normalized southbound subset used by the Rust frontend when it
|
||||
/// talks to Python engine-core over the wire. User-facing request semantics
|
||||
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
|
||||
/// intentionally handled by higher layers before values reach this DTO.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
|
||||
// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
|
||||
// default-valued keys; default the whole struct. Per-field fns cover the
|
||||
// non-zero defaults.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
|
||||
#[serde(default)]
|
||||
pub struct EngineCoreSamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling.
|
||||
#[serde(default = "default_temperature")]
|
||||
pub temperature: f32,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
#[serde(default = "default_top_p")]
|
||||
pub top_p: f32,
|
||||
/// Maximum number of top tokens to consider. `0` means all tokens.
|
||||
pub top_k: u32,
|
||||
/// Random seed used by the sampler when present.
|
||||
pub seed: Option<i64>,
|
||||
/// Maximum number of tokens to generate per output sequence.
|
||||
#[serde(default = "default_max_tokens")]
|
||||
pub max_tokens: u32,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
pub min_tokens: u32,
|
||||
/// Maximum number of reasoning ("thinking") tokens to emit before the
|
||||
/// reasoning section is force-closed. `None` means unlimited; the
|
||||
/// user-facing `-1` sentinel is normalized to `None` by the frontend before
|
||||
/// reaching this DTO, so only non-negative values are sent. Enforced
|
||||
/// engine-side (and only when a reasoning parser is configured).
|
||||
pub thinking_token_budget: Option<u64>,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
|
||||
pub logprobs: Option<i32>,
|
||||
/// Number of log probabilities to return per prompt token.
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
pub frequency_penalty: f32,
|
||||
/// Presence penalty applied by the sampler.
|
||||
pub presence_penalty: f32,
|
||||
/// Repetition penalty applied by the sampler.
|
||||
#[serde(default = "default_repetition_penalty")]
|
||||
pub repetition_penalty: f32,
|
||||
/// Token IDs that stop generation.
|
||||
pub stop_token_ids: Vec<u32>,
|
||||
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
|
||||
///
|
||||
/// This mirrors Python's internal `_eos_token_id` field and is derived by
|
||||
/// the frontend from tokenizer/model metadata rather than supplied directly
|
||||
/// by end users.
|
||||
#[serde(rename = "_eos_token_id")]
|
||||
pub eos_token_id: Option<u32>,
|
||||
/// Complete stop-token set used by engine-core for `min_tokens` masking.
|
||||
///
|
||||
/// This mirrors Python's internal `_all_stop_token_ids` field and should
|
||||
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
|
||||
/// IDs.
|
||||
#[serde(rename = "_all_stop_token_ids")]
|
||||
pub all_stop_token_ids: BTreeSet<u32>,
|
||||
/// Logit biases to apply during sampling.
|
||||
/// Keys are token IDs
|
||||
pub logit_bias: Option<HashMap<u32, f32>>,
|
||||
/// Restrict output to these token IDs only.
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
/// Tokenized bad words to avoid during generation.
|
||||
#[serde(rename = "_bad_words_token_ids")]
|
||||
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
pub structured_outputs: Option<StructuredOutputsParams>,
|
||||
/// Specific token IDs for which log probabilities should be returned at
|
||||
/// each position.
|
||||
///
|
||||
/// When set, the engine returns logprobs for exactly these tokens in
|
||||
/// addition to the sampled/scored token. Mutually exclusive with the
|
||||
/// `logprobs` count field in practice.
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
/// If `Some(true)`, the request will not attempt to read from the prefix
|
||||
/// cache; newly computed blocks may still populate the cache. `None`
|
||||
/// defers to engine-core defaults.
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions (from `vllm_xargs`).
|
||||
pub extra_args: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
impl EngineCoreSamplingParams {
|
||||
/// Constructs a default sampling params for testing purposes only.
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 65536,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: Vec::new(),
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: BTreeSet::new(),
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rmpv::Value;
|
||||
|
||||
use crate::protocol::decode_msgpack;
|
||||
use crate::protocol::request::EngineCoreRequest;
|
||||
|
||||
/// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
|
||||
/// must fall back to defaults. `python_compat` can't catch this since Rust
|
||||
/// encodes full maps (see `engine_core_request_serializes_as_full_array`).
|
||||
#[test]
|
||||
fn decodes_sampling_params_with_omitted_defaults() {
|
||||
let sampling_params = Value::Map(vec![
|
||||
(
|
||||
Value::from("stop_token_ids"),
|
||||
Value::Array(vec![Value::from(151643u32)]),
|
||||
),
|
||||
(Value::from("skip_reading_prefix_cache"), Value::from(false)),
|
||||
]);
|
||||
let request = Value::Array(vec![
|
||||
Value::from("req-omit-defaults"),
|
||||
Value::Array(vec![
|
||||
Value::from(1u32),
|
||||
Value::from(2u32),
|
||||
Value::from(3u32),
|
||||
]),
|
||||
Value::Nil,
|
||||
sampling_params,
|
||||
Value::Nil,
|
||||
Value::from(1.0f64),
|
||||
]);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
rmpv::encode::write_value(&mut bytes, &request).unwrap();
|
||||
|
||||
let decoded: EngineCoreRequest = decode_msgpack(&bytes)
|
||||
.expect("a real omit_defaults request must decode (regression: missing field)");
|
||||
|
||||
assert_eq!(decoded.request_id, "req-omit-defaults");
|
||||
let sampling = decoded.sampling_params.expect("sampling params present");
|
||||
|
||||
assert_eq!(sampling.stop_token_ids, vec![151643]);
|
||||
assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
|
||||
|
||||
// Omitted fields -> Python defaults.
|
||||
assert_eq!(sampling.temperature, 1.0);
|
||||
assert_eq!(sampling.top_p, 1.0);
|
||||
assert_eq!(sampling.top_k, 0);
|
||||
assert_eq!(sampling.seed, None);
|
||||
assert_eq!(sampling.max_tokens, 16);
|
||||
assert_eq!(sampling.min_tokens, 0);
|
||||
assert_eq!(sampling.min_p, 0.0);
|
||||
assert_eq!(sampling.frequency_penalty, 0.0);
|
||||
assert_eq!(sampling.presence_penalty, 0.0);
|
||||
assert_eq!(sampling.repetition_penalty, 1.0);
|
||||
assert_eq!(sampling.logprobs, None);
|
||||
assert_eq!(sampling.prompt_logprobs, None);
|
||||
assert_eq!(sampling.eos_token_id, None);
|
||||
assert!(sampling.all_stop_token_ids.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Structured-output backend selected for EngineCore grammar compilation.
|
||||
///
|
||||
/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
|
||||
/// validation. The Rust frontend currently always lowers structured-output
|
||||
/// requests to guidance, while ignoring any user-supplied `_backend` value.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum StructuredOutputBackend {
|
||||
Xgrammar,
|
||||
#[default]
|
||||
Guidance,
|
||||
Outlines,
|
||||
LmFormatEnforcer,
|
||||
}
|
||||
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
///
|
||||
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
|
||||
/// `json_object`, or `structural_tag`) should be set. The engine-core
|
||||
/// backend selects the appropriate grammar compiler based on which field
|
||||
/// is present.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct StructuredOutputsParams {
|
||||
/// JSON schema (as a dict/object or JSON string) constraining the output.
|
||||
pub json: Option<serde_json::Value>,
|
||||
/// Regular expression the output must match.
|
||||
pub regex: Option<String>,
|
||||
/// List of allowed output strings (the model must produce one of these).
|
||||
pub choice: Option<Vec<String>>,
|
||||
/// Context-free grammar (in EBNF-like notation) the output must conform to.
|
||||
pub grammar: Option<String>,
|
||||
/// When `true`, output must be valid JSON (free-form, no schema).
|
||||
pub json_object: Option<bool>,
|
||||
/// Disable any additional whitespace in guided JSON output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_any_whitespace: bool,
|
||||
/// Disable `additionalProperties` in JSON schema output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_additional_properties: bool,
|
||||
/// Custom whitespace pattern for guided JSON output.
|
||||
pub whitespace_pattern: Option<String>,
|
||||
/// Structural tag configuration (JSON-encoded string).
|
||||
pub structural_tag: Option<String>,
|
||||
/// Structured-output backend, mirroring Python's internal `_backend`.
|
||||
///
|
||||
/// User-supplied values are ignored during deserialization. This matches
|
||||
/// Python's request boundary, where `_backend` is set by validation rather
|
||||
/// than accepted as a request-level backend selector.
|
||||
#[serde(
|
||||
default,
|
||||
rename = "_backend",
|
||||
deserialize_with = "serde_with::rust::deserialize_ignore_any"
|
||||
)]
|
||||
pub backend: StructuredOutputBackend,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn structured_outputs_backend_ignores_deserialized_value() {
|
||||
let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
|
||||
"json_object": true,
|
||||
"_backend": "xgrammar",
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(params.backend, StructuredOutputBackend::Guidance);
|
||||
|
||||
let value = serde_json::to_value(params).unwrap();
|
||||
assert_eq!(value["_backend"], "guidance");
|
||||
}
|
||||
}
|
||||
@@ -22,13 +22,14 @@ use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
|
||||
SliceSpec,
|
||||
};
|
||||
use crate::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, decode_engine_core_outputs,
|
||||
};
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::tensor::WireTensor;
|
||||
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
use crate::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
|
||||
EngineCoreRequestType, EngineCoreSamplingParams, decode_engine_core_outputs,
|
||||
};
|
||||
use crate::test_utils::{
|
||||
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
|
||||
setup_mock_engine_with_init, spawn_mock_engine_task,
|
||||
|
||||
@@ -18,9 +18,8 @@ use crate::error::{Error, Result, bail_unexpected_handshake_message};
|
||||
use crate::protocol::handshake::{
|
||||
EngineCoreReadyResponse, HandshakeAddresses, HandshakeInitMessage, ReadyMessage,
|
||||
};
|
||||
use crate::protocol::{
|
||||
EngineCoreOutputs, decode_engine_core_outputs, decode_msgpack, encode_msgpack,
|
||||
};
|
||||
use crate::protocol::output::{EngineCoreOutputs, decode_engine_core_outputs};
|
||||
use crate::protocol::{decode_msgpack, encode_msgpack};
|
||||
|
||||
/// Dedicated single-frame sentinel emitted by Python `EngineCoreProc` when the
|
||||
/// engine dies.
|
||||
|
||||
Reference in New Issue
Block a user