[Rust Frontend] Use enum-backed domain types for engine outputs and structured outputs (#47283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-02 10:41:46 +01:00
committed by GitHub
parent 08a8a4af3f
commit 25fcb65d51
16 changed files with 762 additions and 605 deletions
+10 -24
View File
@@ -231,32 +231,18 @@ fn convert_structured_output(
StructuredOutput::Json(schema) => {
let json: serde_json::Value = serde_json::from_str(schema)
.map_err(|e| Status::invalid_argument(format!("invalid json schema: {e}")))?;
StructuredOutputsParams {
json: Some(json),
..Default::default()
}
StructuredOutputsParams::json(json)
}
StructuredOutput::Regex(regex) => StructuredOutputsParams {
regex: Some(regex.clone()),
..Default::default()
},
StructuredOutput::Choice(choices) => StructuredOutputsParams {
choice: Some(choices.choices.clone()),
..Default::default()
},
StructuredOutput::Grammar(grammar) => StructuredOutputsParams {
grammar: Some(grammar.clone()),
..Default::default()
},
StructuredOutput::JsonObject(true) => StructuredOutputsParams {
json_object: Some(true),
..Default::default()
},
StructuredOutput::Regex(regex) => StructuredOutputsParams::regex(regex.clone()),
StructuredOutput::Choice(choices) => {
StructuredOutputsParams::choice(choices.choices.clone())
}
StructuredOutput::Grammar(grammar) => StructuredOutputsParams::grammar(grammar.clone()),
StructuredOutput::JsonObject(true) => StructuredOutputsParams::json_object(),
StructuredOutput::JsonObject(false) => return Ok(None),
StructuredOutput::StructuralTag(tag) => StructuredOutputsParams {
structural_tag: Some(tag.clone()),
..Default::default()
},
StructuredOutput::StructuralTag(tag) => {
StructuredOutputsParams::structural_tag(tag.clone())
}
};
Ok(Some(params))
}
+4 -9
View File
@@ -19,7 +19,7 @@ use vllm_chat::{
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
};
use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
@@ -115,19 +115,14 @@ fn engine_outputs_for_request(
request_id: &str,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> EngineCoreOutputs {
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: output_specs
.into_iter()
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
.collect(),
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
..Default::default()
}
.into()
}
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
@@ -19,7 +19,7 @@ use vllm_chat::{
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
};
use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
@@ -111,19 +111,14 @@ fn engine_outputs_for_request(
request_id: &str,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> EngineCoreOutputs {
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: output_specs
.into_iter()
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
.collect(),
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
..Default::default()
}
.into()
}
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
@@ -37,7 +37,7 @@ pub enum ResponseFormat {
},
/// vLLM-specific structural tag format. The entire object (including the
/// `type` field) is JSON-serialized and passed as
/// `StructuredOutputsParams.structural_tag`.
/// `StructuredOutputConstraint::StructuralTag`.
///
/// We capture the payload as a catch-all map so both the legacy
/// (`structures`/`triggers`) and current (`format`) shapes are
@@ -79,14 +79,10 @@ pub fn convert_from_response_format(
};
match fmt {
ResponseFormat::Text => Ok(None),
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams {
json_object: Some(true),
..Default::default()
})),
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams {
json: Some(json_schema.schema.clone()),
..Default::default()
})),
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams::json_object())),
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams::json(
json_schema.schema.clone(),
))),
ResponseFormat::StructuralTag { .. } => {
// The Python frontend dumps the entire response_format object (including the
// `type` field) as a JSON string for the engine-core backend.
@@ -96,10 +92,7 @@ pub fn convert_from_response_format(
Some("response_format"),
)
})?;
Ok(Some(StructuredOutputsParams {
structural_tag: Some(tag_json),
..Default::default()
}))
Ok(Some(StructuredOutputsParams::structural_tag(tag_json)))
}
}
}
+45 -91
View File
@@ -29,7 +29,8 @@ use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, StopReason,
UtilityCallOutput,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
@@ -232,19 +233,14 @@ fn engine_outputs_for_request(
request_id: &str,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
) -> EngineCoreOutputs {
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: output_specs
.into_iter()
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
.collect(),
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
..Default::default()
}
.into()
}
fn test_llm(client: EngineCoreClient) -> Llm {
@@ -390,14 +386,15 @@ fn utility_none_result() -> UtilityResultEnvelope {
}
fn utility_outputs(call_id: u64, result: UtilityResultEnvelope) -> EngineCoreOutputs {
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
UtilityCallOutput {
output: UtilityOutput {
call_id: call_id.into(),
failure_message: None,
result: Some(result),
}),
},
..Default::default()
}
.into()
}
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
@@ -2281,8 +2278,7 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
bytes_to_token_ids(b"hi"),
@@ -2291,13 +2287,9 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
Some(sample_logprobs_for_tokens(&bytes_to_token_ids(b"hi"))),
Some(prompt_logprobs_for_tokens(&prompt_token_ids)),
)],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -3121,8 +3113,7 @@ async fn non_stream_completions_include_logprobs() {
rmp_serde::from_slice(&add[1]).expect("decode request");
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![
request_output_with_logprobs(
&request.request_id,
@@ -3141,13 +3132,10 @@ async fn non_stream_completions_include_logprobs() {
None,
),
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -3224,8 +3212,7 @@ async fn non_stream_completions_include_prompt_logprobs() {
rmp_serde::from_slice(&add[1]).expect("decode request");
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
vec![b'h' as u32, b'i' as u32, b'!' as u32],
@@ -3246,13 +3233,9 @@ async fn non_stream_completions_include_prompt_logprobs() {
}),
Some(prompt_logprobs_for_hello()),
)],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -3588,8 +3571,7 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![
request_output_with_logprobs(
&request.request_id,
@@ -3609,13 +3591,9 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
Some(json!({"connector": "x"})),
),
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -3708,8 +3686,7 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![
request_output_with_logprobs(
&request.request_id,
@@ -3728,13 +3705,9 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
None,
),
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -4426,8 +4399,7 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![
request_output_with_logprobs(
&request.request_id,
@@ -4446,13 +4418,9 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
None,
),
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})
@@ -4601,8 +4569,7 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
bytes_to_token_ids(b"<think>Need tool.</think>"),
@@ -4613,19 +4580,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
))),
None,
)],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
@@ -4636,19 +4598,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
))),
None,
)],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
bytes_to_token_ids(
@@ -4661,13 +4618,10 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
))),
None,
)],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
wave_complete: None,
start_wave: None,
},
..Default::default()
}
.into(),
)
.await;
})