[Rust Frontend] Use enum-backed domain types for engine outputs and structured outputs (#47283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -231,32 +231,18 @@ fn convert_structured_output(
|
||||
StructuredOutput::Json(schema) => {
|
||||
let json: serde_json::Value = serde_json::from_str(schema)
|
||||
.map_err(|e| Status::invalid_argument(format!("invalid json schema: {e}")))?;
|
||||
StructuredOutputsParams {
|
||||
json: Some(json),
|
||||
..Default::default()
|
||||
}
|
||||
StructuredOutputsParams::json(json)
|
||||
}
|
||||
StructuredOutput::Regex(regex) => StructuredOutputsParams {
|
||||
regex: Some(regex.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::Choice(choices) => StructuredOutputsParams {
|
||||
choice: Some(choices.choices.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::Grammar(grammar) => StructuredOutputsParams {
|
||||
grammar: Some(grammar.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::JsonObject(true) => StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::Regex(regex) => StructuredOutputsParams::regex(regex.clone()),
|
||||
StructuredOutput::Choice(choices) => {
|
||||
StructuredOutputsParams::choice(choices.choices.clone())
|
||||
}
|
||||
StructuredOutput::Grammar(grammar) => StructuredOutputsParams::grammar(grammar.clone()),
|
||||
StructuredOutput::JsonObject(true) => StructuredOutputsParams::json_object(),
|
||||
StructuredOutput::JsonObject(false) => return Ok(None),
|
||||
StructuredOutput::StructuralTag(tag) => StructuredOutputsParams {
|
||||
structural_tag: Some(tag.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
StructuredOutput::StructuralTag(tag) => {
|
||||
StructuredOutputsParams::structural_tag(tag.clone())
|
||||
}
|
||||
};
|
||||
Ok(Some(params))
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ use vllm_chat::{
|
||||
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
@@ -115,19 +115,14 @@ fn engine_outputs_for_request(
|
||||
request_id: &str,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: output_specs
|
||||
.into_iter()
|
||||
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
|
||||
.collect(),
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
|
||||
|
||||
@@ -19,7 +19,7 @@ use vllm_chat::{
|
||||
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
@@ -111,19 +111,14 @@ fn engine_outputs_for_request(
|
||||
request_id: &str,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: output_specs
|
||||
.into_iter()
|
||||
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
|
||||
.collect(),
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
|
||||
|
||||
@@ -37,7 +37,7 @@ pub enum ResponseFormat {
|
||||
},
|
||||
/// vLLM-specific structural tag format. The entire object (including the
|
||||
/// `type` field) is JSON-serialized and passed as
|
||||
/// `StructuredOutputsParams.structural_tag`.
|
||||
/// `StructuredOutputConstraint::StructuralTag`.
|
||||
///
|
||||
/// We capture the payload as a catch-all map so both the legacy
|
||||
/// (`structures`/`triggers`) and current (`format`) shapes are
|
||||
@@ -79,14 +79,10 @@ pub fn convert_from_response_format(
|
||||
};
|
||||
match fmt {
|
||||
ResponseFormat::Text => Ok(None),
|
||||
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
..Default::default()
|
||||
})),
|
||||
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams {
|
||||
json: Some(json_schema.schema.clone()),
|
||||
..Default::default()
|
||||
})),
|
||||
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams::json_object())),
|
||||
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams::json(
|
||||
json_schema.schema.clone(),
|
||||
))),
|
||||
ResponseFormat::StructuralTag { .. } => {
|
||||
// The Python frontend dumps the entire response_format object (including the
|
||||
// `type` field) as a JSON string for the engine-core backend.
|
||||
@@ -96,10 +92,7 @@ pub fn convert_from_response_format(
|
||||
Some("response_format"),
|
||||
)
|
||||
})?;
|
||||
Ok(Some(StructuredOutputsParams {
|
||||
structural_tag: Some(tag_json),
|
||||
..Default::default()
|
||||
}))
|
||||
Ok(Some(StructuredOutputsParams::structural_tag(tag_json)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,8 @@ use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, StopReason,
|
||||
UtilityCallOutput,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
@@ -232,19 +233,14 @@ fn engine_outputs_for_request(
|
||||
request_id: &str,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: output_specs
|
||||
.into_iter()
|
||||
.map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
|
||||
.collect(),
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn test_llm(client: EngineCoreClient) -> Llm {
|
||||
@@ -390,14 +386,15 @@ fn utility_none_result() -> UtilityResultEnvelope {
|
||||
}
|
||||
|
||||
fn utility_outputs(call_id: u64, result: UtilityResultEnvelope) -> EngineCoreOutputs {
|
||||
EngineCoreOutputs {
|
||||
utility_output: Some(UtilityOutput {
|
||||
UtilityCallOutput {
|
||||
output: UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: None,
|
||||
result: Some(result),
|
||||
}),
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
|
||||
@@ -2281,8 +2278,7 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
bytes_to_token_ids(b"hi"),
|
||||
@@ -2291,13 +2287,9 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
Some(sample_logprobs_for_tokens(&bytes_to_token_ids(b"hi"))),
|
||||
Some(prompt_logprobs_for_tokens(&prompt_token_ids)),
|
||||
)],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -3121,8 +3113,7 @@ async fn non_stream_completions_include_logprobs() {
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
@@ -3141,13 +3132,10 @@ async fn non_stream_completions_include_logprobs() {
|
||||
None,
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -3224,8 +3212,7 @@ async fn non_stream_completions_include_prompt_logprobs() {
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![b'h' as u32, b'i' as u32, b'!' as u32],
|
||||
@@ -3246,13 +3233,9 @@ async fn non_stream_completions_include_prompt_logprobs() {
|
||||
}),
|
||||
Some(prompt_logprobs_for_hello()),
|
||||
)],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -3588,8 +3571,7 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
@@ -3609,13 +3591,9 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
|
||||
Some(json!({"connector": "x"})),
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -3708,8 +3686,7 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
@@ -3728,13 +3705,9 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
|
||||
None,
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -4426,8 +4399,7 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
@@ -4446,13 +4418,9 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
|
||||
None,
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
@@ -4601,8 +4569,7 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
bytes_to_token_ids(b"<think>Need tool.</think>"),
|
||||
@@ -4613,19 +4580,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
|
||||
))),
|
||||
None,
|
||||
)],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
|
||||
@@ -4636,19 +4598,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
|
||||
))),
|
||||
None,
|
||||
)],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
bytes_to_token_ids(
|
||||
@@ -4661,13 +4618,10 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
|
||||
))),
|
||||
None,
|
||||
)],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user