[Rust Frontend] Introduce parser-owned output grammar interfaces (#55269)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-22 05:35:07 +00:00
committed by GitHub
co-authored by Claude Fable 5.1
parent f84325c48c
commit d50723df04
34 changed files with 814 additions and 698 deletions
+10 -2
View File
@@ -60,6 +60,11 @@ pub enum Error {
#[source]
error: BoxedError,
},
#[error("failed to initialize request output parser")]
OutputParserInitialization {
#[source]
error: BoxedError,
},
#[error(
"gpt_oss uses native Harmony output parsing; generic {kind} parser override `{selection}` is not supported"
)]
@@ -87,8 +92,11 @@ pub enum Error {
ToolChoiceRequiresTools,
#[error("tool_choice function `{name}` was not found in the available tools")]
ToolChoiceFunctionNotFound { name: String },
#[error("failed to build structural tag: {message}")]
StructuralTag { message: String },
#[error("failed to build output grammar")]
OutputGrammar {
#[source]
error: BoxedError,
},
#[error(transparent)]
Text(#[from] vllm_text::Error),
#[error(transparent)]
+29 -14
View File
@@ -22,6 +22,7 @@ pub use event::{
};
use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart;
use output::apply_output_grammar;
pub use output::{
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
HarmonyChatOutputProcessor,
@@ -65,7 +66,7 @@ use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};
use vllm_text::{Prompt, TextLlm, TextRequest, TextRequestProcessor};
/// Chat request preparation shared by inference and render-only frontends.
pub struct ChatRequestProcessor {
@@ -174,15 +175,8 @@ impl ChatRequestProcessor {
async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
// Stamp before rendering so render and tokenize count toward TTFT/e2e.
let arrival_time = vllm_llm::current_unix_timestamp_secs();
let rendered = self.backend.chat_renderer().render(&request)?;
let reasoning_parser_kwargs =
request
.sampling_params
.structured_outputs
.is_some()
.then(|| ReasoningParserKwargs {
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
});
let mut rendered = self.backend.chat_renderer().render(&request)?;
let chat_template_kwargs = std::mem::take(&mut rendered.effective_template_kwargs);
let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
Ok(TextRequest {
request_id: request.request_id,
@@ -198,7 +192,10 @@ impl ChatRequestProcessor {
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id,
kv_hints: None,
reasoning_parser_kwargs,
reasoning_parser_kwargs: ReasoningParserKwargs {
chat_template_kwargs,
},
reasoning_ended: None,
lora_request: request.lora_request,
arrival_time: Some(arrival_time),
})
@@ -211,12 +208,19 @@ impl ChatRequestProcessor {
}
/// Prepare one chat request without submitting it to an engine.
///
/// The returned text request is already tokenized: the output processor
/// is initialized from the final prompt token IDs and its grammar, if any,
/// is applied to the request before it is handed to `text_processor` again
/// for lowering.
pub async fn prepare(
&self,
mut request: ChatRequest,
text_processor: &TextRequestProcessor,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
let output_processor = self.backend.new_chat_output_processor(
let mut output_processor = self.backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_call_parser: &self.tool_call_parser,
@@ -224,7 +228,17 @@ impl ChatRequestProcessor {
tool_strict_level: self.tool_strict_level,
},
)?;
let text_request = self.prepare_text_request(request).await?;
let mut text_request = self.prepare_text_request(request).await?;
// Initialize the output processor from the final prompt token IDs and apply
// the output grammar to the request, if any.
let prompt_token_ids = text_processor.tokenize_in_place(&mut text_request)?;
let grammar = {
output_processor.initialize(prompt_token_ids)?;
output_processor.build_output_grammar()?
};
apply_output_grammar(&mut text_request, grammar)?;
Ok((text_request, output_processor))
}
}
@@ -342,7 +356,8 @@ impl ChatLlm {
/// Render, tokenize, and submit one chat request.
pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
let (text_request, output_processor) = self.processor.prepare(request).await?;
let (text_request, output_processor) =
self.processor.prepare(request, self.text.request_processor()).await?;
let request_id = text_request.request_id.clone();
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
+125 -10
View File
@@ -3,17 +3,17 @@
//! Default output processing pipeline.
mod structural_tag;
mod unified;
use std::sync::Once;
use futures::StreamExt as _;
use tracing::info;
use vllm_parser::output_grammar::{BuiltOutputGrammar, OutputGrammarContext};
use vllm_parser::unified::{CombinedParser, UnifiedParser};
use vllm_text::tokenizer::DynTokenizer;
use xgrammar_structural_tag::ToolChoice;
use self::structural_tag::apply_structural_tag_constraint;
use self::unified::unified_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
@@ -34,6 +34,18 @@ use crate::{Error, Result as ChatResult};
pub struct DefaultChatOutputProcessor {
parser: Box<dyn UnifiedParser>,
parallel_tool_calls: bool,
/// Request facts the initialized parser needs to build its output grammar.
/// Absent for the plain-text-only processor, which never builds one.
grammar_inputs: Option<GrammarInputs>,
}
/// Request-scoped inputs for [`UnifiedParser::build_output_grammar`], captured
/// at construction because the chat request is consumed before the prompt is
/// tokenized.
struct GrammarInputs {
tools: Vec<ChatTool>,
tool_choice: ToolChoice,
tool_strict_level: ToolStrictLevel,
}
impl DefaultChatOutputProcessor {
@@ -42,7 +54,8 @@ impl DefaultChatOutputProcessor {
///
/// Parser resolution happens here so that request validation, prompt
/// rendering, and streaming all observe the same parser-adjusted
/// request state.
/// request state. The parser is initialized and its output grammar built
/// later, once the final prompt token IDs are known.
pub fn new(
request: &mut ChatRequest,
model_id: &str,
@@ -74,12 +87,6 @@ impl DefaultChatOutputProcessor {
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
};
apply_structural_tag_constraint(
request,
parser.structural_tag_builder(),
tool_strict_level,
)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
@@ -87,6 +94,11 @@ impl DefaultChatOutputProcessor {
Ok(Self {
parser,
parallel_tool_calls: request.parallel_tool_calls(),
grammar_inputs: Some(GrammarInputs {
tools: request.tools().to_vec(),
tool_choice: request.tool_choice().into(),
tool_strict_level,
}),
})
}
@@ -99,6 +111,7 @@ impl DefaultChatOutputProcessor {
Self {
parser: Box::new(CombinedParser::plain_text_only()),
parallel_tool_calls: true,
grammar_inputs: None,
}
}
@@ -175,6 +188,30 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
static UNIFIED_PARSER_LOG_ONCE: Once = Once::new();
impl ChatOutputProcessor for DefaultChatOutputProcessor {
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.parser.initialize(prompt_token_ids).map_err(|error| {
Error::OutputParserInitialization {
error: Box::new(error),
}
})
}
fn build_output_grammar(&self) -> Result<Option<BuiltOutputGrammar>> {
let Some(inputs) = &self.grammar_inputs else {
return Ok(None);
};
self.parser
.build_output_grammar(&OutputGrammarContext {
tools: &inputs.tools,
tool_choice: &inputs.tool_choice,
tool_strict_level: inputs.tool_strict_level,
parallel_tool_calls: self.parallel_tool_calls,
})
.map_err(|error| Error::OutputGrammar {
error: Box::new(error),
})
}
/// Transforms a raw generate-output token stream into structured chat
/// events through two sequential stages once text decoding has
/// already happened:
@@ -196,8 +233,9 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::output::ChatOutputProcessor;
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::ChatRequest;
use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
fn tokenizer() -> Arc<TestTokenizer> {
Arc::new(
@@ -207,6 +245,83 @@ mod tests {
)
}
#[test]
fn output_grammar_preserves_tool_strict_level() {
let build = |level, strict, choice| {
let tools = vec![ChatTool {
name: "search".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": { "query": { "type": "string" } },
"required": ["query"]
}),
strict,
}];
let mut request = ChatRequest {
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
..ChatRequest::for_test()
};
let mut processor = DefaultChatOutputProcessor::new(
&mut request,
"other-model",
tokenizer(),
&ParserSelection::Explicit("qwen3_coder".to_string()),
&ParserSelection::None,
level,
)
.unwrap();
processor.initialize(&[]).unwrap();
processor.build_output_grammar().unwrap()
};
assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none());
let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap();
let parameters = build(ToolStrictLevel::Parameter, None, ChatToolChoice::Auto).unwrap();
let strict = build(ToolStrictLevel::Auto, Some(true), ChatToolChoice::Auto).unwrap();
assert_eq!(parameters, strict);
assert_ne!(envelope, parameters);
assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none());
}
#[test]
fn tool_grammar_honors_parallel_call_policy() {
for parallel in [false, true] {
let tools = vec![ChatTool {
name: "lookup".to_string(),
description: None,
parameters: serde_json::json!({"type": "object"}),
strict: Some(true),
}];
let mut request = ChatRequest {
tool_context: ResolvedToolContext::new(
&[],
tools,
Some(ChatToolChoice::Auto),
parallel,
)
.unwrap(),
..ChatRequest::for_test()
};
let mut processor = DefaultChatOutputProcessor::new(
&mut request,
"other-model",
tokenizer(),
&ParserSelection::Explicit("qwen3_coder".to_string()),
&ParserSelection::None,
ToolStrictLevel::Auto,
)
.unwrap();
processor.initialize(&[]).unwrap();
let built = processor.build_output_grammar().unwrap().unwrap();
let tag =
serde_json::to_value(xgrammar_structural_tag::StructuralTag::new(built.format))
.unwrap();
assert_eq!(tag["format"]["type"], "triggered_tags");
assert_eq!(tag["format"]["stop_after_first"], !parallel);
}
}
#[test]
fn equal_explicit_gemma4_uses_unified_parser() {
let mut request = ChatRequest::for_test();
@@ -1,507 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Applies xgrammar structural-tag constraints for strict tool calling.
use thiserror_ext::AsReport;
use vllm_engine_core_client::protocol::structured_outputs::{
StructuredOutputBackend, StructuredOutputsParams,
};
use vllm_parser::tool::StructuralTagBuilder;
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
build_structural_tag,
};
use crate::parser::ToolStrictLevel;
use crate::request::{ChatRequest, ChatToolChoice};
use crate::{Error, Result as ChatResult};
/// Apply structural tag constraints to the request based on the tool parser's structural tag
/// support, the request's tool choice, and the server-side strictness floor.
pub(super) fn apply_structural_tag_constraint(
request: &mut ChatRequest,
builder: Option<&dyn StructuralTagBuilder>,
strict_level: ToolStrictLevel,
) -> ChatResult<()> {
let Some(builder) = builder else {
return Ok(());
};
let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
return Ok(());
};
let tools = request
.tools()
.iter()
.map(|tool| {
// A tool without an explicit `strict` is non-strict: its call envelope is
// constrained, but its arguments stay free unless the server level is `parameter`.
let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: Some(tool.parameters.clone()),
strict: Some(strict),
}))
})
.collect::<Vec<_>>();
let structural_tag = build_structural_tag(
builder,
&tools,
tool_choice,
StructuralTagOptions::default()
.with_reasoning(false)
.with_parallel_tool_calls(request.parallel_tool_calls()),
)
.and_then(|tag| tag.to_json_string())
.map_err(|error| Error::StructuralTag {
message: error.to_report_string(),
})?;
// Overwrite any existing structured output settings with the structural tag constraint.
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::structural_tag(structural_tag)
});
Ok(())
}
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(
request: &ChatRequest,
strict_level: ToolStrictLevel,
) -> Option<StructuralTagToolChoice> {
if request.tools().is_empty() {
return None;
}
match request.tool_choice() {
ChatToolChoice::None => None,
// For `Auto`, apply the structural tag only when the server raises the floor or at
// least one tool opts in with `strict: true`.
ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
|| request.tools().iter().any(|tool| tool.strict == Some(true)))
.then(StructuralTagToolChoice::auto),
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
}
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use vllm_engine_core_client::protocol::structured_outputs::{
StructuredOutputBackend, StructuredOutputsParams,
};
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
use super::*;
use crate::request::{ChatMessage, ResolvedToolContext};
fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
Tool {
name: name.to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"query": { "type": "string" }
},
"required": ["query"]
}),
strict,
}
}
fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
}
fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
ChatRequest {
tool_context: ResolvedToolContext::new(&[], tools, Some(tool_choice), true)
.expect("tool context should resolve"),
..ChatRequest::for_test()
}
}
fn structural_tag_value(request: &ChatRequest) -> Value {
let params = request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set");
assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
let structural_tag = params
.constraint
.as_structural_tag()
.expect("structured output constraint should be structural_tag");
serde_json::from_str(structural_tag).expect("structural_tag should be valid JSON")
}
fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set")
}
#[test]
fn tool_grammar_honors_parallel_call_policy() {
for parallel in [false, true] {
let tools = vec![chat_tool("lookup", Some(true))];
let mut request = ChatRequest {
tool_context: ResolvedToolContext::new(
&[],
tools,
Some(ChatToolChoice::Auto),
parallel,
)
.unwrap(),
..ChatRequest::for_test()
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.unwrap();
let tag = structural_tag_value(&request);
assert_eq!(tag["format"]["type"], "triggered_tags");
assert_eq!(tag["format"]["stop_after_first"], !parallel);
}
}
#[test]
fn auto_strict_tool_choice_builds_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_dynamic_tool_builds_structural_tag_from_effective_tools() {
let messages = vec![ChatMessage::developer(
"",
Some(vec![chat_tool("lookup", None)]),
)];
let tool_context =
ResolvedToolContext::new(&messages, Vec::new(), Some(ChatToolChoice::Required), true)
.expect("dynamic tool context should resolve");
let mut request = ChatRequest {
messages,
tool_context,
..ChatRequest::for_test()
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert!(tag.to_string().contains("lookup"));
}
#[test]
fn required_initial_and_dynamic_tools_build_one_structural_tag() {
let messages = vec![ChatMessage::developer(
"",
Some(vec![chat_tool("lookup", None)]),
)];
let tool_context = ResolvedToolContext::new(
&messages,
vec![chat_tool("search", None)],
Some(ChatToolChoice::Required),
true,
)
.expect("tool context should resolve");
let mut request = ChatRequest {
messages,
tool_context,
..ChatRequest::for_test()
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("search"));
assert!(tag.contains("lookup"));
}
#[test]
fn auto_non_strict_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json(json!({"type": "object"}))
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_builds_structural_tag_without_strict_tools() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_overwrites_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json_object()
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
let mut request = request(
ChatToolChoice::Function {
name: "lookup".to_string(),
},
vec![chat_tool("search", None), chat_tool("lookup", None)],
);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("lookup"));
assert!(!tag.contains("search"));
}
#[test]
fn none_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn none_tool_choice_preserves_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json_object()
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
let params = structured_outputs(&request);
assert!(params.constraint.is_json_object());
}
fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
let mut request = request(tool_choice, tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
.expect("structural tag decision should succeed");
request
}
fn tag_string(request: &ChatRequest) -> String {
structural_tag_value(request).to_string()
}
#[test]
fn function_level_constrains_auto_without_strict_tools() {
let request = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn absent_strict_is_non_strict() {
let unset = build(
ChatToolChoice::Required,
vec![chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let non_strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(false))],
ToolStrictLevel::Auto,
);
let strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&unset), tag_string(&non_strict));
assert_ne!(tag_string(&unset), tag_string(&strict));
}
#[test]
fn strict_tool_does_not_pin_its_neighbour() {
let mixed = build(
ChatToolChoice::Auto,
vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let explicit = build(
ChatToolChoice::Auto,
vec![
chat_tool("weather", Some(true)),
chat_tool("search", Some(false)),
],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&mixed), tag_string(&explicit));
}
#[test]
fn parameter_level_pins_argument_schemas() {
let pinned = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Parameter,
);
let strict = build(
ChatToolChoice::Auto,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
let envelope_only = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
assert_eq!(tag_string(&pinned), tag_string(&strict));
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
}
#[test]
fn tool_strict_level_parses_case_insensitively() {
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
assert_eq!(
"PARAMETER".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Parameter)
);
assert_eq!(
"function".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Function)
);
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
assert_eq!(
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
json!("auto")
);
assert_eq!(
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
ToolStrictLevel::Auto
);
assert!("off".parse::<ToolStrictLevel>().is_err());
assert!("strict".parse::<ToolStrictLevel>().is_err());
}
}
@@ -48,25 +48,6 @@ impl UnifiedParserState {
}
}
/// Initialize parser state once prompt token IDs are available.
fn initialize(&mut self, prompt_token_ids: &[u32]) {
if self.parser_failed {
return;
}
match self.parser.initialize(prompt_token_ids) {
Ok(()) => {}
Err(error) => {
warn!(
error = %error.as_report(),
"failed to initialize unified parser; falling back to plain text deltas"
);
self.parser_failed = true;
self.open_call_index = None;
}
}
}
/// Convert one decoded text delta into zero or more parsed assistant events.
fn process_delta(&mut self, delta: DecodedText) -> Result<Vec<AssistantEvent>> {
if self.parser_failed {
@@ -262,7 +243,6 @@ pub(crate) async fn unified_event_stream(
prompt_token_ids,
prompt_logprobs,
} => {
state.initialize(&prompt_token_ids);
y.yield_ok(AssistantEvent::Start {
prompt_token_ids,
prompt_logprobs,
+20
View File
@@ -7,6 +7,7 @@ use std::sync::Arc;
use futures::Stream;
use trait_set::trait_set;
use uuid::Uuid;
use vllm_parser::output_grammar::BuiltOutputGrammar;
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
use crate::FinishReason;
@@ -15,11 +16,13 @@ use crate::event::{AssistantBlockKind, ChatEvent, ChatTokenUsage};
mod default;
mod harmony;
mod structural_tag;
mod structured;
pub use default::DefaultChatOutputProcessor;
pub use harmony::HarmonyChatOutputProcessor;
pub(crate) use harmony::validate_harmony_parser_overrides;
pub(crate) use structural_tag::apply_output_grammar;
/// Internal assistant event before final assembly.
///
@@ -65,7 +68,24 @@ pub type DynChatEventStream = Pin<Box<dyn Stream<Item = Result<ChatEvent>> + Sen
/// Request-scoped output processor from decoded text events into structured
/// chat events.
///
/// A processor is created before the prompt is rendered, so that parser
/// resolution can adjust the request, and then goes through two more phases
/// once the final prompt token IDs are known: [`Self::initialize`] and
/// [`Self::build_output_grammar`], in that order, both before
/// [`Self::process`].
pub trait ChatOutputProcessor: Send {
/// Initialize request-scoped parser state from the final prompt token IDs.
fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
Ok(())
}
/// Build the structured output grammar implied by the initialized parser
/// state, or `None` when this request needs no parser-owned grammar.
fn build_output_grammar(&self) -> Result<Option<BuiltOutputGrammar>> {
Ok(None)
}
/// Consume decoded text stream and return the structured chat-event stream.
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream>;
}
@@ -0,0 +1,73 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Applies parser-built xgrammar structural-tag constraints.
use vllm_engine_core_client::protocol::structured_outputs::{
StructuredOutputBackend, StructuredOutputsParams,
};
use vllm_parser::output_grammar::{BuiltOutputGrammar, GrammarCoverage};
use vllm_text::TextRequest;
use xgrammar_structural_tag::StructuralTag;
use crate::{Error, Result};
/// Apply one parser-built output grammar to the prepared text request.
///
/// A parser-owned grammar replaces any structured output constraint the user
/// supplied, matching the Python frontend. `None` leaves the request untouched.
pub(crate) fn apply_output_grammar(
request: &mut TextRequest,
built: Option<BuiltOutputGrammar>,
) -> Result<()> {
let Some(built) = built else {
return Ok(());
};
let structural_tag = StructuralTag::new(built.format).to_json_string().map_err(|error| {
Error::OutputGrammar {
error: Box::new(error),
}
})?;
// Overwrite any existing structured output settings with the structural tag constraint.
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::structural_tag(structural_tag)
});
request.reasoning_ended = match built.coverage {
GrammarCoverage::FromTokenZero => Some(true),
GrammarCoverage::FinalOutputOnly => None,
};
Ok(())
}
#[cfg(test)]
mod tests {
use serde_json::json;
use vllm_engine_core_client::protocol::structured_outputs::{
StructuredOutputBackend, StructuredOutputsParams,
};
use xgrammar_structural_tag::format::Format;
use super::*;
#[test]
fn output_grammar_overwrites_answer_constraint() {
let mut request = TextRequest::for_test();
request.sampling_params.structured_outputs =
Some(StructuredOutputsParams::json(json!({ "type": "object" })));
let built = BuiltOutputGrammar::from_token_zero(Format::const_string("answer"));
apply_output_grammar(&mut request, Some(built)).unwrap();
assert_eq!(request.reasoning_ended, Some(true));
let params = request.sampling_params.structured_outputs.unwrap();
assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
let serialized = params.constraint.as_structural_tag().unwrap();
let value: serde_json::Value = serde_json::from_str(serialized).unwrap();
assert_eq!(value["type"], "structural_tag");
assert_eq!(value["format"]["type"], "const_string");
assert_eq!(value["format"]["value"], "answer");
}
}
+2 -60
View File
@@ -12,6 +12,8 @@ use std::str::FromStr;
use serde_with::{DeserializeFromStr, SerializeDisplay};
pub use vllm_parser::output_grammar::ToolStrictLevel;
use self::reasoning::ReasoningParserFactory;
use self::tool::ToolParserFactory;
use self::unified::UnifiedParserFactory;
@@ -57,66 +59,6 @@ impl ParserSelection {
}
}
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
DeserializeFromStr,
SerializeDisplay,
)]
pub enum ToolStrictLevel {
/// Derive tool constraints from the request's tool choice and per-tool strictness.
#[default]
Auto,
/// Constrain the tool-call envelope for every request with tools.
Function,
/// Additionally pin argument schemas, as if every tool were `strict: true`.
Parameter,
}
impl ToolStrictLevel {
pub const AUTO_LITERAL: &str = "auto";
pub const FUNCTION_LITERAL: &str = "function";
pub const PARAMETER_LITERAL: &str = "parameter";
}
impl FromStr for ToolStrictLevel {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
Ok(Self::Auto)
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
Ok(Self::Function)
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
Ok(Self::Parameter)
} else {
Err(format!(
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
Self::AUTO_LITERAL,
Self::FUNCTION_LITERAL,
Self::PARAMETER_LITERAL
))
}
}
}
impl fmt::Display for ToolStrictLevel {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Auto => Self::AUTO_LITERAL,
Self::Function => Self::FUNCTION_LITERAL,
Self::Parameter => Self::PARAMETER_LITERAL,
})
}
}
/// Validate explicit parser override names without starting request processing.
pub fn validate_parser_overrides(
tool_call_parser: &ParserSelection,
+11
View File
@@ -435,6 +435,17 @@ pub enum ChatToolChoice {
},
}
impl From<&ChatToolChoice> for xgrammar_structural_tag::ToolChoice {
fn from(tool_choice: &ChatToolChoice) -> Self {
match tool_choice {
ChatToolChoice::None => Self::none(),
ChatToolChoice::Auto => Self::auto(),
ChatToolChoice::Required => Self::required(),
ChatToolChoice::Function { name } => Self::function(name.clone()),
}
}
}
/// Resolved tool state shared by rendering, output parsing, and constraints.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ResolvedToolContext {
+2 -1
View File
@@ -728,7 +728,7 @@ async fn parse_completion(
};
let mut request = base_request.clone();
let processor = backends.chat_backend.new_chat_output_processor(
let mut processor = backends.chat_backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
@@ -736,6 +736,7 @@ async fn parse_completion(
reasoning_parser: &case.reasoning_parser,
},
)?;
processor.initialize(&prompt_token_ids)?;
let decoded = decoded_completion_stream(
tokenizer.as_ref(),