[Rust Frontend] Introduce unified parser interface & combined parser (#46583)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-25 03:17:31 +00:00
committed by GitHub
parent efb5acffd5
commit 76c3c4ff63
74 changed files with 1204 additions and 1745 deletions
+1 -2
View File
@@ -30,10 +30,9 @@ trait-set.workspace = true
uuid.workspace = true
vllm-engine-core-client.workspace = true
vllm-llm.workspace = true
vllm-reasoning-parser.workspace = true
vllm-parser.workspace = true
vllm-text.workspace = true
vllm-tokenizer.workspace = true
vllm-tool-parser.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
+22 -43
View File
@@ -1,44 +1,34 @@
//! Default output processing pipeline.
mod reasoning;
mod structural_tag;
mod tool;
mod unified;
use std::sync::Once;
use futures::{Stream, StreamExt as _};
use futures::StreamExt as _;
use tracing::info;
use trait_set::trait_set;
use vllm_parser::unified::{CombinedParser, UnifiedParser};
use vllm_text::tokenizer::DynTokenizer;
use self::reasoning::reasoning_event_stream;
use self::structural_tag::apply_structural_tag_constraint;
use self::tool::tool_event_stream;
use self::unified::unified_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
use crate::output::{
AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream,
DynDecodedTextEventStream,
};
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::request::ChatRequest;
use crate::{Error, Result as ChatResult};
trait_set! {
trait ContentEventStream = Stream<Item = Result<ContentEvent>> + Send + 'static;
}
/// Default request-scoped output processor used by Hugging Face style chat
/// backends.
///
/// This implementation assumes the backend already emitted decoded text deltas,
/// then optionally layers reasoning parsing and tool-call parsing before
/// then optionally layers unified reasoning and tool-call parsing before
/// assembling final structured chat events.
pub struct DefaultChatOutputProcessor {
reasoning_parser: Option<Box<dyn ReasoningParser>>,
tool_parser: Option<Box<dyn ToolParser>>,
parser: Box<dyn UnifiedParser>,
parallel_tool_calls: bool,
}
@@ -66,16 +56,17 @@ impl DefaultChatOutputProcessor {
} else {
None
};
let reasoning_parser = Self::resolve_optional_reasoning_parser(
request,
model_id,
tokenizer,
reasoning_parser,
)?;
let reasoning_parser =
Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
let parser: Box<dyn UnifiedParser> =
Box::new(CombinedParser::new(reasoning_parser, tool_parser));
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
Ok(Self {
reasoning_parser,
tool_parser,
parser,
parallel_tool_calls: request.parallel_tool_calls,
})
}
@@ -87,8 +78,7 @@ impl DefaultChatOutputProcessor {
/// content is treated as opaque text.
pub fn plain_text_only() -> Self {
Self {
reasoning_parser: None,
tool_parser: None,
parser: Box::new(CombinedParser::plain_text_only()),
parallel_tool_calls: true,
}
}
@@ -112,10 +102,6 @@ impl DefaultChatOutputProcessor {
let parser = factory.create(parser_name, &request.tools)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
apply_structural_tag_constraint(request, parser.as_ref())?;
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
@@ -123,7 +109,6 @@ impl DefaultChatOutputProcessor {
}
fn resolve_optional_reasoning_parser(
request: &mut ChatRequest,
model_id: &str,
tokenizer: DynTokenizer,
selection: &ParserSelection,
@@ -142,10 +127,6 @@ impl DefaultChatOutputProcessor {
let parser = factory.create(parser_name, tokenizer)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
Ok(Some(parser))
}
@@ -156,16 +137,14 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
impl ChatOutputProcessor for DefaultChatOutputProcessor {
/// Transforms a raw generate-output token stream into structured chat
/// events through three sequential stages once text decoding has
/// events through two sequential stages once text decoding has
/// already happened:
///
/// 1. [`reasoning_event_stream`] — reasoning/content separation
/// 2. [`tool_event_stream`] — tool-call parsing
/// 3. [`structured_chat_event_stream`] — final block assembly
/// 1. [`unified_event_stream`] — reasoning and tool-call parsing
/// 2. [`structured_chat_event_stream`] — final block assembly
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
let tool = tool_event_stream(reasoning, self.tool_parser);
let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
let parsed = unified_event_stream(decoded, self.parser);
let structured = structured_chat_event_stream(parsed, self.parallel_tool_calls);
Ok(structured.boxed())
}
@@ -1,509 +0,0 @@
//! Adapts decoded text updates into reasoning-aware assistant deltas.
//!
//! This stage sits between low-level token decoding and final block assembly.
//! It is the only place in the new pipeline that understands reasoning
//! separation: `decoded.rs` still only produces plain text deltas, while later
//! stages consume the semantic `Text` / `Reasoning` split emitted here.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_text::output::DecodedTextEvent;
use super::ContentEvent;
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::DecodedTextEventStream;
use crate::parser::reasoning::{ReasoningDelta, ReasoningParser};
/// Per-stream reasoning parsing state.
struct ReasoningState {
/// Reasoning parser for the current model family.
parser: Box<dyn ReasoningParser>,
/// Whether reasoning parsing has already failed for this stream.
parser_failed: bool,
}
impl ReasoningState {
/// Create one fresh reasoning-adaptation state for a new streamed response.
fn new(parser: Box<dyn ReasoningParser>) -> Self {
Self {
parser,
parser_failed: false,
}
}
/// Convert one decoded text delta into zero or more semantic assistant
/// deltas.
fn process_delta(&mut self, delta: String) -> Vec<ContentEvent> {
// If the parser has already failed, skip parsing and return plain text deltas.
if self.parser_failed {
return vec![ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta,
}];
}
let mut events = Vec::new();
match self.parser.push(&delta) {
Ok(result) => {
push_reasoning_delta(&mut events, result);
}
Err(error) => {
if !self.parser_failed {
warn!(
error = %error.as_report(),
"reasoning parser failed; falling back to plain text deltas"
);
self.parser_failed = true;
}
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
}
}
events
}
/// Initialize parser state once prompt token IDs are available.
fn initialize(&mut self, prompt_token_ids: &[u32]) {
if self.parser_failed {
return;
}
match self.parser.initialize(prompt_token_ids) {
Ok(()) => {}
Err(error) => {
warn!(
error = %error.as_report(),
"failed to initialize reasoning parser; falling back to plain text deltas"
);
self.parser_failed = true;
}
}
}
/// Flush any parser-held partial delimiter state at end of stream.
fn finish(&mut self) -> Vec<ContentEvent> {
if self.parser_failed {
return Vec::new();
}
match self.parser.finish() {
Ok(result) => {
let mut events = Vec::new();
push_reasoning_delta(&mut events, result);
events
}
Err(error) => {
warn!(error = %error.as_report(), "failed to flush reasoning parser state");
Vec::new()
}
}
}
}
/// Push one semantic text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<ContentEvent>, kind: AssistantBlockKind, delta: String) {
if delta.is_empty() {
return;
}
events.push(ContentEvent::TextDelta { kind, delta });
}
/// Convert one parsed reasoning delta into zero or more content events.
fn push_reasoning_delta(events: &mut Vec<ContentEvent>, delta: ReasoningDelta) {
if let Some(reasoning) = delta.reasoning {
push_text_delta(events, AssistantBlockKind::Reasoning, reasoning);
}
if let Some(content) = delta.content {
push_text_delta(events, AssistantBlockKind::Text, content);
}
}
/// Wrap one decoded-text stream into the internal reasoning event stream.
#[try_stream]
pub(crate) async fn reasoning_event_stream(
decoded_stream: impl DecodedTextEventStream,
reasoning_parser: Option<Box<dyn ReasoningParser>>,
mut y: TryYielder<ContentEvent, Error>,
) -> Result<()> {
pin_mut!(decoded_stream);
// Without a parser, pass through as plain text deltas.
let Some(reasoning_parser) = reasoning_parser else {
while let Some(event) = decoded_stream.next().await.transpose()? {
for next in ContentEvent::from_decoded_plain_text(event) {
y.yield_ok(next).await;
}
}
return Ok(());
};
let mut state = ReasoningState::new(reasoning_parser);
while let Some(event) = decoded_stream.next().await.transpose()? {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
state.initialize(&prompt_token_ids);
y.yield_ok(ContentEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
for next in state.process_delta(delta) {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
y.yield_ok(ContentEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
if let Some(finished) = finished {
for next in state.finish() {
y.yield_ok(next).await;
}
y.yield_ok(ContentEvent::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
.await;
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use futures::{StreamExt as _, stream};
use vllm_llm::FinishReason;
use vllm_text::output::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
};
use vllm_tokenizer::{DynTokenizer, Tokenizer};
use super::super::ContentEvent;
use super::reasoning_event_stream;
use crate::event::AssistantBlockKind;
use crate::parser::reasoning::{
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names,
};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
_ => None,
}
}
}
struct FailingReasoningParser {
fail_next: bool,
}
impl ReasoningParser for FailingReasoningParser {
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>, ReasoningError>
where
Self: Sized + 'static,
{
Ok(Box::new(Self { fail_next: true }))
}
fn push(&mut self, _text: &str) -> Result<ReasoningDelta, ReasoningError> {
if self.fail_next {
self.fail_next = false;
return Err(ReasoningError::MissingToken {
token: "<think>".to_string(),
});
}
Ok(ReasoningDelta::default())
}
}
fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box<dyn ReasoningParser> {
factory.register_parser::<FailingReasoningParser>("failing");
factory.create("failing", Arc::new(FakeTokenizer)).unwrap()
}
#[tokio::test]
async fn reasoning_parser_failure_falls_back_to_plain_text() {
let mut factory = ReasoningParserFactory::new();
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "abc".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "def".to_string(),
token_ids: vec![],
logprobs: None,
finished: Some(vllm_text::Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}),
]);
let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory)))
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
},
ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn reasoning_stream_preserves_logprobs_delta() {
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "abc".to_string(),
token_ids: vec![],
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
}),
finished: None,
}),
]);
let collected = reasoning_event_stream(events, None)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert_eq!(
collected,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
ContentEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
}),
token_ids: vec![],
},
]
);
}
#[tokio::test]
async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() {
let tokenizer = Arc::new(FakeTokenizer);
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "thought ".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "done</think>OK".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
]);
let factory = ReasoningParserFactory::new();
let collected = reasoning_event_stream(
events,
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![2].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "thought ".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "done</think>OK".to_string(),
},
]
);
}
#[tokio::test]
async fn qwen3_parser_tolerates_prompt_prefill_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "thought ".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "done</think>OK".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
]);
let factory = ReasoningParserFactory::new();
let collected = reasoning_event_stream(
events,
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thought ".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "done".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "OK".to_string(),
},
]
);
}
}
@@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
mod tests {
use serde_json::{Value, json};
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
use vllm_parser::tool::{Qwen3CoderToolParser, Tool};
use super::*;
-985
View File
@@ -1,985 +0,0 @@
//! Adapts plain assistant text deltas into tool-call-aware assistant updates.
//!
//! This stage runs after reasoning separation and before final block assembly.
//! It only inspects normal assistant text, leaves reasoning deltas untouched,
//! and translates incremental tool parsing output into internal tool-call
//! events while preserving plain-text fallback behavior.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use super::{AssistantEvent, ContentEvent, ContentEventStream};
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::generate_tool_call_id;
use crate::parser::tool::{ToolCallDelta, ToolParser, ToolParserOutput};
/// Per-stream tool parsing state.
struct ToolState {
/// Parser for the current model family.
parser: Box<dyn ToolParser>,
/// Whether tool parsing has already failed for this stream.
parser_failed: bool,
/// The parser-local index of the currently open tool call, if any.
// NOTE: We only allow single open tool call at a time right now, since that's what all
// supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple
// interleaved calls in the future.
open_call_index: Option<usize>,
}
impl ToolState {
/// Create one fresh tool-parsing state for a new streamed response.
fn new(parser: Box<dyn ToolParser>) -> Self {
Self {
parser,
parser_failed: false,
open_call_index: None,
}
}
/// Convert one semantic assistant text delta into zero or more tool-aware
/// internal events.
fn process_text_delta(
&mut self,
kind: AssistantBlockKind,
delta: String,
) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
// Only normal assistant text is eligible for tool parsing. Reasoning
// blocks and plain-text fallback should pass through unchanged.
if kind != AssistantBlockKind::Text || self.parser_failed {
self.open_call_index = None;
events.push(AssistantEvent::TextDelta { kind, delta });
return Ok(events);
}
let mut output = ToolParserOutput::default();
let parse_result = self.parser.parse_into(&delta, &mut output);
match parse_result {
Ok(()) => self.process_parser_output(kind, output, &mut events)?,
Err(error) => {
warn!(
error = %error.as_report(),
"tool parser failed; falling back to plain text deltas"
);
// Permanently mark this parser as failed.
// TODO: we may consider recovering from parsing errors in the future.
self.parser_failed = true;
// On parsing failure, we still apply the partial parser output if any, but we close
// any open tool calls and emit the remaining buffered text as a plain-text delta to
// preserve as much of the output as possible.
self.process_parser_output(kind, output, &mut events)?;
self.open_call_index = None;
push_text_delta(&mut events, kind, self.parser.reset());
}
}
Ok(events)
}
/// Apply one parsed tool output to the current stream state.
fn process_parser_output(
&mut self,
kind: AssistantBlockKind,
output: ToolParserOutput,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
// When we are not currently streaming a tool call, preserve plain
// text first and then surface any new tool call items.
if self.open_call_index.is_none() {
push_text_delta(events, kind, output.normal_text);
self.process_tool_items(output.calls, events)?;
} else {
// Once a tool call is open, prioritize tool deltas first. If the
// parser emits normal text again, close the tool call and resume
// plain text output.
self.process_tool_items(output.calls, events)?;
if !output.normal_text.is_empty() {
self.open_call_index = None;
push_text_delta(events, kind, output.normal_text);
}
}
Ok(())
}
/// Apply one batch of parsed tool-call deltas emitted by the parser.
fn process_tool_items(
&mut self,
items: Vec<ToolCallDelta>,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
for item in items {
if let Some(name) = item.name {
let is_new_tool = match self.open_call_index {
Some(open_call_index) => open_call_index != item.tool_index,
None => true,
};
if is_new_tool {
let id = self
.parser
.tool_call_id(item.tool_index)
.map(str::to_string)
.unwrap_or_else(generate_tool_call_id);
self.open_call_index = Some(item.tool_index);
events.push(AssistantEvent::ToolCallStart { id, name });
}
}
if item.arguments.is_empty() {
// No arguments delta to apply.
continue;
}
let Some(open_call_index) = self.open_call_index else {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} before any tool-call start",
item.tool_index
),
});
};
if open_call_index != item.tool_index {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} while tool index {} is open",
item.tool_index, open_call_index
),
});
}
events.push(AssistantEvent::ToolCallArgumentsDelta {
delta: item.arguments,
});
}
Ok(())
}
/// Flush parser state at end-of-stream and close any remaining open calls.
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
if self.parser_failed {
return Ok(events);
}
match self.parser.finish() {
Ok(output) => {
self.process_parser_output(AssistantBlockKind::Text, output, &mut events)?
}
Err(error) => {
warn!(
error = %error.as_report(),
"tool parser finish failed; closing open tool calls with buffered state"
);
self.parser_failed = true;
}
}
Ok(events)
}
}
/// Push one plain-text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
if delta.is_empty() {
return;
}
events.push(AssistantEvent::TextDelta { kind, delta });
}
/// Wrap one semantic assistant stream into the internal tool-aware assistant
/// stream.
#[try_stream]
pub(crate) async fn tool_event_stream(
stream: impl ContentEventStream,
parser: Option<Box<dyn ToolParser>>,
mut y: TryYielder<AssistantEvent, Error>,
) -> Result<()> {
// Without a parser, pass through the input stream unchanged.
let Some(parser) = parser else {
pin_mut!(stream);
while let Some(event) = stream.next().await.transpose()? {
y.yield_ok(event.into()).await;
}
return Ok(());
};
pin_mut!(stream);
let mut state = ToolState::new(parser);
while let Some(event) = stream.next().await.transpose()? {
match event {
ContentEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
y.yield_ok(AssistantEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
ContentEvent::TextDelta { kind, delta } => {
for next in state.process_text_delta(kind, delta)? {
y.yield_ok(next).await;
}
}
ContentEvent::LogprobsDelta {
logprobs,
token_ids,
} => {
y.yield_ok(AssistantEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
ContentEvent::Done {
usage,
finish_reason,
kv_transfer_params,
} => {
for next in state.finish()? {
y.yield_ok(next).await;
}
y.yield_ok(AssistantEvent::Done {
usage,
finish_reason,
kv_transfer_params,
})
.await;
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use futures::{StreamExt as _, stream};
use vllm_llm::FinishReason;
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use vllm_tool_parser::Result;
use super::super::{AssistantEvent, ContentEvent};
use super::tool_event_stream;
use crate::error::Error;
use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
use crate::output::structured::structured_chat_event_stream;
use crate::parser::tool::{
DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput,
};
use crate::request::ChatTool;
use crate::stream::{ChatEventStream, CollectedAssistantMessage};
struct FailingParser {
fail_next: bool,
buffered: String,
}
struct ScriptedParser {
push_outputs: Vec<ToolParserOutput>,
finish_output: ToolParserOutput,
}
struct PartialThenFailParser {
buffered: String,
}
struct IdScriptedParser {
output: ToolParserOutput,
tool_call_id: Option<String>,
}
impl ToolParser for FailingParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
fail_next: false,
buffered: String::new(),
}))
}
fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
self.buffered.push_str(chunk);
if self.fail_next {
self.fail_next = false;
return Err(ToolParserError::ParsingFailed {
message: "boom".to_string(),
});
}
self.buffered.clear();
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
impl ToolParser for ScriptedParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
push_outputs: Vec::new(),
finish_output: ToolParserOutput::default(),
}))
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
let mut next = self.push_outputs.pop().unwrap_or_default();
output.normal_text.push_str(&next.normal_text);
output.calls.append(&mut next.calls);
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(std::mem::take(&mut self.finish_output))
}
fn reset(&mut self) -> String {
String::new()
}
}
impl ToolParser for IdScriptedParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
output: ToolParserOutput::default(),
tool_call_id: None,
}))
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
(tool_index == 0).then_some(self.tool_call_id.as_deref()).flatten()
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
output.append(std::mem::take(&mut self.output));
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
String::new()
}
}
impl ToolParser for PartialThenFailParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
buffered: String::new(),
}))
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
output.calls.extend([
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: String::new(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: r#"{"location":"SF"}"#.to_string(),
},
]);
self.buffered.push_str(" trailing text");
Err(ToolParserError::ParsingFailed {
message: "boom".to_string(),
})
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
fn deepseek_v4_test_tools() -> Vec<ChatTool> {
vec![
ChatTool {
name: "get_weather".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"location": { "type": "string" }
}
}),
strict: None,
},
ChatTool {
name: "add".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"x": { "type": "integer" },
"y": { "type": "integer" }
}
}),
strict: None,
},
]
}
async fn collect_deepseek_v4_message(chunks: Vec<String>) -> CollectedAssistantMessage {
let events = chunks
.into_iter()
.map(|delta| {
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta,
})
})
.chain(std::iter::once(Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})));
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
let chat_events = structured_chat_event_stream(assistant_events, true);
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
.collect_message()
.await
.unwrap()
}
fn message_tool_projection(
message: &CollectedAssistantMessage,
) -> (String, Vec<(String, serde_json::Value)>) {
(
message.message.text(),
message
.message
.tool_calls()
.map(|call| {
(
call.name.clone(),
serde_json::from_str(&call.arguments).unwrap(),
)
})
.collect(),
)
}
#[tokio::test]
async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(
events,
Some(Box::new(PartialThenFailParser {
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { name, .. } if name == "get_weather"
));
assert!(matches!(
&events[1],
AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"#
));
assert_eq!(
events[2],
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: " trailing text".to_string(),
}
);
assert!(matches!(events[3], AssistantEvent::Done { .. }));
}
#[tokio::test]
async fn tool_stream_preserves_parser_provided_tool_call_id() {
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
})]);
let parser = IdScriptedParser {
output: ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: "{}".to_string(),
}],
},
tool_call_id: Some("functions.get_weather:0".to_string()),
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { id, name }
if id == "functions.get_weather:0" && name == "get_weather"
));
}
#[tokio::test]
async fn tool_stream_generates_tool_call_id_when_parser_omits_one() {
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
})]);
let parser = IdScriptedParser {
output: ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: "{}".to_string(),
}],
},
tool_call_id: None,
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { id, name }
if id.starts_with("call_") && name == "get_weather"
));
}
#[tokio::test]
async fn real_buffered_parser_error_matches_streaming_and_non_streaming() {
let prefix = "I will check both.\n";
let first_tool_call = concat!(
"<|DSML|tool_calls>\n",
"<|DSML|invoke name=\"get_weather\">\n",
"<|DSML|parameter name=\"location\" string=\"true\">Tokyo</|DSML|parameter>\n",
"</|DSML|invoke>",
);
let malformed_second_tool_call = concat!(
"\n<|DSML|invoke name=\"add\">\n",
"not a parameter\n",
"</|DSML|invoke>\n",
"</|DSML|tool_calls>",
);
let streaming_chunks = vec![
prefix.to_string(),
first_tool_call.to_string(),
malformed_second_tool_call.to_string(),
];
let full_output = streaming_chunks.concat();
let streaming = collect_deepseek_v4_message(streaming_chunks).await;
let non_streaming = collect_deepseek_v4_message(vec![full_output]).await;
let expected = (
format!("{prefix}{malformed_second_tool_call}"),
vec![(
"get_weather".to_string(),
serde_json::json!({ "location": "Tokyo" }),
)],
);
assert_eq!(message_tool_projection(&streaming), expected);
assert_eq!(message_tool_projection(&non_streaming), expected);
}
#[tokio::test]
async fn tool_parser_failure_falls_back_to_plain_text() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let collected = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: true,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("tool stream should not fail");
assert_eq!(
events,
vec![
AssistantEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
let message = ChatEventStream::new(
"req_fallback".to_string(),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
.expect("collect_message should succeed");
assert_eq!(message.message.text(), "abcdef");
assert!(message.message.tool_calls().next().is_none());
}
#[tokio::test]
async fn tool_stream_preserves_logprobs_delta() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
}],
}),
token_ids: vec![],
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: false,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert_eq!(
events,
vec![
AssistantEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
AssistantEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
}],
}),
token_ids: vec![],
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn tool_stream_rejects_interleaved_tool_indices() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let parser = ScriptedParser {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: String::new(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 1,
name: None,
arguments: "{}".to_string(),
},
],
}],
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
}
#[tokio::test]
async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "start".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "text".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "args".to_string(),
}),
]);
let parser = ScriptedParser {
push_outputs: vec![
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: "}".to_string(),
}],
},
ToolParserOutput {
normal_text: "plain text".to_string(),
calls: Vec::new(),
},
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: "{".to_string(),
}],
},
],
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(
err,
Error::ToolCallStreamInvariant { message }
if message == "received arguments for tool index 0 before any tool-call start"
));
}
#[tokio::test]
async fn tool_stream_emits_start_and_args_for_terminal_text() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let parser = ScriptedParser {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: r#"{"a":1}"#.to_string(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 1,
name: Some("second".to_string()),
arguments: r#"{"b":2}"#.to_string(),
},
],
}],
finish_output: ToolParserOutput::default(),
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. }));
assert!(matches!(
events[2],
AssistantEvent::ToolCallArgumentsDelta { .. }
));
assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. }));
assert!(matches!(
events[4],
AssistantEvent::ToolCallArgumentsDelta { .. }
));
let collected = ChatEventStream::new(
"req_final_only".to_string(),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
.unwrap();
let tool_calls = collected.message.tool_calls().collect::<Vec<_>>();
assert_eq!(tool_calls.len(), 2);
assert_eq!(tool_calls[0].name, "first");
assert_eq!(tool_calls[1].name, "second");
}
}
+582
View File
@@ -0,0 +1,582 @@
//! Adapts decoded text updates into parsed assistant deltas.
//!
//! This stage sits between low-level token decoding and final block assembly.
//! It drives one unified parser that may emit normal text, reasoning text, or
//! tool-call deltas, then normalizes those parser events into internal
//! assistant events.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
use vllm_text::output::DecodedTextEvent;
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
/// Per-stream unified parsing state.
struct UnifiedParserState {
/// Parser for the current request stream.
parser: Box<dyn UnifiedParser>,
/// Whether unified parsing has already failed for this stream.
parser_failed: bool,
/// The parser-local index of the currently open tool call, if any.
///
/// Supported parsers currently emit at most one active tool call at a time.
/// Change this to an indexed map if a model needs interleaved calls later.
open_call_index: Option<usize>,
}
impl UnifiedParserState {
/// Create one fresh unified parsing state for a new streamed response.
fn new(parser: Box<dyn UnifiedParser>) -> Self {
Self {
parser,
parser_failed: false,
open_call_index: None,
}
}
/// Initialize parser state once prompt token IDs are available.
fn initialize(&mut self, prompt_token_ids: &[u32]) {
if self.parser_failed {
return;
}
match self.parser.initialize(prompt_token_ids) {
Ok(()) => {}
Err(error) => {
warn!(
error = %error.as_report(),
"failed to initialize unified parser; falling back to plain text deltas"
);
self.parser_failed = true;
self.open_call_index = None;
}
}
}
/// Convert one decoded text delta into zero or more parsed assistant events.
fn process_delta(&mut self, delta: String) -> Result<Vec<AssistantEvent>> {
if self.parser_failed {
self.open_call_index = None;
return Ok(text_event(AssistantBlockKind::Text, delta).into_iter().collect());
}
let mut output = UnifiedParserOutput::default();
match self.parser.parse_into(&delta, &mut output) {
Ok(()) => {
let mut events = Vec::new();
self.process_parser_output(output, &mut events)?;
Ok(events)
}
Err(error) => {
warn!(
error = %error.as_report(),
"unified parser failed; falling back to plain text deltas"
);
self.parser_failed = true;
let mut events = Vec::new();
self.process_parser_output(output, &mut events)?;
self.open_call_index = None;
let recovered = self.parser.reset();
if recovered.is_empty() && events.is_empty() {
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
} else {
push_text_delta(&mut events, AssistantBlockKind::Text, recovered);
}
Ok(events)
}
}
}
/// Flush parser state at end-of-stream and close any remaining open calls.
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
if self.parser_failed {
return Ok(events);
}
match self.parser.finish() {
Ok(output) => self.process_parser_output(output, &mut events)?,
Err(error) => {
warn!(
error = %error.as_report(),
"unified parser finish failed; closing open parser state"
);
self.parser_failed = true;
self.open_call_index = None;
// TODO: should we reset and emit the buffered text?
}
}
Ok(events)
}
/// Apply one parsed unified output to the current stream state.
fn process_parser_output(
&mut self,
output: UnifiedParserOutput,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
for event in output.events {
match event {
UnifiedParserEvent::Text(delta) => {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Text, delta);
}
UnifiedParserEvent::Reasoning(delta) => {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Reasoning, delta);
}
UnifiedParserEvent::ToolCall(item) => {
self.process_tool_item(item, events)?;
}
}
}
Ok(())
}
/// Apply one parsed tool-call delta emitted by the parser.
fn process_tool_item(
&mut self,
item: vllm_parser::tool::ToolCallDelta,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
if let Some(name) = item.name {
let is_new_tool = match self.open_call_index {
Some(open_call_index) => open_call_index != item.tool_index,
None => true,
};
if is_new_tool {
let id = self
.parser
.tool_call_id(item.tool_index)
.map(str::to_string)
.unwrap_or_else(generate_tool_call_id);
self.open_call_index = Some(item.tool_index);
events.push(AssistantEvent::ToolCallStart { id, name });
}
}
if item.arguments.is_empty() {
return Ok(());
}
let Some(open_call_index) = self.open_call_index else {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} before any tool-call start",
item.tool_index
),
});
};
if open_call_index != item.tool_index {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} while tool index {} is open",
item.tool_index, open_call_index
),
});
}
events.push(AssistantEvent::ToolCallArgumentsDelta {
delta: item.arguments,
});
Ok(())
}
}
/// Build one plain text event if `delta` is non-empty.
fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
if delta.is_empty() {
return None;
}
Some(AssistantEvent::TextDelta { kind, delta })
}
/// Push one plain text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
if let Some(event) = text_event(kind, delta) {
events.push(event);
}
}
/// Wrap one decoded-text stream into the internal unified assistant stream.
#[try_stream]
pub(crate) async fn unified_event_stream(
decoded_stream: impl DecodedTextEventStream,
parser: Box<dyn UnifiedParser>,
mut y: TryYielder<AssistantEvent, Error>,
) -> Result<()> {
pin_mut!(decoded_stream);
let mut state = UnifiedParserState::new(parser);
while let Some(event) = decoded_stream.next().await.transpose()? {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
state.initialize(&prompt_token_ids);
y.yield_ok(AssistantEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
for next in state.process_delta(delta)? {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
y.yield_ok(AssistantEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
if let Some(finished) = finished {
for next in state.finish()? {
y.yield_ok(next).await;
}
y.yield_ok(AssistantEvent::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
.await;
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::VecDeque;
use futures::{StreamExt as _, stream};
use vllm_parser::reasoning::ReasoningError;
use vllm_parser::tool::ToolCallDelta;
use vllm_parser::unified::{UnifiedParserError, UnifiedParserOutput};
use super::unified_event_stream;
use crate::event::AssistantBlockKind;
use crate::output::AssistantEvent;
enum ScriptedStep {
Output(UnifiedParserOutput),
Error {
committed: UnifiedParserOutput,
reset_text: String,
},
}
struct ScriptedParser {
steps: VecDeque<ScriptedStep>,
reset_text: String,
tool_call_id: Option<String>,
finish_error_reset_text: Option<String>,
}
impl ScriptedParser {
fn new(steps: impl IntoIterator<Item = ScriptedStep>) -> Self {
Self {
steps: steps.into_iter().collect(),
reset_text: String::new(),
tool_call_id: Some("call_test".to_string()),
finish_error_reset_text: None,
}
}
fn with_finish_error(mut self, reset_text: &str) -> Self {
self.finish_error_reset_text = Some(reset_text.to_string());
self
}
}
impl vllm_parser::unified::UnifiedParser for ScriptedParser {
fn create(
_tools: &[vllm_parser::tool::Tool],
_tokenizer: vllm_tokenizer::DynTokenizer,
) -> vllm_parser::unified::Result<Box<dyn vllm_parser::unified::UnifiedParser>>
where
Self: Sized + 'static,
{
unreachable!("ScriptedParser is constructed directly in tests")
}
fn parse_into(
&mut self,
_delta: &str,
output: &mut UnifiedParserOutput,
) -> vllm_parser::unified::Result<()> {
match self.steps.pop_front().expect("unexpected parser call") {
ScriptedStep::Output(next) => {
output.append(next);
Ok(())
}
ScriptedStep::Error {
committed,
reset_text,
} => {
output.append(committed);
self.reset_text = reset_text;
Err(UnifiedParserError::Reasoning(
ReasoningError::MissingToken {
token: "<think>".to_string(),
},
))
}
}
}
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
self.tool_call_id.as_deref()
}
fn finish(&mut self) -> vllm_parser::unified::Result<UnifiedParserOutput> {
if let Some(reset_text) = self.finish_error_reset_text.take() {
self.reset_text = reset_text;
return Err(UnifiedParserError::Reasoning(
ReasoningError::MissingToken {
token: "</tool_call>".to_string(),
},
));
}
Ok(UnifiedParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.reset_text)
}
}
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: None,
}
}
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(vllm_text::output::Finished {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
}),
}
}
async fn collect(
parser: ScriptedParser,
events: Vec<vllm_text::output::DecodedTextEvent>,
) -> Vec<AssistantEvent> {
let stream = stream::iter(events.into_iter().map(Ok));
unified_event_stream(stream, Box::new(parser))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap()
}
fn text(delta: &str) -> UnifiedParserOutput {
let mut output = UnifiedParserOutput::default();
output.push_text(delta.to_string());
output
}
fn reasoning(delta: &str) -> UnifiedParserOutput {
let mut output = UnifiedParserOutput::default();
output.push_reasoning(delta.to_string());
output
}
fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
UnifiedParserOutput {
events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
ToolCallDelta {
tool_index: 0,
name: Some(name.to_string()),
arguments: arguments.to_string(),
},
)],
}
}
fn combined(first: UnifiedParserOutput, second: UnifiedParserOutput) -> UnifiedParserOutput {
let mut output = first;
output.append(second);
output
}
#[tokio::test]
async fn unified_stream_emits_reasoning_only_deltas() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(reasoning("thinking"))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
}]
);
}
#[tokio::test]
async fn unified_stream_emits_tool_only_deltas() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(tool_call(
"get_weather",
r#"{"location":"Paris"}"#,
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_emits_reasoning_followed_by_tool_call() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(combined(
reasoning("thinking"),
tool_call("get_weather", r#"{"location":"Paris"}"#),
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_emits_visible_text_followed_by_tool_call() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(combined(
text("visible "),
tool_call("get_weather", r#"{"location":"Paris"}"#),
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "visible ".to_string(),
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_fallback_keeps_committed_output_and_disables_later_parsing() {
let events = collect(
ScriptedParser::new([ScriptedStep::Error {
committed: text("committed"),
reset_text: "buffered".to_string(),
}]),
vec![decoded_delta("bad"), decoded_delta("later")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "committed".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "buffered".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "later".to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_finish_error_closes_parser_without_reset_text() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(UnifiedParserOutput::default())])
.with_finish_error("buffered"),
vec![finished_delta("")],
)
.await;
assert_eq!(
events,
vec![AssistantEvent::Done {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
}]
);
}
}
+2 -52
View File
@@ -2,7 +2,6 @@ use std::pin::Pin;
use std::sync::Arc;
use futures::Stream;
use subenum::subenum;
use trait_set::trait_set;
use uuid::Uuid;
use vllm_llm::TokenUsage;
@@ -22,23 +21,19 @@ pub(crate) use harmony::validate_harmony_parser_overrides;
/// Internal assistant event before final assembly.
///
/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content.
/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants.
#[subenum(ContentEvent)]
/// Unified parsing produces these events, and structured assembly consumes
/// them to build public chat events.
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum AssistantEvent {
#[subenum(ContentEvent)]
Start {
prompt_token_ids: Arc<[u32]>,
prompt_logprobs: Option<DecodedPromptLogprobs>,
},
#[subenum(ContentEvent)]
TextDelta {
kind: AssistantBlockKind,
delta: String,
},
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
#[subenum(ContentEvent)]
LogprobsDelta {
logprobs: Option<DecodedLogprobs>,
token_ids: Vec<u32>,
@@ -48,7 +43,6 @@ pub(crate) enum AssistantEvent {
/// A delta for the arguments of the currently open tool call. Must follow a
/// `ToolCallStart`.
ToolCallArgumentsDelta { delta: String },
#[subenum(ContentEvent)]
Done {
usage: TokenUsage,
finish_reason: FinishReason,
@@ -57,50 +51,6 @@ pub(crate) enum AssistantEvent {
},
}
impl ContentEvent {
/// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values
/// by treating all text as plain (non-reasoning) content.
fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec<Self> {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => vec![Self::Start {
prompt_token_ids,
prompt_logprobs,
}],
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
let mut events = Vec::new();
if !delta.is_empty() {
events.push(Self::TextDelta {
kind: AssistantBlockKind::Text,
delta,
});
}
if logprobs.is_some() || !token_ids.is_empty() {
events.push(Self::LogprobsDelta {
logprobs,
token_ids,
});
}
if let Some(finished) = finished {
events.push(Self::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
});
}
events
}
}
}
}
/// Boxed stream of decoded text events coming from [`vllm_text`].
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
+2 -2
View File
@@ -2,7 +2,7 @@
use std::sync::LazyLock;
pub use vllm_reasoning_parser::{
pub use vllm_parser::reasoning::{
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
@@ -34,7 +34,7 @@ pub mod names {
/// Constructor signature for one registered reasoning parser implementation.
type ReasoningParserCreator =
fn(DynTokenizer) -> vllm_reasoning_parser::Result<Box<dyn ReasoningParser>>;
fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>>;
/// Registry and model matcher for reasoning parsers.
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
+3 -4
View File
@@ -2,13 +2,12 @@
use std::sync::LazyLock;
pub use vllm_tool_parser::{
pub use vllm_parser::tool::{
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError,
ToolParserOutput,
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError,
};
use crate::parser::ParserFactory;
@@ -41,7 +40,7 @@ pub mod names {
}
/// Constructor signature for one registered tool parser implementation.
type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>;
type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>>;
/// Registry and model matcher for tool parsers.
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
+2 -2
View File
@@ -1,6 +1,6 @@
use vllm_tool_parser::Result;
use vllm_parser::tool::{Result, ToolParserOutput};
use super::{ToolParser, ToolParserFactory, ToolParserOutput, names};
use super::{ToolParser, ToolParserFactory, names};
use crate::Error;
use crate::request::ChatTool;
+1 -1
View File
@@ -4,9 +4,9 @@ use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
pub use vllm_parser::tool::Tool as ChatTool;
pub use vllm_text::SamplingParams;
use vllm_text::TextDecodeOptions;
pub use vllm_tool_parser::Tool as ChatTool;
use crate::AssistantMessageExt;
use crate::error::{Error, Result};
+1 -1
View File
@@ -1180,7 +1180,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
),
request_output(
"chat-tool",
bytes_to_token_ids(
bytes_with_special_stop_token(
b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>",
),
Some(EngineCoreFinishReason::Stop),