iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Introduce unified parser interface & combined parser (#46583)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-25 03:17:31 +00:00
committed by GitHub
parent efb5acffd5
commit 76c3c4ff63
74 changed files with 1204 additions and 1745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+2 -2
View File
@@ -129,8 +129,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
"rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
"rust/src/tool-parser/src/gemma4.rs", "rust/src/text/src/output/decoded.rs",
"rust/src/tokenizer/src/incremental.rs", "rust/src/reasoning-parser/src/tests.rs"]
"rust/src/parser/src/tool/gemma4.rs", "rust/src/text/src/output/decoded.rs",
"rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"]
ignore-hidden = false
[tool.typos.default]
rust/Cargo.lock
Generated
+14 -22
View File
@@ -5733,10 +5733,9 @@ dependencies = [
"uuid",
"vllm-engine-core-client",
"vllm-llm",
"vllm-reasoning-parser",
"vllm-parser",
"vllm-text",
"vllm-tokenizer",
"vllm-tool-parser",
"xgrammar-structural-tag",
"zeromq",
]
@@ -5872,11 +5871,22 @@ dependencies = [
]
[[package]]
name = "vllm-reasoning-parser"
name = "vllm-parser"
version = "0.1.0"
dependencies = [
"criterion",
"easy-ext",
"expect-test",
"futures",
"openai-protocol",
"serde",
"serde_json",
"thiserror 2.0.18",
"thiserror-ext",
"tool-parser",
"vllm-tokenizer",
"winnow",
"xgrammar-structural-tag",
]
[[package]]
@@ -5978,24 +5988,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "vllm-tool-parser"
version = "0.1.0"
dependencies = [
"criterion",
"easy-ext",
"expect-test",
"futures",
"openai-protocol",
"serde",
"serde_json",
"thiserror 2.0.18",
"thiserror-ext",
"tool-parser",
"winnow",
"xgrammar-structural-tag",
]
[[package]]
name = "vllm-tool-parser-py"
version = "0.1.0"
@@ -6004,7 +5996,7 @@ dependencies = [
"pythonize",
"serde_json",
"thiserror-ext",
"vllm-tool-parser",
"vllm-parser",
]
[[package]]
rust/Cargo.toml
+3 -5
View File
@@ -7,12 +7,11 @@ members = [
"src/managed-engine",
"src/metrics",
"src/mock-engine",
"src/reasoning-parser",
"src/parser",
"src/parser/python",
"src/server",
"src/text",
"src/tokenizer",
"src/tool-parser",
"src/tool-parser/python",
]
resolver = "3"
@@ -117,11 +116,10 @@ vllm-engine-core-client = { path = "src/engine-core-client" }
vllm-llm = { path = "src/llm" }
vllm-managed-engine = { path = "src/managed-engine" }
vllm-metrics = { path = "src/metrics" }
vllm-reasoning-parser = { path = "src/reasoning-parser" }
vllm-parser = { path = "src/parser" }
vllm-server = { path = "src/server" }
vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
vllm-tool-parser = { path = "src/tool-parser" }
winnow = "1.0.2"
xgrammar-structural-tag = "0.1.0"
zeromq = { version = "0.6.0", default-features = false, features = [
rust/src/chat/Cargo.toml
+1 -2
View File
@@ -30,10 +30,9 @@ trait-set.workspace = true
uuid.workspace = true
vllm-engine-core-client.workspace = true
vllm-llm.workspace = true
vllm-reasoning-parser.workspace = true
vllm-parser.workspace = true
vllm-text.workspace = true
vllm-tokenizer.workspace = true
vllm-tool-parser.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
rust/src/chat/src/output/default/mod.rs
+22 -43
View File
@@ -1,44 +1,34 @@
//! Default output processing pipeline.
mod reasoning;
mod structural_tag;
mod tool;
mod unified;
use std::sync::Once;
use futures::{Stream, StreamExt as _};
use futures::StreamExt as _;
use tracing::info;
use trait_set::trait_set;
use vllm_parser::unified::{CombinedParser, UnifiedParser};
use vllm_text::tokenizer::DynTokenizer;
use self::reasoning::reasoning_event_stream;
use self::structural_tag::apply_structural_tag_constraint;
use self::tool::tool_event_stream;
use self::unified::unified_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
use crate::output::{
AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream,
DynDecodedTextEventStream,
};
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::request::ChatRequest;
use crate::{Error, Result as ChatResult};
trait_set! {
trait ContentEventStream = Stream<Item = Result<ContentEvent>> + Send + 'static;
}
/// Default request-scoped output processor used by Hugging Face style chat
/// backends.
///
/// This implementation assumes the backend already emitted decoded text deltas,
/// then optionally layers reasoning parsing and tool-call parsing before
/// then optionally layers unified reasoning and tool-call parsing before
/// assembling final structured chat events.
pub struct DefaultChatOutputProcessor {
reasoning_parser: Option<Box<dyn ReasoningParser>>,
tool_parser: Option<Box<dyn ToolParser>>,
parser: Box<dyn UnifiedParser>,
parallel_tool_calls: bool,
}
@@ -66,16 +56,17 @@ impl DefaultChatOutputProcessor {
} else {
None
};
let reasoning_parser = Self::resolve_optional_reasoning_parser(
request,
model_id,
tokenizer,
reasoning_parser,
)?;
let reasoning_parser =
Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
let parser: Box<dyn UnifiedParser> =
Box::new(CombinedParser::new(reasoning_parser, tool_parser));
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
Ok(Self {
reasoning_parser,
tool_parser,
parser,
parallel_tool_calls: request.parallel_tool_calls,
})
}
@@ -87,8 +78,7 @@ impl DefaultChatOutputProcessor {
/// content is treated as opaque text.
pub fn plain_text_only() -> Self {
Self {
reasoning_parser: None,
tool_parser: None,
parser: Box::new(CombinedParser::plain_text_only()),
parallel_tool_calls: true,
}
}
@@ -112,10 +102,6 @@ impl DefaultChatOutputProcessor {
let parser = factory.create(parser_name, &request.tools)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
apply_structural_tag_constraint(request, parser.as_ref())?;
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
@@ -123,7 +109,6 @@ impl DefaultChatOutputProcessor {
}
fn resolve_optional_reasoning_parser(
request: &mut ChatRequest,
model_id: &str,
tokenizer: DynTokenizer,
selection: &ParserSelection,
@@ -142,10 +127,6 @@ impl DefaultChatOutputProcessor {
let parser = factory.create(parser_name, tokenizer)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
}
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
Ok(Some(parser))
}
@@ -156,16 +137,14 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
impl ChatOutputProcessor for DefaultChatOutputProcessor {
/// Transforms a raw generate-output token stream into structured chat
/// events through three sequential stages once text decoding has
/// events through two sequential stages once text decoding has
/// already happened:
///
/// 1. [`reasoning_event_stream`] — reasoning/content separation
/// 2. [`tool_event_stream`] — tool-call parsing
/// 3. [`structured_chat_event_stream`] — final block assembly
/// 1. [`unified_event_stream`] — reasoning and tool-call parsing
/// 2. [`structured_chat_event_stream`] — final block assembly
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
let tool = tool_event_stream(reasoning, self.tool_parser);
let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
let parsed = unified_event_stream(decoded, self.parser);
let structured = structured_chat_event_stream(parsed, self.parallel_tool_calls);
Ok(structured.boxed())
}
rust/src/chat/src/output/default/reasoning.rs
-509
View File
@@ -1,509 +0,0 @@
//! Adapts decoded text updates into reasoning-aware assistant deltas.
//!
//! This stage sits between low-level token decoding and final block assembly.
//! It is the only place in the new pipeline that understands reasoning
//! separation: `decoded.rs` still only produces plain text deltas, while later
//! stages consume the semantic `Text` / `Reasoning` split emitted here.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_text::output::DecodedTextEvent;
use super::ContentEvent;
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::DecodedTextEventStream;
use crate::parser::reasoning::{ReasoningDelta, ReasoningParser};
/// Per-stream reasoning parsing state.
struct ReasoningState {
/// Reasoning parser for the current model family.
parser: Box<dyn ReasoningParser>,
/// Whether reasoning parsing has already failed for this stream.
parser_failed: bool,
}
impl ReasoningState {
/// Create one fresh reasoning-adaptation state for a new streamed response.
fn new(parser: Box<dyn ReasoningParser>) -> Self {
Self {
parser,
parser_failed: false,
}
}
/// Convert one decoded text delta into zero or more semantic assistant
/// deltas.
fn process_delta(&mut self, delta: String) -> Vec<ContentEvent> {
// If the parser has already failed, skip parsing and return plain text deltas.
if self.parser_failed {
return vec![ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta,
}];
}
let mut events = Vec::new();
match self.parser.push(&delta) {
Ok(result) => {
push_reasoning_delta(&mut events, result);
}
Err(error) => {
if !self.parser_failed {
warn!(
error = %error.as_report(),
"reasoning parser failed; falling back to plain text deltas"
);
self.parser_failed = true;
}
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
}
}
events
}
/// Initialize parser state once prompt token IDs are available.
fn initialize(&mut self, prompt_token_ids: &[u32]) {
if self.parser_failed {
return;
}
match self.parser.initialize(prompt_token_ids) {
Ok(()) => {}
Err(error) => {
warn!(
error = %error.as_report(),
"failed to initialize reasoning parser; falling back to plain text deltas"
);
self.parser_failed = true;
}
}
}
/// Flush any parser-held partial delimiter state at end of stream.
fn finish(&mut self) -> Vec<ContentEvent> {
if self.parser_failed {
return Vec::new();
}
match self.parser.finish() {
Ok(result) => {
let mut events = Vec::new();
push_reasoning_delta(&mut events, result);
events
}
Err(error) => {
warn!(error = %error.as_report(), "failed to flush reasoning parser state");
Vec::new()
}
}
}
}
/// Push one semantic text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<ContentEvent>, kind: AssistantBlockKind, delta: String) {
if delta.is_empty() {
return;
}
events.push(ContentEvent::TextDelta { kind, delta });
}
/// Convert one parsed reasoning delta into zero or more content events.
fn push_reasoning_delta(events: &mut Vec<ContentEvent>, delta: ReasoningDelta) {
if let Some(reasoning) = delta.reasoning {
push_text_delta(events, AssistantBlockKind::Reasoning, reasoning);
}
if let Some(content) = delta.content {
push_text_delta(events, AssistantBlockKind::Text, content);
}
}
/// Wrap one decoded-text stream into the internal reasoning event stream.
#[try_stream]
pub(crate) async fn reasoning_event_stream(
decoded_stream: impl DecodedTextEventStream,
reasoning_parser: Option<Box<dyn ReasoningParser>>,
mut y: TryYielder<ContentEvent, Error>,
) -> Result<()> {
pin_mut!(decoded_stream);
// Without a parser, pass through as plain text deltas.
let Some(reasoning_parser) = reasoning_parser else {
while let Some(event) = decoded_stream.next().await.transpose()? {
for next in ContentEvent::from_decoded_plain_text(event) {
y.yield_ok(next).await;
}
}
return Ok(());
};
let mut state = ReasoningState::new(reasoning_parser);
while let Some(event) = decoded_stream.next().await.transpose()? {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
state.initialize(&prompt_token_ids);
y.yield_ok(ContentEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
for next in state.process_delta(delta) {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
y.yield_ok(ContentEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
if let Some(finished) = finished {
for next in state.finish() {
y.yield_ok(next).await;
}
y.yield_ok(ContentEvent::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
.await;
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use futures::{StreamExt as _, stream};
use vllm_llm::FinishReason;
use vllm_text::output::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
};
use vllm_tokenizer::{DynTokenizer, Tokenizer};
use super::super::ContentEvent;
use super::reasoning_event_stream;
use crate::event::AssistantBlockKind;
use crate::parser::reasoning::{
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names,
};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
_ => None,
}
}
}
struct FailingReasoningParser {
fail_next: bool,
}
impl ReasoningParser for FailingReasoningParser {
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>, ReasoningError>
where
Self: Sized + 'static,
{
Ok(Box::new(Self { fail_next: true }))
}
fn push(&mut self, _text: &str) -> Result<ReasoningDelta, ReasoningError> {
if self.fail_next {
self.fail_next = false;
return Err(ReasoningError::MissingToken {
token: "<think>".to_string(),
});
}
Ok(ReasoningDelta::default())
}
}
fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box<dyn ReasoningParser> {
factory.register_parser::<FailingReasoningParser>("failing");
factory.create("failing", Arc::new(FakeTokenizer)).unwrap()
}
#[tokio::test]
async fn reasoning_parser_failure_falls_back_to_plain_text() {
let mut factory = ReasoningParserFactory::new();
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "abc".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "def".to_string(),
token_ids: vec![],
logprobs: None,
finished: Some(vllm_text::Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}),
]);
let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory)))
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
},
ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn reasoning_stream_preserves_logprobs_delta() {
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "abc".to_string(),
token_ids: vec![],
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
}),
finished: None,
}),
]);
let collected = reasoning_event_stream(events, None)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert_eq!(
collected,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
ContentEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
}),
token_ids: vec![],
},
]
);
}
#[tokio::test]
async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() {
let tokenizer = Arc::new(FakeTokenizer);
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "thought ".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "done</think>OK".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
]);
let factory = ReasoningParserFactory::new();
let collected = reasoning_event_stream(
events,
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![2].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "thought ".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "done</think>OK".to_string(),
},
]
);
}
#[tokio::test]
async fn qwen3_parser_tolerates_prompt_prefill_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let events = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "thought ".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "done</think>OK".to_string(),
token_ids: vec![],
logprobs: None,
finished: None,
}),
]);
let factory = ReasoningParserFactory::new();
let collected = reasoning_event_stream(
events,
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("reasoning stream should not fail");
assert_eq!(
events,
vec![
ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thought ".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "done".to_string(),
},
ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "OK".to_string(),
},
]
);
}
}
rust/src/chat/src/output/default/structural_tag.rs
+1 -1
View File
@@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
mod tests {
use serde_json::{Value, json};
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
use vllm_parser::tool::{Qwen3CoderToolParser, Tool};
use super::*;
rust/src/chat/src/output/default/tool.rs
-985
View File
@@ -1,985 +0,0 @@
//! Adapts plain assistant text deltas into tool-call-aware assistant updates.
//!
//! This stage runs after reasoning separation and before final block assembly.
//! It only inspects normal assistant text, leaves reasoning deltas untouched,
//! and translates incremental tool parsing output into internal tool-call
//! events while preserving plain-text fallback behavior.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use super::{AssistantEvent, ContentEvent, ContentEventStream};
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::generate_tool_call_id;
use crate::parser::tool::{ToolCallDelta, ToolParser, ToolParserOutput};
/// Per-stream tool parsing state.
struct ToolState {
/// Parser for the current model family.
parser: Box<dyn ToolParser>,
/// Whether tool parsing has already failed for this stream.
parser_failed: bool,
/// The parser-local index of the currently open tool call, if any.
// NOTE: We only allow single open tool call at a time right now, since that's what all
// supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple
// interleaved calls in the future.
open_call_index: Option<usize>,
}
impl ToolState {
/// Create one fresh tool-parsing state for a new streamed response.
fn new(parser: Box<dyn ToolParser>) -> Self {
Self {
parser,
parser_failed: false,
open_call_index: None,
}
}
/// Convert one semantic assistant text delta into zero or more tool-aware
/// internal events.
fn process_text_delta(
&mut self,
kind: AssistantBlockKind,
delta: String,
) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
// Only normal assistant text is eligible for tool parsing. Reasoning
// blocks and plain-text fallback should pass through unchanged.
if kind != AssistantBlockKind::Text || self.parser_failed {
self.open_call_index = None;
events.push(AssistantEvent::TextDelta { kind, delta });
return Ok(events);
}
let mut output = ToolParserOutput::default();
let parse_result = self.parser.parse_into(&delta, &mut output);
match parse_result {
Ok(()) => self.process_parser_output(kind, output, &mut events)?,
Err(error) => {
warn!(
error = %error.as_report(),
"tool parser failed; falling back to plain text deltas"
);
// Permanently mark this parser as failed.
// TODO: we may consider recovering from parsing errors in the future.
self.parser_failed = true;
// On parsing failure, we still apply the partial parser output if any, but we close
// any open tool calls and emit the remaining buffered text as a plain-text delta to
// preserve as much of the output as possible.
self.process_parser_output(kind, output, &mut events)?;
self.open_call_index = None;
push_text_delta(&mut events, kind, self.parser.reset());
}
}
Ok(events)
}
/// Apply one parsed tool output to the current stream state.
fn process_parser_output(
&mut self,
kind: AssistantBlockKind,
output: ToolParserOutput,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
// When we are not currently streaming a tool call, preserve plain
// text first and then surface any new tool call items.
if self.open_call_index.is_none() {
push_text_delta(events, kind, output.normal_text);
self.process_tool_items(output.calls, events)?;
} else {
// Once a tool call is open, prioritize tool deltas first. If the
// parser emits normal text again, close the tool call and resume
// plain text output.
self.process_tool_items(output.calls, events)?;
if !output.normal_text.is_empty() {
self.open_call_index = None;
push_text_delta(events, kind, output.normal_text);
}
}
Ok(())
}
/// Apply one batch of parsed tool-call deltas emitted by the parser.
fn process_tool_items(
&mut self,
items: Vec<ToolCallDelta>,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
for item in items {
if let Some(name) = item.name {
let is_new_tool = match self.open_call_index {
Some(open_call_index) => open_call_index != item.tool_index,
None => true,
};
if is_new_tool {
let id = self
.parser
.tool_call_id(item.tool_index)
.map(str::to_string)
.unwrap_or_else(generate_tool_call_id);
self.open_call_index = Some(item.tool_index);
events.push(AssistantEvent::ToolCallStart { id, name });
}
}
if item.arguments.is_empty() {
// No arguments delta to apply.
continue;
}
let Some(open_call_index) = self.open_call_index else {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} before any tool-call start",
item.tool_index
),
});
};
if open_call_index != item.tool_index {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} while tool index {} is open",
item.tool_index, open_call_index
),
});
}
events.push(AssistantEvent::ToolCallArgumentsDelta {
delta: item.arguments,
});
}
Ok(())
}
/// Flush parser state at end-of-stream and close any remaining open calls.
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
if self.parser_failed {
return Ok(events);
}
match self.parser.finish() {
Ok(output) => {
self.process_parser_output(AssistantBlockKind::Text, output, &mut events)?
}
Err(error) => {
warn!(
error = %error.as_report(),
"tool parser finish failed; closing open tool calls with buffered state"
);
self.parser_failed = true;
}
}
Ok(events)
}
}
/// Push one plain-text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
if delta.is_empty() {
return;
}
events.push(AssistantEvent::TextDelta { kind, delta });
}
/// Wrap one semantic assistant stream into the internal tool-aware assistant
/// stream.
#[try_stream]
pub(crate) async fn tool_event_stream(
stream: impl ContentEventStream,
parser: Option<Box<dyn ToolParser>>,
mut y: TryYielder<AssistantEvent, Error>,
) -> Result<()> {
// Without a parser, pass through the input stream unchanged.
let Some(parser) = parser else {
pin_mut!(stream);
while let Some(event) = stream.next().await.transpose()? {
y.yield_ok(event.into()).await;
}
return Ok(());
};
pin_mut!(stream);
let mut state = ToolState::new(parser);
while let Some(event) = stream.next().await.transpose()? {
match event {
ContentEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
y.yield_ok(AssistantEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
ContentEvent::TextDelta { kind, delta } => {
for next in state.process_text_delta(kind, delta)? {
y.yield_ok(next).await;
}
}
ContentEvent::LogprobsDelta {
logprobs,
token_ids,
} => {
y.yield_ok(AssistantEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
ContentEvent::Done {
usage,
finish_reason,
kv_transfer_params,
} => {
for next in state.finish()? {
y.yield_ok(next).await;
}
y.yield_ok(AssistantEvent::Done {
usage,
finish_reason,
kv_transfer_params,
})
.await;
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use futures::{StreamExt as _, stream};
use vllm_llm::FinishReason;
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use vllm_tool_parser::Result;
use super::super::{AssistantEvent, ContentEvent};
use super::tool_event_stream;
use crate::error::Error;
use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
use crate::output::structured::structured_chat_event_stream;
use crate::parser::tool::{
DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput,
};
use crate::request::ChatTool;
use crate::stream::{ChatEventStream, CollectedAssistantMessage};
struct FailingParser {
fail_next: bool,
buffered: String,
}
struct ScriptedParser {
push_outputs: Vec<ToolParserOutput>,
finish_output: ToolParserOutput,
}
struct PartialThenFailParser {
buffered: String,
}
struct IdScriptedParser {
output: ToolParserOutput,
tool_call_id: Option<String>,
}
impl ToolParser for FailingParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
fail_next: false,
buffered: String::new(),
}))
}
fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
self.buffered.push_str(chunk);
if self.fail_next {
self.fail_next = false;
return Err(ToolParserError::ParsingFailed {
message: "boom".to_string(),
});
}
self.buffered.clear();
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
impl ToolParser for ScriptedParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
push_outputs: Vec::new(),
finish_output: ToolParserOutput::default(),
}))
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
let mut next = self.push_outputs.pop().unwrap_or_default();
output.normal_text.push_str(&next.normal_text);
output.calls.append(&mut next.calls);
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(std::mem::take(&mut self.finish_output))
}
fn reset(&mut self) -> String {
String::new()
}
}
impl ToolParser for IdScriptedParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
output: ToolParserOutput::default(),
tool_call_id: None,
}))
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
(tool_index == 0).then_some(self.tool_call_id.as_deref()).flatten()
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
output.append(std::mem::take(&mut self.output));
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
String::new()
}
}
impl ToolParser for PartialThenFailParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
buffered: String::new(),
}))
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
output.calls.extend([
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: String::new(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: r#"{"location":"SF"}"#.to_string(),
},
]);
self.buffered.push_str(" trailing text");
Err(ToolParserError::ParsingFailed {
message: "boom".to_string(),
})
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
fn deepseek_v4_test_tools() -> Vec<ChatTool> {
vec![
ChatTool {
name: "get_weather".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"location": { "type": "string" }
}
}),
strict: None,
},
ChatTool {
name: "add".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"x": { "type": "integer" },
"y": { "type": "integer" }
}
}),
strict: None,
},
]
}
async fn collect_deepseek_v4_message(chunks: Vec<String>) -> CollectedAssistantMessage {
let events = chunks
.into_iter()
.map(|delta| {
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta,
})
})
.chain(std::iter::once(Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})));
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
let chat_events = structured_chat_event_stream(assistant_events, true);
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
.collect_message()
.await
.unwrap()
}
fn message_tool_projection(
message: &CollectedAssistantMessage,
) -> (String, Vec<(String, serde_json::Value)>) {
(
message.message.text(),
message
.message
.tool_calls()
.map(|call| {
(
call.name.clone(),
serde_json::from_str(&call.arguments).unwrap(),
)
})
.collect(),
)
}
#[tokio::test]
async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(
events,
Some(Box::new(PartialThenFailParser {
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { name, .. } if name == "get_weather"
));
assert!(matches!(
&events[1],
AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"#
));
assert_eq!(
events[2],
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: " trailing text".to_string(),
}
);
assert!(matches!(events[3], AssistantEvent::Done { .. }));
}
#[tokio::test]
async fn tool_stream_preserves_parser_provided_tool_call_id() {
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
})]);
let parser = IdScriptedParser {
output: ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: "{}".to_string(),
}],
},
tool_call_id: Some("functions.get_weather:0".to_string()),
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { id, name }
if id == "functions.get_weather:0" && name == "get_weather"
));
}
#[tokio::test]
async fn tool_stream_generates_tool_call_id_when_parser_omits_one() {
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
})]);
let parser = IdScriptedParser {
output: ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: "{}".to_string(),
}],
},
tool_call_id: None,
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { id, name }
if id.starts_with("call_") && name == "get_weather"
));
}
#[tokio::test]
async fn real_buffered_parser_error_matches_streaming_and_non_streaming() {
let prefix = "I will check both.\n";
let first_tool_call = concat!(
"<｜DSML｜tool_calls>\n",
"<｜DSML｜invoke name=\"get_weather\">\n",
"<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo</｜DSML｜parameter>\n",
"</｜DSML｜invoke>",
);
let malformed_second_tool_call = concat!(
"\n<｜DSML｜invoke name=\"add\">\n",
"not a parameter\n",
"</｜DSML｜invoke>\n",
"</｜DSML｜tool_calls>",
);
let streaming_chunks = vec![
prefix.to_string(),
first_tool_call.to_string(),
malformed_second_tool_call.to_string(),
];
let full_output = streaming_chunks.concat();
let streaming = collect_deepseek_v4_message(streaming_chunks).await;
let non_streaming = collect_deepseek_v4_message(vec![full_output]).await;
let expected = (
format!("{prefix}{malformed_second_tool_call}"),
vec![(
"get_weather".to_string(),
serde_json::json!({ "location": "Tokyo" }),
)],
);
assert_eq!(message_tool_projection(&streaming), expected);
assert_eq!(message_tool_projection(&non_streaming), expected);
}
#[tokio::test]
async fn tool_parser_failure_falls_back_to_plain_text() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let collected = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: true,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.expect("tool stream should not fail");
assert_eq!(
events,
vec![
AssistantEvent::Start {
prompt_token_ids: vec![1, 2, 3].into(),
prompt_logprobs: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "abc".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "def".to_string(),
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
let message = ChatEventStream::new(
"req_fallback".to_string(),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
.expect("collect_message should succeed");
assert_eq!(message.message.text(), "abcdef");
assert!(message.message.tool_calls().next().is_none());
}
#[tokio::test]
async fn tool_stream_preserves_logprobs_delta() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
}],
}),
token_ids: vec![],
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: false,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert_eq!(
events,
vec![
AssistantEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
},
AssistantEvent::LogprobsDelta {
logprobs: Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
}],
}),
token_ids: vec![],
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn tool_stream_rejects_interleaved_tool_indices() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let parser = ScriptedParser {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: String::new(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 1,
name: None,
arguments: "{}".to_string(),
},
],
}],
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
}
#[tokio::test]
async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "start".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "text".to_string(),
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "args".to_string(),
}),
]);
let parser = ScriptedParser {
push_outputs: vec![
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: "}".to_string(),
}],
},
ToolParserOutput {
normal_text: "plain text".to_string(),
calls: Vec::new(),
},
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: "{".to_string(),
}],
},
],
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(
err,
Error::ToolCallStreamInvariant { message }
if message == "received arguments for tool index 0 before any tool-call start"
));
}
#[tokio::test]
async fn tool_stream_emits_start_and_args_for_terminal_text() {
let events = stream::iter(vec![
Ok(ContentEvent::Start {
prompt_token_ids: vec![1].into(),
prompt_logprobs: None,
}),
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let parser = ScriptedParser {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("first".to_string()),
arguments: r#"{"a":1}"#.to_string(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 1,
name: Some("second".to_string()),
arguments: r#"{"b":2}"#.to_string(),
},
],
}],
finish_output: ToolParserOutput::default(),
};
let events = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. }));
assert!(matches!(
events[2],
AssistantEvent::ToolCallArgumentsDelta { .. }
));
assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. }));
assert!(matches!(
events[4],
AssistantEvent::ToolCallArgumentsDelta { .. }
));
let collected = ChatEventStream::new(
"req_final_only".to_string(),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
.unwrap();
let tool_calls = collected.message.tool_calls().collect::<Vec<_>>();
assert_eq!(tool_calls.len(), 2);
assert_eq!(tool_calls[0].name, "first");
assert_eq!(tool_calls[1].name, "second");
}
}
rust/src/chat/src/output/default/unified.rs
+582
View File
@@ -0,0 +1,582 @@
//! Adapts decoded text updates into parsed assistant deltas.
//!
//! This stage sits between low-level token decoding and final block assembly.
//! It drives one unified parser that may emit normal text, reasoning text, or
//! tool-call deltas, then normalizes those parser events into internal
//! assistant events.
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
use vllm_text::output::DecodedTextEvent;
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
/// Per-stream unified parsing state.
struct UnifiedParserState {
/// Parser for the current request stream.
parser: Box<dyn UnifiedParser>,
/// Whether unified parsing has already failed for this stream.
parser_failed: bool,
/// The parser-local index of the currently open tool call, if any.
///
/// Supported parsers currently emit at most one active tool call at a time.
/// Change this to an indexed map if a model needs interleaved calls later.
open_call_index: Option<usize>,
}
impl UnifiedParserState {
/// Create one fresh unified parsing state for a new streamed response.
fn new(parser: Box<dyn UnifiedParser>) -> Self {
Self {
parser,
parser_failed: false,
open_call_index: None,
}
}
/// Initialize parser state once prompt token IDs are available.
fn initialize(&mut self, prompt_token_ids: &[u32]) {
if self.parser_failed {
return;
}
match self.parser.initialize(prompt_token_ids) {
Ok(()) => {}
Err(error) => {
warn!(
error = %error.as_report(),
"failed to initialize unified parser; falling back to plain text deltas"
);
self.parser_failed = true;
self.open_call_index = None;
}
}
}
/// Convert one decoded text delta into zero or more parsed assistant events.
fn process_delta(&mut self, delta: String) -> Result<Vec<AssistantEvent>> {
if self.parser_failed {
self.open_call_index = None;
return Ok(text_event(AssistantBlockKind::Text, delta).into_iter().collect());
}
let mut output = UnifiedParserOutput::default();
match self.parser.parse_into(&delta, &mut output) {
Ok(()) => {
let mut events = Vec::new();
self.process_parser_output(output, &mut events)?;
Ok(events)
}
Err(error) => {
warn!(
error = %error.as_report(),
"unified parser failed; falling back to plain text deltas"
);
self.parser_failed = true;
let mut events = Vec::new();
self.process_parser_output(output, &mut events)?;
self.open_call_index = None;
let recovered = self.parser.reset();
if recovered.is_empty() && events.is_empty() {
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
} else {
push_text_delta(&mut events, AssistantBlockKind::Text, recovered);
}
Ok(events)
}
}
}
/// Flush parser state at end-of-stream and close any remaining open calls.
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
let mut events = Vec::new();
if self.parser_failed {
return Ok(events);
}
match self.parser.finish() {
Ok(output) => self.process_parser_output(output, &mut events)?,
Err(error) => {
warn!(
error = %error.as_report(),
"unified parser finish failed; closing open parser state"
);
self.parser_failed = true;
self.open_call_index = None;
// TODO: should we reset and emit the buffered text?
}
}
Ok(events)
}
/// Apply one parsed unified output to the current stream state.
fn process_parser_output(
&mut self,
output: UnifiedParserOutput,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
for event in output.events {
match event {
UnifiedParserEvent::Text(delta) => {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Text, delta);
}
UnifiedParserEvent::Reasoning(delta) => {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Reasoning, delta);
}
UnifiedParserEvent::ToolCall(item) => {
self.process_tool_item(item, events)?;
}
}
}
Ok(())
}
/// Apply one parsed tool-call delta emitted by the parser.
fn process_tool_item(
&mut self,
item: vllm_parser::tool::ToolCallDelta,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
if let Some(name) = item.name {
let is_new_tool = match self.open_call_index {
Some(open_call_index) => open_call_index != item.tool_index,
None => true,
};
if is_new_tool {
let id = self
.parser
.tool_call_id(item.tool_index)
.map(str::to_string)
.unwrap_or_else(generate_tool_call_id);
self.open_call_index = Some(item.tool_index);
events.push(AssistantEvent::ToolCallStart { id, name });
}
}
if item.arguments.is_empty() {
return Ok(());
}
let Some(open_call_index) = self.open_call_index else {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} before any tool-call start",
item.tool_index
),
});
};
if open_call_index != item.tool_index {
return Err(Error::ToolCallStreamInvariant {
message: format!(
"received arguments for tool index {} while tool index {} is open",
item.tool_index, open_call_index
),
});
}
events.push(AssistantEvent::ToolCallArgumentsDelta {
delta: item.arguments,
});
Ok(())
}
}
/// Build one plain text event if `delta` is non-empty.
fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
if delta.is_empty() {
return None;
}
Some(AssistantEvent::TextDelta { kind, delta })
}
/// Push one plain text delta if it is non-empty.
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
if let Some(event) = text_event(kind, delta) {
events.push(event);
}
}
/// Wrap one decoded-text stream into the internal unified assistant stream.
#[try_stream]
pub(crate) async fn unified_event_stream(
decoded_stream: impl DecodedTextEventStream,
parser: Box<dyn UnifiedParser>,
mut y: TryYielder<AssistantEvent, Error>,
) -> Result<()> {
pin_mut!(decoded_stream);
let mut state = UnifiedParserState::new(parser);
while let Some(event) = decoded_stream.next().await.transpose()? {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => {
state.initialize(&prompt_token_ids);
y.yield_ok(AssistantEvent::Start {
prompt_token_ids,
prompt_logprobs,
})
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
for next in state.process_delta(delta)? {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
y.yield_ok(AssistantEvent::LogprobsDelta {
logprobs,
token_ids,
})
.await;
}
if let Some(finished) = finished {
for next in state.finish()? {
y.yield_ok(next).await;
}
y.yield_ok(AssistantEvent::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
.await;
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::VecDeque;
use futures::{StreamExt as _, stream};
use vllm_parser::reasoning::ReasoningError;
use vllm_parser::tool::ToolCallDelta;
use vllm_parser::unified::{UnifiedParserError, UnifiedParserOutput};
use super::unified_event_stream;
use crate::event::AssistantBlockKind;
use crate::output::AssistantEvent;
enum ScriptedStep {
Output(UnifiedParserOutput),
Error {
committed: UnifiedParserOutput,
reset_text: String,
},
}
struct ScriptedParser {
steps: VecDeque<ScriptedStep>,
reset_text: String,
tool_call_id: Option<String>,
finish_error_reset_text: Option<String>,
}
impl ScriptedParser {
fn new(steps: impl IntoIterator<Item = ScriptedStep>) -> Self {
Self {
steps: steps.into_iter().collect(),
reset_text: String::new(),
tool_call_id: Some("call_test".to_string()),
finish_error_reset_text: None,
}
}
fn with_finish_error(mut self, reset_text: &str) -> Self {
self.finish_error_reset_text = Some(reset_text.to_string());
self
}
}
impl vllm_parser::unified::UnifiedParser for ScriptedParser {
fn create(
_tools: &[vllm_parser::tool::Tool],
_tokenizer: vllm_tokenizer::DynTokenizer,
) -> vllm_parser::unified::Result<Box<dyn vllm_parser::unified::UnifiedParser>>
where
Self: Sized + 'static,
{
unreachable!("ScriptedParser is constructed directly in tests")
}
fn parse_into(
&mut self,
_delta: &str,
output: &mut UnifiedParserOutput,
) -> vllm_parser::unified::Result<()> {
match self.steps.pop_front().expect("unexpected parser call") {
ScriptedStep::Output(next) => {
output.append(next);
Ok(())
}
ScriptedStep::Error {
committed,
reset_text,
} => {
output.append(committed);
self.reset_text = reset_text;
Err(UnifiedParserError::Reasoning(
ReasoningError::MissingToken {
token: "<think>".to_string(),
},
))
}
}
}
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
self.tool_call_id.as_deref()
}
fn finish(&mut self) -> vllm_parser::unified::Result<UnifiedParserOutput> {
if let Some(reset_text) = self.finish_error_reset_text.take() {
self.reset_text = reset_text;
return Err(UnifiedParserError::Reasoning(
ReasoningError::MissingToken {
token: "</tool_call>".to_string(),
},
));
}
Ok(UnifiedParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.reset_text)
}
}
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: None,
}
}
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(vllm_text::output::Finished {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
}),
}
}
async fn collect(
parser: ScriptedParser,
events: Vec<vllm_text::output::DecodedTextEvent>,
) -> Vec<AssistantEvent> {
let stream = stream::iter(events.into_iter().map(Ok));
unified_event_stream(stream, Box::new(parser))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap()
}
fn text(delta: &str) -> UnifiedParserOutput {
let mut output = UnifiedParserOutput::default();
output.push_text(delta.to_string());
output
}
fn reasoning(delta: &str) -> UnifiedParserOutput {
let mut output = UnifiedParserOutput::default();
output.push_reasoning(delta.to_string());
output
}
fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
UnifiedParserOutput {
events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
ToolCallDelta {
tool_index: 0,
name: Some(name.to_string()),
arguments: arguments.to_string(),
},
)],
}
}
fn combined(first: UnifiedParserOutput, second: UnifiedParserOutput) -> UnifiedParserOutput {
let mut output = first;
output.append(second);
output
}
#[tokio::test]
async fn unified_stream_emits_reasoning_only_deltas() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(reasoning("thinking"))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
}]
);
}
#[tokio::test]
async fn unified_stream_emits_tool_only_deltas() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(tool_call(
"get_weather",
r#"{"location":"Paris"}"#,
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_emits_reasoning_followed_by_tool_call() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(combined(
reasoning("thinking"),
tool_call("get_weather", r#"{"location":"Paris"}"#),
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_emits_visible_text_followed_by_tool_call() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(combined(
text("visible "),
tool_call("get_weather", r#"{"location":"Paris"}"#),
))]),
vec![decoded_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "visible ".to_string(),
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
name: "get_weather".to_string(),
},
AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"location":"Paris"}"#.to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_fallback_keeps_committed_output_and_disables_later_parsing() {
let events = collect(
ScriptedParser::new([ScriptedStep::Error {
committed: text("committed"),
reset_text: "buffered".to_string(),
}]),
vec![decoded_delta("bad"), decoded_delta("later")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "committed".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "buffered".to_string(),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "later".to_string(),
},
]
);
}
#[tokio::test]
async fn unified_stream_finish_error_closes_parser_without_reset_text() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(UnifiedParserOutput::default())])
.with_finish_error("buffered"),
vec![finished_delta("")],
)
.await;
assert_eq!(
events,
vec![AssistantEvent::Done {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
}]
);
}
}
rust/src/chat/src/output/mod.rs
+2 -52
View File
@@ -2,7 +2,6 @@ use std::pin::Pin;
use std::sync::Arc;
use futures::Stream;
use subenum::subenum;
use trait_set::trait_set;
use uuid::Uuid;
use vllm_llm::TokenUsage;
@@ -22,23 +21,19 @@ pub(crate) use harmony::validate_harmony_parser_overrides;
/// Internal assistant event before final assembly.
///
/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content.
/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants.
#[subenum(ContentEvent)]
/// Unified parsing produces these events, and structured assembly consumes
/// them to build public chat events.
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum AssistantEvent {
#[subenum(ContentEvent)]
Start {
prompt_token_ids: Arc<[u32]>,
prompt_logprobs: Option<DecodedPromptLogprobs>,
},
#[subenum(ContentEvent)]
TextDelta {
kind: AssistantBlockKind,
delta: String,
},
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
#[subenum(ContentEvent)]
LogprobsDelta {
logprobs: Option<DecodedLogprobs>,
token_ids: Vec<u32>,
@@ -48,7 +43,6 @@ pub(crate) enum AssistantEvent {
/// A delta for the arguments of the currently open tool call. Must follow a
/// `ToolCallStart`.
ToolCallArgumentsDelta { delta: String },
#[subenum(ContentEvent)]
Done {
usage: TokenUsage,
finish_reason: FinishReason,
@@ -57,50 +51,6 @@ pub(crate) enum AssistantEvent {
},
}
impl ContentEvent {
/// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values
/// by treating all text as plain (non-reasoning) content.
fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec<Self> {
match event {
DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
} => vec![Self::Start {
prompt_token_ids,
prompt_logprobs,
}],
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
finished,
} => {
let mut events = Vec::new();
if !delta.is_empty() {
events.push(Self::TextDelta {
kind: AssistantBlockKind::Text,
delta,
});
}
if logprobs.is_some() || !token_ids.is_empty() {
events.push(Self::LogprobsDelta {
logprobs,
token_ids,
});
}
if let Some(finished) = finished {
events.push(Self::Done {
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
});
}
events
}
}
}
}
/// Boxed stream of decoded text events coming from [`vllm_text`].
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
rust/src/chat/src/parser/reasoning/mod.rs
+2 -2
View File
@@ -2,7 +2,7 @@
use std::sync::LazyLock;
pub use vllm_reasoning_parser::{
pub use vllm_parser::reasoning::{
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
@@ -34,7 +34,7 @@ pub mod names {
/// Constructor signature for one registered reasoning parser implementation.
type ReasoningParserCreator =
fn(DynTokenizer) -> vllm_reasoning_parser::Result<Box<dyn ReasoningParser>>;
fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>>;
/// Registry and model matcher for reasoning parsers.
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
rust/src/chat/src/parser/tool/mod.rs
+3 -4
View File
@@ -2,13 +2,12 @@
use std::sync::LazyLock;
pub use vllm_tool_parser::{
pub use vllm_parser::tool::{
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError,
ToolParserOutput,
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError,
};
use crate::parser::ParserFactory;
@@ -41,7 +40,7 @@ pub mod names {
}
/// Constructor signature for one registered tool parser implementation.
type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>;
type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>>;
/// Registry and model matcher for tool parsers.
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
rust/src/chat/src/parser/tool/tests.rs
+2 -2
View File
@@ -1,6 +1,6 @@
use vllm_tool_parser::Result;
use vllm_parser::tool::{Result, ToolParserOutput};
use super::{ToolParser, ToolParserFactory, ToolParserOutput, names};
use super::{ToolParser, ToolParserFactory, names};
use crate::Error;
use crate::request::ChatTool;
rust/src/chat/src/request.rs
+1 -1
View File
@@ -4,9 +4,9 @@ use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
pub use vllm_parser::tool::Tool as ChatTool;
pub use vllm_text::SamplingParams;
use vllm_text::TextDecodeOptions;
pub use vllm_tool_parser::Tool as ChatTool;
use crate::AssistantMessageExt;
use crate::error::{Error, Result};
rust/src/chat/tests/chat.rs
+1 -1
View File
@@ -1180,7 +1180,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
),
request_output(
"chat-tool",
bytes_to_token_ids(
bytes_with_special_stop_token(
b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>",
),
Some(EngineCoreFinishReason::Stop),
rust/src/tool-parser/Cargo.toml → rust/src/parser/Cargo.toml
+2 -1
View File
@@ -1,5 +1,5 @@
[package]
name = "vllm-tool-parser"
name = "vllm-parser"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -13,6 +13,7 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
vllm-tokenizer.workspace = true
winnow.workspace = true
xgrammar-structural-tag.workspace = true
rust/src/tool-parser/benches/deepseek_v3.rs → rust/src/parser/benches/deepseek_v3.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{DeepSeekV3ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/deepseek_v31.rs → rust/src/parser/benches/deepseek_v31.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{DeepSeekV31ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/deepseek_v32.rs → rust/src/parser/benches/deepseek_v32.rs
+2 -2
View File
@@ -1,8 +1,8 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{DeepSeekV32ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser};
mod utils;
use utils::feed_parser;
rust/src/tool-parser/benches/gemma4.rs → rust/src/parser/benches/gemma4.rs
+2 -2
View File
@@ -1,8 +1,8 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{Gemma4ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser};
mod utils;
use utils::feed_parser;
rust/src/tool-parser/benches/glm45_moe.rs → rust/src/parser/benches/glm45_moe.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/kimi_k2.rs → rust/src/parser/benches/kimi_k2.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{KimiK2ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/llama3_json.rs → rust/src/parser/benches/llama3_json.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{Llama3JsonToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/minimax_m2.rs → rust/src/parser/benches/minimax_m2.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{MinimaxM2ToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/qwen3_coder.rs → rust/src/parser/benches/qwen3_coder.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{Qwen3CoderToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/qwen3_xml.rs → rust/src/parser/benches/qwen3_xml.rs
+2 -2
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::QwenParser as ExternalQwenParser;
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
use vllm_tool_parser::{Qwen3XmlToolParser, Tool, ToolParser};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/tool-parser/benches/utils/mod.rs → rust/src/parser/benches/utils/mod.rs
+2 -2
View File
@@ -3,8 +3,8 @@
use futures::FutureExt as _;
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
use tool_parser::traits::ToolParser as ExternalToolParser;
use vllm_tool_parser::test_utils::collect_stream;
use vllm_tool_parser::{Tool, ToolParser};
use vllm_parser::tool::test_utils::collect_stream;
use vllm_parser::tool::{Tool, ToolParser};
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
tools
rust/src/tool-parser/python/Cargo.toml → rust/src/parser/python/Cargo.toml
+1 -1
View File
@@ -13,7 +13,7 @@ pyo3.workspace = true
pythonize = { workspace = true, features = ["serde_json"] }
serde_json.workspace = true
thiserror-ext.workspace = true
vllm-tool-parser.workspace = true
vllm-parser.workspace = true
[lints]
workspace = true
rust/src/tool-parser/python/src/lib.rs → rust/src/parser/python/src/lib.rs
+3 -3
View File
@@ -1,4 +1,4 @@
//! Thin PyO3 bindings for `vllm_tool_parser`.
//! Thin PyO3 bindings for `vllm_parser::tool`.
//!
//! This crate exposes the Rust tool parser trait and data shapes to Python
//! while keeping parser state, grammar, and schema-aware argument conversion in
@@ -11,7 +11,7 @@ use pyo3::types::{PyAny, PyModule};
use pythonize::{depythonize, pythonize};
use serde_json::Value;
use thiserror_ext::AsReport as _;
use vllm_tool_parser::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
use vllm_parser::tool::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
macro_rules! tool_parser_factory {
($($parser:ident),+ $(,)?) => {
@@ -22,7 +22,7 @@ macro_rules! tool_parser_factory {
match name {
$(
stringify!($parser) => {
<vllm_tool_parser::$parser as ToolParser>::create(tools)
<vllm_parser::tool::$parser as ToolParser>::create(tools)
}
)+
_ => {
rust/src/parser/src/lib.rs
+5
View File
@@ -0,0 +1,5 @@
//! Streaming parsers for chat completions.
pub mod reasoning;
pub mod tool;
pub mod unified;
rust/src/reasoning-parser/src/cohere_cmd.rs → rust/src/parser/src/reasoning/cohere_cmd.rs
View File
rust/src/reasoning-parser/src/deepseek_r1.rs → rust/src/parser/src/reasoning/deepseek_r1.rs
View File
rust/src/reasoning-parser/src/delimited.rs → rust/src/parser/src/reasoning/delimited.rs
View File
rust/src/reasoning-parser/src/gemma4.rs → rust/src/parser/src/reasoning/gemma4.rs
+1 -1
View File
@@ -119,7 +119,7 @@ mod tests {
use vllm_tokenizer::Tokenizer;
use super::Gemma4ReasoningParser;
use crate::ReasoningParser;
use crate::reasoning::ReasoningParser;
struct FakeTokenizer;
rust/src/reasoning-parser/src/kimi.rs → rust/src/parser/src/reasoning/kimi.rs
View File
rust/src/reasoning-parser/src/minimax_m3.rs → rust/src/parser/src/reasoning/minimax_m3.rs
View File
rust/src/reasoning-parser/src/lib.rs → rust/src/parser/src/reasoning/mod.rs
View File
rust/src/reasoning-parser/src/qwen3.rs → rust/src/parser/src/reasoning/qwen3.rs
View File
rust/src/reasoning-parser/src/seed_oss.rs → rust/src/parser/src/reasoning/seed_oss.rs
+1 -1
View File
@@ -49,7 +49,7 @@ mod tests {
use std::sync::Arc;
use super::SeedOssReasoningParser;
use crate::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
#[test]
fn without_prompt_markers_expects_start_token() {
rust/src/reasoning-parser/src/step3p5.rs → rust/src/parser/src/reasoning/step3p5.rs
+1 -1
View File
@@ -127,7 +127,7 @@ mod tests {
use std::sync::Arc;
use super::Step3p5ReasoningParser;
use crate::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
#[test]
fn picks_up_prompt_start_boundary() {
rust/src/reasoning-parser/src/tests.rs → rust/src/parser/src/reasoning/tests.rs
View File
rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs → rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3.2 models.
///
@@ -67,8 +67,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::DeepSeekV32ToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs → rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V4 models.
///
@@ -69,8 +69,8 @@ mod tests {
use serde_json::{Value, json};
use super::DeepSeekV4ToolParser;
use crate::test_utils::{collect_stream, test_tools};
use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, test_tools};
use crate::tool::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/deepseek_dsml/mod.rs → rust/src/parser/src/tool/deepseek_dsml/mod.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParserOutput};
use crate::Tool;
use crate::tool::Tool;
mod deepseek_v32;
mod deepseek_v4;
rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs → rust/src/parser/src/tool/deepseek_json/deepseek_v3.rs
+4 -4
View File
@@ -1,5 +1,5 @@
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
///
@@ -55,12 +55,12 @@ mod tests {
use thiserror_ext::AsReport;
use super::DeepSeekV3ToolParser;
use crate::deepseek_json::{
use crate::tool::deepseek_json::{
TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, V3_ARGUMENT_END,
V3_JSON_START,
};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn v3_tool_call(function_name: &str, arguments: &str) -> String {
format!(
rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs → rust/src/parser/src/tool/deepseek_json/deepseek_v31.rs
+4 -4
View File
@@ -1,5 +1,5 @@
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
///
@@ -51,11 +51,11 @@ mod tests {
use thiserror_ext::AsReport;
use super::DeepSeekV31ToolParser;
use crate::deepseek_json::{
use crate::tool::deepseek_json::{
TOOL_CALL_END, TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START,
};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn v31_tool_call(function_name: &str, arguments: &str) -> String {
format!("{TOOL_CALL_START}{function_name}{TOOL_CALL_SEPARATOR}{arguments}{TOOL_CALL_END}")
rust/src/tool-parser/src/deepseek_json/mod.rs → rust/src/parser/src/tool/deepseek_json/mod.rs
View File
rust/src/tool-parser/src/error.rs → rust/src/parser/src/tool/error.rs
+1 -1
View File
@@ -6,7 +6,7 @@ pub type Result<T> = std::result::Result<T, ToolParserError>;
/// Errors produced while creating or running tool parsers.
#[derive(Debug, Error, Macro)]
#[thiserror_ext(macro(path = "crate::error"))]
#[thiserror_ext(macro(path = "crate::tool::error"))]
pub enum ToolParserError {
#[error("tool parser parsing failed: {message}")]
ParsingFailed { message: String },
rust/src/tool-parser/src/gemma4.rs → rust/src/parser/src/tool/gemma4.rs
+2 -2
View File
@@ -8,7 +8,7 @@ use winnow::token::{literal, take_till, take_until};
use super::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_text_len};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::tool::Tool;
const TOOL_CALL_START: &str = "<|tool_call>";
const TOOL_CALL_END: &str = "<tool_call|>";
@@ -428,7 +428,7 @@ mod tests {
Gemma4ToolParser, ToolCallDelta, ToolParser, ToolParserOutput, gemma4_array_content,
parse_gemma4_args,
};
use crate::{Tool, ToolParserTestExt as _};
use crate::tool::{Tool, ToolParserTestExt as _};
fn parse_gemma4_array(array: &str) -> super::Result<Vec<Value>> {
let mut input = array;
rust/src/tool-parser/src/glm_xml/glm45_moe.rs → rust/src/parser/src/tool/glm_xml/glm45_moe.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use super::{GlmXmlToolParser, Separator};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
/// Tool parser for GLM-4.5/4.6 MoE XML-style tool calls.
///
rust/src/tool-parser/src/glm_xml/glm47_moe.rs → rust/src/parser/src/tool/glm_xml/glm47_moe.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{GlmXmlToolParser, Separator};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
///
@@ -44,8 +44,8 @@ mod tests {
use serde_json::{Value, json};
use super::Glm47MoeToolParser;
use crate::ToolParserTestExt as _;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::ToolParserTestExt as _;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
fn glm47_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/glm_xml/mod.rs → rust/src/parser/src/tool/glm_xml/mod.rs
+3 -3
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until, take_while};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParserOutput};
use crate::Tool;
use crate::tool::Tool;
mod glm45_moe;
mod glm47_moe;
@@ -264,8 +264,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Glm45MoeToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserTestExt as _};
fn glm45_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/hy_v3.rs → rust/src/parser/src/tool/hy_v3.rs
+3 -3
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::{StructuralTagModel, Tool};
use crate::tool::{StructuralTagModel, Tool};
const TOOL_CALLS_START: &str = "<tool_calls>";
const TOOL_CALLS_END: &str = "</tool_calls>";
@@ -249,8 +249,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::{HyV3ToolParser, ToolParser};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/json/granite4.rs → rust/src/parser/src/tool/json/granite4.rs
+4 -4
View File
@@ -8,10 +8,10 @@ use super::{
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
tool_call_header_event,
};
use crate::utils::{
use crate::tool::utils::{
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
};
use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::tool::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
const TOOL_CALL_START: &str = "<tool_call>";
const TOOL_CALL_END: &str = "</tool_call>";
@@ -279,8 +279,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Granite4ToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
#[test]
fn granite4_parse_complete_without_tool_call_keeps_text() {
rust/src/tool-parser/src/json/hermes.rs → rust/src/parser/src/tool/json/hermes.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Hermes",
@@ -68,8 +68,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::HermesToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, arguments: &str) -> String {
format!(r#"<tool_call>{{"name":"{function_name}","arguments":{arguments}}}</tool_call>"#)
rust/src/tool-parser/src/json/internlm2.rs → rust/src/parser/src/tool/json/internlm2.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "InternLM2",
@@ -123,8 +123,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Internlm2ToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
const ACTION_START: &str = "<|action_start|><|plugin|>";
const ACTION_END: &str = "<|action_end|>";
rust/src/tool-parser/src/json/llama.rs → rust/src/parser/src/tool/json/llama.rs
+4 -4
View File
@@ -8,8 +8,8 @@ use super::{
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
argument_delta_event, tool_call_header_event,
};
use crate::utils::{JsonObjectScanState, parse_buffered_event};
use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::tool::utils::{JsonObjectScanState, parse_buffered_event};
use crate::tool::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
#[derive(Debug, Clone, PartialEq, Eq)]
enum LlamaJsonMode {
@@ -256,8 +256,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Llama3JsonToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, parameters: &str) -> String {
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
rust/src/tool-parser/src/json/mistral.rs → rust/src/parser/src/tool/json/mistral.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Mistral",
@@ -61,8 +61,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::MistralToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, arguments: &str) -> String {
format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#)
rust/src/tool-parser/src/json/mod.rs → rust/src/parser/src/tool/json/mod.rs
+1 -1
View File
@@ -374,7 +374,7 @@ mod tests {
use expect_test::expect;
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::ToolParserOutput;
use crate::tool::ToolParserOutput;
const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Delimited JSON",
rust/src/tool-parser/src/json/phi4mini.rs → rust/src/parser/src/tool/json/phi4mini.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Phi4Mini",
@@ -69,8 +69,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Phi4MiniJsonToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserTestExt as _};
/// Build one phi-4-mini tool-call object: `{"name":..,"<args_key>":<args>}`.
fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String {
rust/src/tool-parser/src/json/qwen.rs → rust/src/parser/src/tool/json/qwen.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Qwen XML",
@@ -70,8 +70,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::Qwen3XmlToolParser;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, arguments: &str) -> String {
format!(
rust/src/tool-parser/src/kimi_k2.rs → rust/src/parser/src/tool/kimi_k2.rs
+3 -3
View File
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::{StructuralTagModel, Tool};
use crate::tool::{StructuralTagModel, Tool};
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
@@ -339,8 +339,8 @@ mod tests {
KimiK2ToolParser, TOOL_CALL_ARGUMENT_START, TOOL_CALL_END, TOOL_CALL_START, TOOL_CALLS_END,
TOOL_CALLS_START, ToolParser, tool_header,
};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, index: usize, arguments: &str) -> String {
format!(
rust/src/tool-parser/src/minimax_m2.rs → rust/src/parser/src/tool/minimax_m2.rs
+3 -3
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::{StructuralTagModel, Tool};
use crate::tool::{StructuralTagModel, Tool};
const TOOL_CALL_START: &str = "<minimax:tool_call>";
const TOOL_CALL_END: &str = "</minimax:tool_call>";
@@ -272,8 +272,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
use crate::ToolParserTestExt as _;
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::ToolParserTestExt as _;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
let invokes = invokes
rust/src/tool-parser/src/minimax_m3.rs → rust/src/parser/src/tool/minimax_m3.rs
+3 -3
View File
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::{ParamElement, ParamInput, ToolSchemas};
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::tool::Tool;
const NAMESPACE: &str = "]<]minimax[>[";
const TOOL_CALL_START: &str = "]<]minimax[>[<tool_call>";
@@ -388,8 +388,8 @@ mod tests {
ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser,
TOOL_CALL_END, TOOL_CALL_START, ToolParser,
};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{Tool, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{Tool, ToolParserTestExt as _};
fn element(name: &str, body: &str) -> String {
format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>")
rust/src/tool-parser/src/lib.rs → rust/src/parser/src/tool/mod.rs
+3 -3
View File
@@ -1,9 +1,9 @@
//! Streaming tool parsers for chat completions.
#[macro_use]
mod error;
pub(crate) mod error;
mod deepseek_dsml;
mod deepseek_json;
pub(crate) mod deepseek_json;
mod gemma4;
mod glm_xml;
mod hy_v3;
@@ -15,7 +15,7 @@ mod parameters;
mod qwen_coder;
#[cfg(any(test, feature = "test-util"))]
pub mod test_utils;
mod utils;
pub(crate) mod utils;
use std::collections::{BTreeMap, btree_map};
rust/src/tool-parser/src/parameters.rs → rust/src/parser/src/tool/parameters.rs
+2 -2
View File
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use serde_json::{Map, Number, Value};
use crate::Tool;
use crate::tool::Tool;
/// Normalized parameter schemas for all tools in one request.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
@@ -416,7 +416,7 @@ mod tests {
use serde_json::{Value, json};
use super::{ParamElement, ParamInput, ToolSchema, ToolSchemas};
use crate::Tool;
use crate::tool::Tool;
fn test_tool(name: &str, parameters: serde_json::Value) -> Tool {
Tool {
rust/src/tool-parser/src/qwen_coder.rs → rust/src/parser/src/tool/qwen_coder.rs
+3 -3
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::tool::Tool;
const TOOL_CALL_START: &str = "<tool_call>";
const TOOL_CALL_END: &str = "</tool_call>";
@@ -241,8 +241,8 @@ mod tests {
use thiserror_ext::AsReport;
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParserOutput, ToolParserTestExt as _};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
rust/src/tool-parser/src/test_utils.rs → rust/src/parser/src/tool/test_utils.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use serde_json::json;
use super::{ToolParser, ToolParserOutput};
use crate::{Tool, ToolParserTestExt as _};
use crate::tool::{Tool, ToolParserTestExt as _};
/// Build a reusable set of function tools for parser unit tests.
pub fn test_tools() -> Vec<Tool> {
rust/src/tool-parser/src/tests.rs → rust/src/parser/src/tool/tests.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use super::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::ToolParserTestExt as _;
use crate::tool::ToolParserTestExt as _;
struct DefaultParser;
rust/src/tool-parser/src/utils.rs → rust/src/parser/src/tool/utils.rs
View File
rust/src/parser/src/unified/combined.rs
+346
View File
@@ -0,0 +1,346 @@
//! Adapter that combines reasoning and tool parsers.
use vllm_tokenizer::DynTokenizer;
use crate::reasoning::ReasoningParser;
use crate::tool::{StructuralTagModel, Tool, ToolParser, ToolParserOutput};
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
/// Unified parser that composes existing reasoning and tool parsers.
pub struct CombinedParser {
reasoning: Option<Box<dyn ReasoningParser>>,
tool: Option<Box<dyn ToolParser>>,
}
impl CombinedParser {
/// Create a combined parser from optional reasoning and tool parsers.
pub fn new(
reasoning: Option<Box<dyn ReasoningParser>>,
tool: Option<Box<dyn ToolParser>>,
) -> Self {
Self { reasoning, tool }
}
/// Create a text-only combined parser.
pub fn plain_text_only() -> Self {
Self {
reasoning: None,
tool: None,
}
}
fn parse_tool(&mut self, content: &str, output: &mut UnifiedParserOutput) -> Result<()> {
let Some(tool) = self.tool.as_mut() else {
output.push_text(content.to_string());
return Ok(());
};
// Preserve any tool output that was already produced before the error.
let mut tool_output = ToolParserOutput::default();
let result = tool.parse_into(content, &mut tool_output);
output.append_tool_output(tool_output);
result?;
Ok(())
}
fn flush_tool(&mut self) -> Result<UnifiedParserOutput> {
let Some(tool) = self.tool.as_mut() else {
return Ok(UnifiedParserOutput::default());
};
let output = tool.finish()?;
let mut unified = UnifiedParserOutput::default();
unified.append_tool_output(output);
Ok(unified)
}
}
impl UnifiedParser for CombinedParser {
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static,
{
Err(UnifiedParserError::CombinedParserConstructor)
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
if let Some(reasoning) = self.reasoning.as_mut() {
reasoning.initialize(prompt_token_ids)?;
}
Ok(())
}
fn preserve_special_tokens(&self) -> bool {
self.reasoning.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
self.tool.as_ref().and_then(|parser| parser.structural_tag_model())
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
self.tool.as_ref().and_then(|parser| parser.tool_call_id(tool_index))
}
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
let Some(reasoning) = self.reasoning.as_mut() else {
return self.parse_tool(delta, output);
};
let reasoning_delta = reasoning.push(delta)?;
if let Some(reasoning) = reasoning_delta.reasoning {
output.push_reasoning(reasoning);
}
if let Some(content) = reasoning_delta.content {
self.parse_tool(&content, output)?;
}
Ok(())
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
let mut output = UnifiedParserOutput::default();
if let Some(reasoning) = self.reasoning.as_mut() {
let reasoning_delta = reasoning.finish()?;
if let Some(reasoning) = reasoning_delta.reasoning {
output.push_reasoning(reasoning);
}
if let Some(content) = reasoning_delta.content {
self.parse_tool(&content, &mut output)?;
}
}
output.append(self.flush_tool()?);
Ok(output)
}
fn reset(&mut self) -> String {
self.tool.as_mut().map_or_else(String::new, |parser| parser.reset())
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use super::CombinedParser;
use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
_ => None,
}
}
}
fn test_tools() -> Vec<Tool> {
vec![Tool {
name: "get_weather".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"location": { "type": "string" }
},
}),
strict: None,
}]
}
fn collect(parser: &mut dyn UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput {
let mut output = UnifiedParserOutput::default();
for chunk in chunks {
parser.parse_into(chunk, &mut output).unwrap();
}
output.append(parser.finish().unwrap());
output
}
struct PreserveReasoningParser;
impl ReasoningParser for PreserveReasoningParser {
fn create(
_tokenizer: vllm_tokenizer::DynTokenizer,
) -> crate::reasoning::Result<Box<dyn ReasoningParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self))
}
fn preserve_special_tokens(&self) -> bool {
true
}
fn push(&mut self, delta: &str) -> crate::reasoning::Result<ReasoningDelta> {
Ok(ReasoningDelta {
reasoning: None,
content: Some(delta.to_string()),
})
}
}
struct PreserveToolParser;
impl ToolParser for PreserveToolParser {
fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self))
}
fn preserve_special_tokens(&self) -> bool {
true
}
fn parse_into(
&mut self,
chunk: &str,
output: &mut crate::tool::ToolParserOutput,
) -> crate::tool::Result<()> {
output.normal_text.push_str(chunk);
Ok(())
}
fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
Ok(crate::tool::ToolParserOutput::default())
}
fn reset(&mut self) -> String {
String::new()
}
}
struct PartialThenErrorToolParser;
impl ToolParser for PartialThenErrorToolParser {
fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self))
}
fn parse_into(
&mut self,
_chunk: &str,
output: &mut crate::tool::ToolParserOutput,
) -> crate::tool::Result<()> {
output.normal_text.push_str("committed");
Err(crate::tool::ToolParserError::ParsingFailed {
message: "synthetic failure".to_string(),
})
}
fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
Ok(crate::tool::ToolParserOutput::default())
}
fn reset(&mut self) -> String {
String::new()
}
}
#[test]
fn combined_parser_emits_reasoning_and_text() {
let tokenizer = Arc::new(FakeTokenizer);
let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
let mut parser = CombinedParser::new(Some(reasoning), None);
let output = collect(&mut parser, &["<think>work</think>answer"]);
assert_eq!(
output.events,
vec![
UnifiedParserEvent::Reasoning("work".to_string()),
UnifiedParserEvent::Text("answer".to_string()),
]
);
}
#[test]
fn combined_parser_emits_tool_calls_from_visible_content() {
let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap();
let mut parser = CombinedParser::new(None, Some(tool));
assert!(matches!(
parser.structural_tag_model(),
Some(crate::tool::StructuralTagModel::Qwen3)
));
let output = collect(
&mut parser,
&[r#"<tool_call>
{"name":"get_weather","arguments":{"location":"Paris"}}
</tool_call>"#],
);
assert_eq!(
output.events,
vec![
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: String::new(),
}),
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: r#"{"location":"Paris"}"#.to_string(),
}),
]
);
}
#[test]
fn combined_parser_preserves_tool_output_on_parse_error() {
let mut parser = CombinedParser::new(None, Some(Box::new(PartialThenErrorToolParser)));
let mut output = UnifiedParserOutput::default();
let error = parser.parse_into("bad", &mut output).unwrap_err();
assert!(matches!(error, crate::unified::UnifiedParserError::Tool(_)));
assert_eq!(
output.events,
vec![UnifiedParserEvent::Text("committed".to_string())]
);
}
#[test]
fn combined_parser_preserves_special_tokens_when_either_inner_parser_needs_it() {
let mut parser = CombinedParser::new(Some(Box::new(PreserveReasoningParser)), None);
assert!(parser.preserve_special_tokens());
parser = CombinedParser::new(None, Some(Box::new(PreserveToolParser)));
assert!(parser.preserve_special_tokens());
}
}
rust/src/parser/src/unified/mod.rs
+114
View File
@@ -0,0 +1,114 @@
//! Unified parser interface for reasoning and tool-call deltas.
mod combined;
use thiserror::Error;
use vllm_tokenizer::DynTokenizer;
pub use combined::CombinedParser;
use crate::reasoning::ReasoningError;
use crate::tool::{StructuralTagModel, Tool, ToolCallDelta, ToolParserError, ToolParserOutput};
/// Result alias for unified parser operations.
pub type Result<T> = std::result::Result<T, UnifiedParserError>;
/// One parsed event emitted by a unified parser.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum UnifiedParserEvent {
/// Normal assistant-visible text.
Text(String),
/// Reasoning text hidden from the normal content stream.
Reasoning(String),
/// A tool-call update extracted from visible assistant text.
ToolCall(ToolCallDelta),
}
/// Result of advancing unified parsing with one assistant-text input.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UnifiedParserOutput {
/// Ordered parser events committed by this input.
pub events: Vec<UnifiedParserEvent>,
}
impl UnifiedParserOutput {
/// Append one visible text event if `delta` is non-empty.
pub fn push_text(&mut self, delta: String) {
if delta.is_empty() {
return;
}
self.events.push(UnifiedParserEvent::Text(delta));
}
/// Append one reasoning text event if `delta` is non-empty.
pub fn push_reasoning(&mut self, delta: String) {
if delta.is_empty() {
return;
}
self.events.push(UnifiedParserEvent::Reasoning(delta));
}
/// Append parsed tool parser output as unified events.
pub fn append_tool_output(&mut self, output: ToolParserOutput) {
// TODO: make ToolParserOutput carry ordered events and remove this text-first flattening.
self.push_text(output.normal_text);
self.events.extend(output.calls.into_iter().map(UnifiedParserEvent::ToolCall));
}
/// Append another parser output onto this one.
pub fn append(&mut self, mut other: Self) {
self.events.append(&mut other.events);
}
}
/// Incremental parser that extracts reasoning and tool-call events from assistant output.
pub trait UnifiedParser: Send {
/// Construct a boxed parser instance for one request stream.
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static;
/// Initialize parser state from prompt token IDs before output deltas arrive.
fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
Ok(())
}
/// Return whether decoded output must preserve tokenizer special tokens.
fn preserve_special_tokens(&self) -> bool {
false
}
/// Return the xgrammar structural-tag model used for strict tool calling.
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
None
}
/// Return the parser-provided ID for a tool call by index, if the model emitted one.
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
None
}
/// Feed one decoded text delta into the parser, appending committed output into `output`.
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()>;
/// Flush any buffered parser state at end of stream.
fn finish(&mut self) -> Result<UnifiedParserOutput> {
Ok(UnifiedParserOutput::default())
}
/// Clear parser state and return currently uncommitted buffered text.
fn reset(&mut self) -> String {
String::new()
}
}
/// Errors produced while creating or running unified parsers.
#[derive(Debug, Error)]
pub enum UnifiedParserError {
#[error("combined parser is constructed from split parser instances")]
CombinedParserConstructor,
#[error(transparent)]
Reasoning(#[from] ReasoningError),
#[error(transparent)]
Tool(#[from] ToolParserError),
}
rust/src/reasoning-parser/Cargo.toml
-12
View File
@@ -1,12 +0,0 @@
[package]
name = "vllm-reasoning-parser"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
thiserror.workspace = true
vllm-tokenizer.workspace = true
[lints]
workspace = true
tools/build_rust.py
+1 -1
View File
@@ -27,7 +27,7 @@ def rust_extensions(*, optional: bool = False) -> list[RustExtension]:
),
RustExtension(
target="vllm._rust_tool_parser",
path="rust/src/tool-parser/python/Cargo.toml",
path="rust/src/parser/python/Cargo.toml",
features=["pyo3/abi3-py38"],
binding=Binding.PyO3,
optional=optional,