[Rust Frontend] Introduce unified parser interface & combined parser (#46583)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+2
-2
@@ -129,8 +129,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
|
|||||||
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
|
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
|
||||||
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
|
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
|
||||||
"rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
|
"rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
|
||||||
"rust/src/tool-parser/src/gemma4.rs", "rust/src/text/src/output/decoded.rs",
|
"rust/src/parser/src/tool/gemma4.rs", "rust/src/text/src/output/decoded.rs",
|
||||||
"rust/src/tokenizer/src/incremental.rs", "rust/src/reasoning-parser/src/tests.rs"]
|
"rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"]
|
||||||
ignore-hidden = false
|
ignore-hidden = false
|
||||||
|
|
||||||
[tool.typos.default]
|
[tool.typos.default]
|
||||||
|
|||||||
Generated
+14
-22
@@ -5733,10 +5733,9 @@ dependencies = [
|
|||||||
"uuid",
|
"uuid",
|
||||||
"vllm-engine-core-client",
|
"vllm-engine-core-client",
|
||||||
"vllm-llm",
|
"vllm-llm",
|
||||||
"vllm-reasoning-parser",
|
"vllm-parser",
|
||||||
"vllm-text",
|
"vllm-text",
|
||||||
"vllm-tokenizer",
|
"vllm-tokenizer",
|
||||||
"vllm-tool-parser",
|
|
||||||
"xgrammar-structural-tag",
|
"xgrammar-structural-tag",
|
||||||
"zeromq",
|
"zeromq",
|
||||||
]
|
]
|
||||||
@@ -5872,11 +5871,22 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vllm-reasoning-parser"
|
name = "vllm-parser"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"criterion",
|
||||||
|
"easy-ext",
|
||||||
|
"expect-test",
|
||||||
|
"futures",
|
||||||
|
"openai-protocol",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
|
"thiserror-ext",
|
||||||
|
"tool-parser",
|
||||||
"vllm-tokenizer",
|
"vllm-tokenizer",
|
||||||
|
"winnow",
|
||||||
|
"xgrammar-structural-tag",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5978,24 +5988,6 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "vllm-tool-parser"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"criterion",
|
|
||||||
"easy-ext",
|
|
||||||
"expect-test",
|
|
||||||
"futures",
|
|
||||||
"openai-protocol",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"thiserror 2.0.18",
|
|
||||||
"thiserror-ext",
|
|
||||||
"tool-parser",
|
|
||||||
"winnow",
|
|
||||||
"xgrammar-structural-tag",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vllm-tool-parser-py"
|
name = "vllm-tool-parser-py"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -6004,7 +5996,7 @@ dependencies = [
|
|||||||
"pythonize",
|
"pythonize",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror-ext",
|
"thiserror-ext",
|
||||||
"vllm-tool-parser",
|
"vllm-parser",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+3
-5
@@ -7,12 +7,11 @@ members = [
|
|||||||
"src/managed-engine",
|
"src/managed-engine",
|
||||||
"src/metrics",
|
"src/metrics",
|
||||||
"src/mock-engine",
|
"src/mock-engine",
|
||||||
"src/reasoning-parser",
|
"src/parser",
|
||||||
|
"src/parser/python",
|
||||||
"src/server",
|
"src/server",
|
||||||
"src/text",
|
"src/text",
|
||||||
"src/tokenizer",
|
"src/tokenizer",
|
||||||
"src/tool-parser",
|
|
||||||
"src/tool-parser/python",
|
|
||||||
]
|
]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
|
|
||||||
@@ -117,11 +116,10 @@ vllm-engine-core-client = { path = "src/engine-core-client" }
|
|||||||
vllm-llm = { path = "src/llm" }
|
vllm-llm = { path = "src/llm" }
|
||||||
vllm-managed-engine = { path = "src/managed-engine" }
|
vllm-managed-engine = { path = "src/managed-engine" }
|
||||||
vllm-metrics = { path = "src/metrics" }
|
vllm-metrics = { path = "src/metrics" }
|
||||||
vllm-reasoning-parser = { path = "src/reasoning-parser" }
|
vllm-parser = { path = "src/parser" }
|
||||||
vllm-server = { path = "src/server" }
|
vllm-server = { path = "src/server" }
|
||||||
vllm-text = { path = "src/text" }
|
vllm-text = { path = "src/text" }
|
||||||
vllm-tokenizer = { path = "src/tokenizer" }
|
vllm-tokenizer = { path = "src/tokenizer" }
|
||||||
vllm-tool-parser = { path = "src/tool-parser" }
|
|
||||||
winnow = "1.0.2"
|
winnow = "1.0.2"
|
||||||
xgrammar-structural-tag = "0.1.0"
|
xgrammar-structural-tag = "0.1.0"
|
||||||
zeromq = { version = "0.6.0", default-features = false, features = [
|
zeromq = { version = "0.6.0", default-features = false, features = [
|
||||||
|
|||||||
@@ -30,10 +30,9 @@ trait-set.workspace = true
|
|||||||
uuid.workspace = true
|
uuid.workspace = true
|
||||||
vllm-engine-core-client.workspace = true
|
vllm-engine-core-client.workspace = true
|
||||||
vllm-llm.workspace = true
|
vllm-llm.workspace = true
|
||||||
vllm-reasoning-parser.workspace = true
|
vllm-parser.workspace = true
|
||||||
vllm-text.workspace = true
|
vllm-text.workspace = true
|
||||||
vllm-tokenizer.workspace = true
|
vllm-tokenizer.workspace = true
|
||||||
vllm-tool-parser.workspace = true
|
|
||||||
xgrammar-structural-tag.workspace = true
|
xgrammar-structural-tag.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|||||||
@@ -1,44 +1,34 @@
|
|||||||
//! Default output processing pipeline.
|
//! Default output processing pipeline.
|
||||||
|
|
||||||
mod reasoning;
|
|
||||||
mod structural_tag;
|
mod structural_tag;
|
||||||
mod tool;
|
mod unified;
|
||||||
|
|
||||||
use std::sync::Once;
|
use std::sync::Once;
|
||||||
|
|
||||||
use futures::{Stream, StreamExt as _};
|
use futures::StreamExt as _;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
use trait_set::trait_set;
|
use vllm_parser::unified::{CombinedParser, UnifiedParser};
|
||||||
use vllm_text::tokenizer::DynTokenizer;
|
use vllm_text::tokenizer::DynTokenizer;
|
||||||
|
|
||||||
use self::reasoning::reasoning_event_stream;
|
|
||||||
use self::structural_tag::apply_structural_tag_constraint;
|
use self::structural_tag::apply_structural_tag_constraint;
|
||||||
use self::tool::tool_event_stream;
|
use self::unified::unified_event_stream;
|
||||||
use super::structured::structured_chat_event_stream;
|
use super::structured::structured_chat_event_stream;
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
use crate::output::{
|
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
|
||||||
AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream,
|
|
||||||
DynDecodedTextEventStream,
|
|
||||||
};
|
|
||||||
use crate::parser::ParserSelection;
|
use crate::parser::ParserSelection;
|
||||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||||
use crate::request::ChatRequest;
|
use crate::request::ChatRequest;
|
||||||
use crate::{Error, Result as ChatResult};
|
use crate::{Error, Result as ChatResult};
|
||||||
|
|
||||||
trait_set! {
|
|
||||||
trait ContentEventStream = Stream<Item = Result<ContentEvent>> + Send + 'static;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Default request-scoped output processor used by Hugging Face style chat
|
/// Default request-scoped output processor used by Hugging Face style chat
|
||||||
/// backends.
|
/// backends.
|
||||||
///
|
///
|
||||||
/// This implementation assumes the backend already emitted decoded text deltas,
|
/// This implementation assumes the backend already emitted decoded text deltas,
|
||||||
/// then optionally layers reasoning parsing and tool-call parsing before
|
/// then optionally layers unified reasoning and tool-call parsing before
|
||||||
/// assembling final structured chat events.
|
/// assembling final structured chat events.
|
||||||
pub struct DefaultChatOutputProcessor {
|
pub struct DefaultChatOutputProcessor {
|
||||||
reasoning_parser: Option<Box<dyn ReasoningParser>>,
|
parser: Box<dyn UnifiedParser>,
|
||||||
tool_parser: Option<Box<dyn ToolParser>>,
|
|
||||||
parallel_tool_calls: bool,
|
parallel_tool_calls: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,16 +56,17 @@ impl DefaultChatOutputProcessor {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let reasoning_parser = Self::resolve_optional_reasoning_parser(
|
let reasoning_parser =
|
||||||
request,
|
Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
|
||||||
model_id,
|
let parser: Box<dyn UnifiedParser> =
|
||||||
tokenizer,
|
Box::new(CombinedParser::new(reasoning_parser, tool_parser));
|
||||||
reasoning_parser,
|
|
||||||
)?;
|
if parser.preserve_special_tokens() {
|
||||||
|
request.decode_options.skip_special_tokens = false;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
reasoning_parser,
|
parser,
|
||||||
tool_parser,
|
|
||||||
parallel_tool_calls: request.parallel_tool_calls,
|
parallel_tool_calls: request.parallel_tool_calls,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -87,8 +78,7 @@ impl DefaultChatOutputProcessor {
|
|||||||
/// content is treated as opaque text.
|
/// content is treated as opaque text.
|
||||||
pub fn plain_text_only() -> Self {
|
pub fn plain_text_only() -> Self {
|
||||||
Self {
|
Self {
|
||||||
reasoning_parser: None,
|
parser: Box::new(CombinedParser::plain_text_only()),
|
||||||
tool_parser: None,
|
|
||||||
parallel_tool_calls: true,
|
parallel_tool_calls: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -112,10 +102,6 @@ impl DefaultChatOutputProcessor {
|
|||||||
|
|
||||||
let parser = factory.create(parser_name, &request.tools)?;
|
let parser = factory.create(parser_name, &request.tools)?;
|
||||||
|
|
||||||
if parser.preserve_special_tokens() {
|
|
||||||
request.decode_options.skip_special_tokens = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
apply_structural_tag_constraint(request, parser.as_ref())?;
|
apply_structural_tag_constraint(request, parser.as_ref())?;
|
||||||
|
|
||||||
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
|
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
|
||||||
@@ -123,7 +109,6 @@ impl DefaultChatOutputProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn resolve_optional_reasoning_parser(
|
fn resolve_optional_reasoning_parser(
|
||||||
request: &mut ChatRequest,
|
|
||||||
model_id: &str,
|
model_id: &str,
|
||||||
tokenizer: DynTokenizer,
|
tokenizer: DynTokenizer,
|
||||||
selection: &ParserSelection,
|
selection: &ParserSelection,
|
||||||
@@ -142,10 +127,6 @@ impl DefaultChatOutputProcessor {
|
|||||||
|
|
||||||
let parser = factory.create(parser_name, tokenizer)?;
|
let parser = factory.create(parser_name, tokenizer)?;
|
||||||
|
|
||||||
if parser.preserve_special_tokens() {
|
|
||||||
request.decode_options.skip_special_tokens = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
|
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
|
||||||
Ok(Some(parser))
|
Ok(Some(parser))
|
||||||
}
|
}
|
||||||
@@ -156,16 +137,14 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
|
|||||||
|
|
||||||
impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||||
/// Transforms a raw generate-output token stream into structured chat
|
/// Transforms a raw generate-output token stream into structured chat
|
||||||
/// events through three sequential stages once text decoding has
|
/// events through two sequential stages once text decoding has
|
||||||
/// already happened:
|
/// already happened:
|
||||||
///
|
///
|
||||||
/// 1. [`reasoning_event_stream`] — reasoning/content separation
|
/// 1. [`unified_event_stream`] — reasoning and tool-call parsing
|
||||||
/// 2. [`tool_event_stream`] — tool-call parsing
|
/// 2. [`structured_chat_event_stream`] — final block assembly
|
||||||
/// 3. [`structured_chat_event_stream`] — final block assembly
|
|
||||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
||||||
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
|
let parsed = unified_event_stream(decoded, self.parser);
|
||||||
let tool = tool_event_stream(reasoning, self.tool_parser);
|
let structured = structured_chat_event_stream(parsed, self.parallel_tool_calls);
|
||||||
let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
|
|
||||||
|
|
||||||
Ok(structured.boxed())
|
Ok(structured.boxed())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,509 +0,0 @@
|
|||||||
//! Adapts decoded text updates into reasoning-aware assistant deltas.
|
|
||||||
//!
|
|
||||||
//! This stage sits between low-level token decoding and final block assembly.
|
|
||||||
//! It is the only place in the new pipeline that understands reasoning
|
|
||||||
//! separation: `decoded.rs` still only produces plain text deltas, while later
|
|
||||||
//! stages consume the semantic `Text` / `Reasoning` split emitted here.
|
|
||||||
|
|
||||||
use asynk_strim_attr::{TryYielder, try_stream};
|
|
||||||
use futures::{StreamExt as _, pin_mut};
|
|
||||||
use thiserror_ext::AsReport;
|
|
||||||
use tracing::warn;
|
|
||||||
use vllm_text::output::DecodedTextEvent;
|
|
||||||
|
|
||||||
use super::ContentEvent;
|
|
||||||
use crate::Result;
|
|
||||||
use crate::error::Error;
|
|
||||||
use crate::event::AssistantBlockKind;
|
|
||||||
use crate::output::DecodedTextEventStream;
|
|
||||||
use crate::parser::reasoning::{ReasoningDelta, ReasoningParser};
|
|
||||||
|
|
||||||
/// Per-stream reasoning parsing state.
|
|
||||||
struct ReasoningState {
|
|
||||||
/// Reasoning parser for the current model family.
|
|
||||||
parser: Box<dyn ReasoningParser>,
|
|
||||||
/// Whether reasoning parsing has already failed for this stream.
|
|
||||||
parser_failed: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReasoningState {
|
|
||||||
/// Create one fresh reasoning-adaptation state for a new streamed response.
|
|
||||||
fn new(parser: Box<dyn ReasoningParser>) -> Self {
|
|
||||||
Self {
|
|
||||||
parser,
|
|
||||||
parser_failed: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convert one decoded text delta into zero or more semantic assistant
|
|
||||||
/// deltas.
|
|
||||||
fn process_delta(&mut self, delta: String) -> Vec<ContentEvent> {
|
|
||||||
// If the parser has already failed, skip parsing and return plain text deltas.
|
|
||||||
if self.parser_failed {
|
|
||||||
return vec![ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta,
|
|
||||||
}];
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut events = Vec::new();
|
|
||||||
|
|
||||||
match self.parser.push(&delta) {
|
|
||||||
Ok(result) => {
|
|
||||||
push_reasoning_delta(&mut events, result);
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
if !self.parser_failed {
|
|
||||||
warn!(
|
|
||||||
error = %error.as_report(),
|
|
||||||
"reasoning parser failed; falling back to plain text deltas"
|
|
||||||
);
|
|
||||||
self.parser_failed = true;
|
|
||||||
}
|
|
||||||
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
events
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Initialize parser state once prompt token IDs are available.
|
|
||||||
fn initialize(&mut self, prompt_token_ids: &[u32]) {
|
|
||||||
if self.parser_failed {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
match self.parser.initialize(prompt_token_ids) {
|
|
||||||
Ok(()) => {}
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
error = %error.as_report(),
|
|
||||||
"failed to initialize reasoning parser; falling back to plain text deltas"
|
|
||||||
);
|
|
||||||
self.parser_failed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Flush any parser-held partial delimiter state at end of stream.
|
|
||||||
fn finish(&mut self) -> Vec<ContentEvent> {
|
|
||||||
if self.parser_failed {
|
|
||||||
return Vec::new();
|
|
||||||
}
|
|
||||||
|
|
||||||
match self.parser.finish() {
|
|
||||||
Ok(result) => {
|
|
||||||
let mut events = Vec::new();
|
|
||||||
push_reasoning_delta(&mut events, result);
|
|
||||||
events
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
warn!(error = %error.as_report(), "failed to flush reasoning parser state");
|
|
||||||
Vec::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Push one semantic text delta if it is non-empty.
|
|
||||||
fn push_text_delta(events: &mut Vec<ContentEvent>, kind: AssistantBlockKind, delta: String) {
|
|
||||||
if delta.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
events.push(ContentEvent::TextDelta { kind, delta });
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convert one parsed reasoning delta into zero or more content events.
|
|
||||||
fn push_reasoning_delta(events: &mut Vec<ContentEvent>, delta: ReasoningDelta) {
|
|
||||||
if let Some(reasoning) = delta.reasoning {
|
|
||||||
push_text_delta(events, AssistantBlockKind::Reasoning, reasoning);
|
|
||||||
}
|
|
||||||
if let Some(content) = delta.content {
|
|
||||||
push_text_delta(events, AssistantBlockKind::Text, content);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Wrap one decoded-text stream into the internal reasoning event stream.
|
|
||||||
#[try_stream]
|
|
||||||
pub(crate) async fn reasoning_event_stream(
|
|
||||||
decoded_stream: impl DecodedTextEventStream,
|
|
||||||
reasoning_parser: Option<Box<dyn ReasoningParser>>,
|
|
||||||
mut y: TryYielder<ContentEvent, Error>,
|
|
||||||
) -> Result<()> {
|
|
||||||
pin_mut!(decoded_stream);
|
|
||||||
|
|
||||||
// Without a parser, pass through as plain text deltas.
|
|
||||||
let Some(reasoning_parser) = reasoning_parser else {
|
|
||||||
while let Some(event) = decoded_stream.next().await.transpose()? {
|
|
||||||
for next in ContentEvent::from_decoded_plain_text(event) {
|
|
||||||
y.yield_ok(next).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut state = ReasoningState::new(reasoning_parser);
|
|
||||||
|
|
||||||
while let Some(event) = decoded_stream.next().await.transpose()? {
|
|
||||||
match event {
|
|
||||||
DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
} => {
|
|
||||||
state.initialize(&prompt_token_ids);
|
|
||||||
y.yield_ok(ContentEvent::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
DecodedTextEvent::TextDelta {
|
|
||||||
delta,
|
|
||||||
token_ids,
|
|
||||||
logprobs,
|
|
||||||
finished,
|
|
||||||
} => {
|
|
||||||
for next in state.process_delta(delta) {
|
|
||||||
y.yield_ok(next).await;
|
|
||||||
}
|
|
||||||
if logprobs.is_some() || !token_ids.is_empty() {
|
|
||||||
y.yield_ok(ContentEvent::LogprobsDelta {
|
|
||||||
logprobs,
|
|
||||||
token_ids,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
if let Some(finished) = finished {
|
|
||||||
for next in state.finish() {
|
|
||||||
y.yield_ok(next).await;
|
|
||||||
}
|
|
||||||
y.yield_ok(ContentEvent::Done {
|
|
||||||
usage: finished.usage,
|
|
||||||
finish_reason: finished.finish_reason,
|
|
||||||
kv_transfer_params: finished.kv_transfer_params,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use futures::{StreamExt as _, stream};
|
|
||||||
use vllm_llm::FinishReason;
|
|
||||||
use vllm_text::output::{
|
|
||||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
|
|
||||||
};
|
|
||||||
use vllm_tokenizer::{DynTokenizer, Tokenizer};
|
|
||||||
|
|
||||||
use super::super::ContentEvent;
|
|
||||||
use super::reasoning_event_stream;
|
|
||||||
use crate::event::AssistantBlockKind;
|
|
||||||
use crate::parser::reasoning::{
|
|
||||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names,
|
|
||||||
};
|
|
||||||
|
|
||||||
struct FakeTokenizer;
|
|
||||||
|
|
||||||
impl Tokenizer for FakeTokenizer {
|
|
||||||
fn encode(
|
|
||||||
&self,
|
|
||||||
text: &str,
|
|
||||||
_add_special_tokens: bool,
|
|
||||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
|
||||||
Ok(text.chars().map(u32::from).collect())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode(
|
|
||||||
&self,
|
|
||||||
token_ids: &[u32],
|
|
||||||
_skip_special_tokens: bool,
|
|
||||||
) -> vllm_tokenizer::Result<String> {
|
|
||||||
Ok(token_ids
|
|
||||||
.iter()
|
|
||||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
|
||||||
.collect())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
|
||||||
match token {
|
|
||||||
"<think>" => Some(1),
|
|
||||||
"</think>" => Some(2),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct FailingReasoningParser {
|
|
||||||
fail_next: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReasoningParser for FailingReasoningParser {
|
|
||||||
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>, ReasoningError>
|
|
||||||
where
|
|
||||||
Self: Sized + 'static,
|
|
||||||
{
|
|
||||||
Ok(Box::new(Self { fail_next: true }))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn push(&mut self, _text: &str) -> Result<ReasoningDelta, ReasoningError> {
|
|
||||||
if self.fail_next {
|
|
||||||
self.fail_next = false;
|
|
||||||
return Err(ReasoningError::MissingToken {
|
|
||||||
token: "<think>".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(ReasoningDelta::default())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box<dyn ReasoningParser> {
|
|
||||||
factory.register_parser::<FailingReasoningParser>("failing");
|
|
||||||
|
|
||||||
factory.create("failing", Arc::new(FakeTokenizer)).unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn reasoning_parser_failure_falls_back_to_plain_text() {
|
|
||||||
let mut factory = ReasoningParserFactory::new();
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids: vec![1, 2, 3].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "def".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: Some(vllm_text::Finished {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 3,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let events = collected
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.expect("reasoning stream should not fail");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
events,
|
|
||||||
vec![
|
|
||||||
ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1, 2, 3].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "def".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 3,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn reasoning_stream_preserves_logprobs_delta() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: Some(DecodedLogprobs {
|
|
||||||
positions: vec![DecodedPositionLogprobs {
|
|
||||||
entries: vec![DecodedTokenLogprob {
|
|
||||||
token_id: 0,
|
|
||||||
token: "a".to_string(),
|
|
||||||
logprob: -0.1,
|
|
||||||
rank: 1,
|
|
||||||
}],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let collected = reasoning_event_stream(events, None)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
collected,
|
|
||||||
vec![
|
|
||||||
ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::LogprobsDelta {
|
|
||||||
logprobs: Some(DecodedLogprobs {
|
|
||||||
positions: vec![DecodedPositionLogprobs {
|
|
||||||
entries: vec![DecodedTokenLogprob {
|
|
||||||
token_id: 0,
|
|
||||||
token: "a".to_string(),
|
|
||||||
logprob: -0.1,
|
|
||||||
rank: 1,
|
|
||||||
}],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
token_ids: vec![],
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() {
|
|
||||||
let tokenizer = Arc::new(FakeTokenizer);
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids: vec![2].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "thought ".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "done</think>OK".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let factory = ReasoningParserFactory::new();
|
|
||||||
let collected = reasoning_event_stream(
|
|
||||||
events,
|
|
||||||
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
|
|
||||||
)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let events = collected
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.expect("reasoning stream should not fail");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
events,
|
|
||||||
vec![
|
|
||||||
ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![2].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "thought ".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "done</think>OK".to_string(),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn qwen3_parser_tolerates_prompt_prefill_reasoning() {
|
|
||||||
let tokenizer = Arc::new(FakeTokenizer);
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "thought ".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
|
||||||
delta: "done</think>OK".to_string(),
|
|
||||||
token_ids: vec![],
|
|
||||||
logprobs: None,
|
|
||||||
finished: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let factory = ReasoningParserFactory::new();
|
|
||||||
let collected = reasoning_event_stream(
|
|
||||||
events,
|
|
||||||
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
|
|
||||||
)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let events = collected
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.expect("reasoning stream should not fail");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
events,
|
|
||||||
vec![
|
|
||||||
ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Reasoning,
|
|
||||||
delta: "thought ".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Reasoning,
|
|
||||||
delta: "done".to_string(),
|
|
||||||
},
|
|
||||||
ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "OK".to_string(),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
|
|||||||
mod tests {
|
mod tests {
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
||||||
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
|
use vllm_parser::tool::{Qwen3CoderToolParser, Tool};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
|||||||
@@ -1,985 +0,0 @@
|
|||||||
//! Adapts plain assistant text deltas into tool-call-aware assistant updates.
|
|
||||||
//!
|
|
||||||
//! This stage runs after reasoning separation and before final block assembly.
|
|
||||||
//! It only inspects normal assistant text, leaves reasoning deltas untouched,
|
|
||||||
//! and translates incremental tool parsing output into internal tool-call
|
|
||||||
//! events while preserving plain-text fallback behavior.
|
|
||||||
|
|
||||||
use asynk_strim_attr::{TryYielder, try_stream};
|
|
||||||
use futures::{StreamExt as _, pin_mut};
|
|
||||||
use thiserror_ext::AsReport;
|
|
||||||
use tracing::warn;
|
|
||||||
|
|
||||||
use super::{AssistantEvent, ContentEvent, ContentEventStream};
|
|
||||||
use crate::Result;
|
|
||||||
use crate::error::Error;
|
|
||||||
use crate::event::AssistantBlockKind;
|
|
||||||
use crate::output::generate_tool_call_id;
|
|
||||||
use crate::parser::tool::{ToolCallDelta, ToolParser, ToolParserOutput};
|
|
||||||
|
|
||||||
/// Per-stream tool parsing state.
|
|
||||||
struct ToolState {
|
|
||||||
/// Parser for the current model family.
|
|
||||||
parser: Box<dyn ToolParser>,
|
|
||||||
/// Whether tool parsing has already failed for this stream.
|
|
||||||
parser_failed: bool,
|
|
||||||
/// The parser-local index of the currently open tool call, if any.
|
|
||||||
// NOTE: We only allow single open tool call at a time right now, since that's what all
|
|
||||||
// supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple
|
|
||||||
// interleaved calls in the future.
|
|
||||||
open_call_index: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ToolState {
|
|
||||||
/// Create one fresh tool-parsing state for a new streamed response.
|
|
||||||
fn new(parser: Box<dyn ToolParser>) -> Self {
|
|
||||||
Self {
|
|
||||||
parser,
|
|
||||||
parser_failed: false,
|
|
||||||
open_call_index: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convert one semantic assistant text delta into zero or more tool-aware
|
|
||||||
/// internal events.
|
|
||||||
fn process_text_delta(
|
|
||||||
&mut self,
|
|
||||||
kind: AssistantBlockKind,
|
|
||||||
delta: String,
|
|
||||||
) -> Result<Vec<AssistantEvent>> {
|
|
||||||
let mut events = Vec::new();
|
|
||||||
|
|
||||||
// Only normal assistant text is eligible for tool parsing. Reasoning
|
|
||||||
// blocks and plain-text fallback should pass through unchanged.
|
|
||||||
if kind != AssistantBlockKind::Text || self.parser_failed {
|
|
||||||
self.open_call_index = None;
|
|
||||||
events.push(AssistantEvent::TextDelta { kind, delta });
|
|
||||||
return Ok(events);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut output = ToolParserOutput::default();
|
|
||||||
let parse_result = self.parser.parse_into(&delta, &mut output);
|
|
||||||
|
|
||||||
match parse_result {
|
|
||||||
Ok(()) => self.process_parser_output(kind, output, &mut events)?,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
error = %error.as_report(),
|
|
||||||
"tool parser failed; falling back to plain text deltas"
|
|
||||||
);
|
|
||||||
// Permanently mark this parser as failed.
|
|
||||||
// TODO: we may consider recovering from parsing errors in the future.
|
|
||||||
self.parser_failed = true;
|
|
||||||
|
|
||||||
// On parsing failure, we still apply the partial parser output if any, but we close
|
|
||||||
// any open tool calls and emit the remaining buffered text as a plain-text delta to
|
|
||||||
// preserve as much of the output as possible.
|
|
||||||
self.process_parser_output(kind, output, &mut events)?;
|
|
||||||
self.open_call_index = None;
|
|
||||||
push_text_delta(&mut events, kind, self.parser.reset());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(events)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Apply one parsed tool output to the current stream state.
|
|
||||||
fn process_parser_output(
|
|
||||||
&mut self,
|
|
||||||
kind: AssistantBlockKind,
|
|
||||||
output: ToolParserOutput,
|
|
||||||
events: &mut Vec<AssistantEvent>,
|
|
||||||
) -> Result<()> {
|
|
||||||
// When we are not currently streaming a tool call, preserve plain
|
|
||||||
// text first and then surface any new tool call items.
|
|
||||||
if self.open_call_index.is_none() {
|
|
||||||
push_text_delta(events, kind, output.normal_text);
|
|
||||||
self.process_tool_items(output.calls, events)?;
|
|
||||||
} else {
|
|
||||||
// Once a tool call is open, prioritize tool deltas first. If the
|
|
||||||
// parser emits normal text again, close the tool call and resume
|
|
||||||
// plain text output.
|
|
||||||
self.process_tool_items(output.calls, events)?;
|
|
||||||
if !output.normal_text.is_empty() {
|
|
||||||
self.open_call_index = None;
|
|
||||||
push_text_delta(events, kind, output.normal_text);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Apply one batch of parsed tool-call deltas emitted by the parser.
|
|
||||||
fn process_tool_items(
|
|
||||||
&mut self,
|
|
||||||
items: Vec<ToolCallDelta>,
|
|
||||||
events: &mut Vec<AssistantEvent>,
|
|
||||||
) -> Result<()> {
|
|
||||||
for item in items {
|
|
||||||
if let Some(name) = item.name {
|
|
||||||
let is_new_tool = match self.open_call_index {
|
|
||||||
Some(open_call_index) => open_call_index != item.tool_index,
|
|
||||||
None => true,
|
|
||||||
};
|
|
||||||
if is_new_tool {
|
|
||||||
let id = self
|
|
||||||
.parser
|
|
||||||
.tool_call_id(item.tool_index)
|
|
||||||
.map(str::to_string)
|
|
||||||
.unwrap_or_else(generate_tool_call_id);
|
|
||||||
self.open_call_index = Some(item.tool_index);
|
|
||||||
events.push(AssistantEvent::ToolCallStart { id, name });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if item.arguments.is_empty() {
|
|
||||||
// No arguments delta to apply.
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Some(open_call_index) = self.open_call_index else {
|
|
||||||
return Err(Error::ToolCallStreamInvariant {
|
|
||||||
message: format!(
|
|
||||||
"received arguments for tool index {} before any tool-call start",
|
|
||||||
item.tool_index
|
|
||||||
),
|
|
||||||
});
|
|
||||||
};
|
|
||||||
if open_call_index != item.tool_index {
|
|
||||||
return Err(Error::ToolCallStreamInvariant {
|
|
||||||
message: format!(
|
|
||||||
"received arguments for tool index {} while tool index {} is open",
|
|
||||||
item.tool_index, open_call_index
|
|
||||||
),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
events.push(AssistantEvent::ToolCallArgumentsDelta {
|
|
||||||
delta: item.arguments,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Flush parser state at end-of-stream and close any remaining open calls.
|
|
||||||
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
|
|
||||||
let mut events = Vec::new();
|
|
||||||
|
|
||||||
if self.parser_failed {
|
|
||||||
return Ok(events);
|
|
||||||
}
|
|
||||||
|
|
||||||
match self.parser.finish() {
|
|
||||||
Ok(output) => {
|
|
||||||
self.process_parser_output(AssistantBlockKind::Text, output, &mut events)?
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
error = %error.as_report(),
|
|
||||||
"tool parser finish failed; closing open tool calls with buffered state"
|
|
||||||
);
|
|
||||||
self.parser_failed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(events)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Push one plain-text delta if it is non-empty.
|
|
||||||
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
|
|
||||||
if delta.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
events.push(AssistantEvent::TextDelta { kind, delta });
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Wrap one semantic assistant stream into the internal tool-aware assistant
|
|
||||||
/// stream.
|
|
||||||
#[try_stream]
|
|
||||||
pub(crate) async fn tool_event_stream(
|
|
||||||
stream: impl ContentEventStream,
|
|
||||||
parser: Option<Box<dyn ToolParser>>,
|
|
||||||
mut y: TryYielder<AssistantEvent, Error>,
|
|
||||||
) -> Result<()> {
|
|
||||||
// Without a parser, pass through the input stream unchanged.
|
|
||||||
let Some(parser) = parser else {
|
|
||||||
pin_mut!(stream);
|
|
||||||
while let Some(event) = stream.next().await.transpose()? {
|
|
||||||
y.yield_ok(event.into()).await;
|
|
||||||
}
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
|
|
||||||
pin_mut!(stream);
|
|
||||||
let mut state = ToolState::new(parser);
|
|
||||||
|
|
||||||
while let Some(event) = stream.next().await.transpose()? {
|
|
||||||
match event {
|
|
||||||
ContentEvent::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
} => {
|
|
||||||
y.yield_ok(AssistantEvent::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
ContentEvent::TextDelta { kind, delta } => {
|
|
||||||
for next in state.process_text_delta(kind, delta)? {
|
|
||||||
y.yield_ok(next).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ContentEvent::LogprobsDelta {
|
|
||||||
logprobs,
|
|
||||||
token_ids,
|
|
||||||
} => {
|
|
||||||
y.yield_ok(AssistantEvent::LogprobsDelta {
|
|
||||||
logprobs,
|
|
||||||
token_ids,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
ContentEvent::Done {
|
|
||||||
usage,
|
|
||||||
finish_reason,
|
|
||||||
kv_transfer_params,
|
|
||||||
} => {
|
|
||||||
for next in state.finish()? {
|
|
||||||
y.yield_ok(next).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
y.yield_ok(AssistantEvent::Done {
|
|
||||||
usage,
|
|
||||||
finish_reason,
|
|
||||||
kv_transfer_params,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
|
|
||||||
use futures::{StreamExt as _, stream};
|
|
||||||
use vllm_llm::FinishReason;
|
|
||||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
|
||||||
use vllm_tool_parser::Result;
|
|
||||||
|
|
||||||
use super::super::{AssistantEvent, ContentEvent};
|
|
||||||
use super::tool_event_stream;
|
|
||||||
use crate::error::Error;
|
|
||||||
use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
|
|
||||||
use crate::output::structured::structured_chat_event_stream;
|
|
||||||
use crate::parser::tool::{
|
|
||||||
DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput,
|
|
||||||
};
|
|
||||||
use crate::request::ChatTool;
|
|
||||||
use crate::stream::{ChatEventStream, CollectedAssistantMessage};
|
|
||||||
|
|
||||||
struct FailingParser {
|
|
||||||
fail_next: bool,
|
|
||||||
buffered: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct ScriptedParser {
|
|
||||||
push_outputs: Vec<ToolParserOutput>,
|
|
||||||
finish_output: ToolParserOutput,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct PartialThenFailParser {
|
|
||||||
buffered: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct IdScriptedParser {
|
|
||||||
output: ToolParserOutput,
|
|
||||||
tool_call_id: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ToolParser for FailingParser {
|
|
||||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
|
||||||
where
|
|
||||||
Self: Sized + 'static,
|
|
||||||
{
|
|
||||||
Ok(Box::new(Self {
|
|
||||||
fail_next: false,
|
|
||||||
buffered: String::new(),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
|
|
||||||
self.buffered.push_str(chunk);
|
|
||||||
if self.fail_next {
|
|
||||||
self.fail_next = false;
|
|
||||||
return Err(ToolParserError::ParsingFailed {
|
|
||||||
message: "boom".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
self.buffered.clear();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
|
||||||
Ok(ToolParserOutput::default())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reset(&mut self) -> String {
|
|
||||||
std::mem::take(&mut self.buffered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ToolParser for ScriptedParser {
|
|
||||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
|
||||||
where
|
|
||||||
Self: Sized + 'static,
|
|
||||||
{
|
|
||||||
Ok(Box::new(Self {
|
|
||||||
push_outputs: Vec::new(),
|
|
||||||
finish_output: ToolParserOutput::default(),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
|
||||||
let mut next = self.push_outputs.pop().unwrap_or_default();
|
|
||||||
output.normal_text.push_str(&next.normal_text);
|
|
||||||
output.calls.append(&mut next.calls);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
|
||||||
Ok(std::mem::take(&mut self.finish_output))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reset(&mut self) -> String {
|
|
||||||
String::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ToolParser for IdScriptedParser {
|
|
||||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
|
||||||
where
|
|
||||||
Self: Sized + 'static,
|
|
||||||
{
|
|
||||||
Ok(Box::new(Self {
|
|
||||||
output: ToolParserOutput::default(),
|
|
||||||
tool_call_id: None,
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
|
||||||
(tool_index == 0).then_some(self.tool_call_id.as_deref()).flatten()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
|
||||||
output.append(std::mem::take(&mut self.output));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
|
||||||
Ok(ToolParserOutput::default())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reset(&mut self) -> String {
|
|
||||||
String::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ToolParser for PartialThenFailParser {
|
|
||||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
|
||||||
where
|
|
||||||
Self: Sized + 'static,
|
|
||||||
{
|
|
||||||
Ok(Box::new(Self {
|
|
||||||
buffered: String::new(),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
|
||||||
output.calls.extend([
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("get_weather".to_string()),
|
|
||||||
arguments: String::new(),
|
|
||||||
},
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: None,
|
|
||||||
arguments: r#"{"location":"SF"}"#.to_string(),
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
self.buffered.push_str(" trailing text");
|
|
||||||
Err(ToolParserError::ParsingFailed {
|
|
||||||
message: "boom".to_string(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
|
||||||
Ok(ToolParserOutput::default())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reset(&mut self) -> String {
|
|
||||||
std::mem::take(&mut self.buffered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn deepseek_v4_test_tools() -> Vec<ChatTool> {
|
|
||||||
vec![
|
|
||||||
ChatTool {
|
|
||||||
name: "get_weather".to_string(),
|
|
||||||
description: None,
|
|
||||||
parameters: serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"location": { "type": "string" }
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
strict: None,
|
|
||||||
},
|
|
||||||
ChatTool {
|
|
||||||
name: "add".to_string(),
|
|
||||||
description: None,
|
|
||||||
parameters: serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"x": { "type": "integer" },
|
|
||||||
"y": { "type": "integer" }
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
strict: None,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn collect_deepseek_v4_message(chunks: Vec<String>) -> CollectedAssistantMessage {
|
|
||||||
let events = chunks
|
|
||||||
.into_iter()
|
|
||||||
.map(|delta| {
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.chain(std::iter::once(Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 1,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
})));
|
|
||||||
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
|
|
||||||
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
|
|
||||||
let chat_events = structured_chat_event_stream(assistant_events, true);
|
|
||||||
|
|
||||||
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
|
|
||||||
.collect_message()
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn message_tool_projection(
|
|
||||||
message: &CollectedAssistantMessage,
|
|
||||||
) -> (String, Vec<(String, serde_json::Value)>) {
|
|
||||||
(
|
|
||||||
message.message.text(),
|
|
||||||
message
|
|
||||||
.message
|
|
||||||
.tool_calls()
|
|
||||||
.map(|call| {
|
|
||||||
(
|
|
||||||
call.name.clone(),
|
|
||||||
serde_json::from_str(&call.arguments).unwrap(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "ignored".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 1,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let events = tool_event_stream(
|
|
||||||
events,
|
|
||||||
Some(Box::new(PartialThenFailParser {
|
|
||||||
buffered: String::new(),
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
&events[0],
|
|
||||||
AssistantEvent::ToolCallStart { name, .. } if name == "get_weather"
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&events[1],
|
|
||||||
AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"#
|
|
||||||
));
|
|
||||||
assert_eq!(
|
|
||||||
events[2],
|
|
||||||
AssistantEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: " trailing text".to_string(),
|
|
||||||
}
|
|
||||||
);
|
|
||||||
assert!(matches!(events[3], AssistantEvent::Done { .. }));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_preserves_parser_provided_tool_call_id() {
|
|
||||||
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "ignored".to_string(),
|
|
||||||
})]);
|
|
||||||
let parser = IdScriptedParser {
|
|
||||||
output: ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("get_weather".to_string()),
|
|
||||||
arguments: "{}".to_string(),
|
|
||||||
}],
|
|
||||||
},
|
|
||||||
tool_call_id: Some("functions.get_weather:0".to_string()),
|
|
||||||
};
|
|
||||||
|
|
||||||
let events = tool_event_stream(events, Some(Box::new(parser)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
&events[0],
|
|
||||||
AssistantEvent::ToolCallStart { id, name }
|
|
||||||
if id == "functions.get_weather:0" && name == "get_weather"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_generates_tool_call_id_when_parser_omits_one() {
|
|
||||||
let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "ignored".to_string(),
|
|
||||||
})]);
|
|
||||||
let parser = IdScriptedParser {
|
|
||||||
output: ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("get_weather".to_string()),
|
|
||||||
arguments: "{}".to_string(),
|
|
||||||
}],
|
|
||||||
},
|
|
||||||
tool_call_id: None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let events = tool_event_stream(events, Some(Box::new(parser)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
&events[0],
|
|
||||||
AssistantEvent::ToolCallStart { id, name }
|
|
||||||
if id.starts_with("call_") && name == "get_weather"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn real_buffered_parser_error_matches_streaming_and_non_streaming() {
|
|
||||||
let prefix = "I will check both.\n";
|
|
||||||
let first_tool_call = concat!(
|
|
||||||
"<|DSML|tool_calls>\n",
|
|
||||||
"<|DSML|invoke name=\"get_weather\">\n",
|
|
||||||
"<|DSML|parameter name=\"location\" string=\"true\">Tokyo</|DSML|parameter>\n",
|
|
||||||
"</|DSML|invoke>",
|
|
||||||
);
|
|
||||||
let malformed_second_tool_call = concat!(
|
|
||||||
"\n<|DSML|invoke name=\"add\">\n",
|
|
||||||
"not a parameter\n",
|
|
||||||
"</|DSML|invoke>\n",
|
|
||||||
"</|DSML|tool_calls>",
|
|
||||||
);
|
|
||||||
let streaming_chunks = vec![
|
|
||||||
prefix.to_string(),
|
|
||||||
first_tool_call.to_string(),
|
|
||||||
malformed_second_tool_call.to_string(),
|
|
||||||
];
|
|
||||||
let full_output = streaming_chunks.concat();
|
|
||||||
|
|
||||||
let streaming = collect_deepseek_v4_message(streaming_chunks).await;
|
|
||||||
let non_streaming = collect_deepseek_v4_message(vec![full_output]).await;
|
|
||||||
|
|
||||||
let expected = (
|
|
||||||
format!("{prefix}{malformed_second_tool_call}"),
|
|
||||||
vec![(
|
|
||||||
"get_weather".to_string(),
|
|
||||||
serde_json::json!({ "location": "Tokyo" }),
|
|
||||||
)],
|
|
||||||
);
|
|
||||||
assert_eq!(message_tool_projection(&streaming), expected);
|
|
||||||
assert_eq!(message_tool_projection(&non_streaming), expected);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_parser_failure_falls_back_to_plain_text() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1, 2, 3].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "def".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 3,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let collected = tool_event_stream(
|
|
||||||
events,
|
|
||||||
Some(Box::new(FailingParser {
|
|
||||||
fail_next: true,
|
|
||||||
buffered: String::new(),
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let events = collected
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.expect("tool stream should not fail");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
events,
|
|
||||||
vec![
|
|
||||||
AssistantEvent::Start {
|
|
||||||
prompt_token_ids: vec![1, 2, 3].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
AssistantEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "abc".to_string(),
|
|
||||||
},
|
|
||||||
AssistantEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "def".to_string(),
|
|
||||||
},
|
|
||||||
AssistantEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 3,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
let message = ChatEventStream::new(
|
|
||||||
"req_fallback".to_string(),
|
|
||||||
Box::pin(structured_chat_event_stream(
|
|
||||||
stream::iter(events.into_iter().map(Ok)),
|
|
||||||
true,
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
.collect_message()
|
|
||||||
.await
|
|
||||||
.expect("collect_message should succeed");
|
|
||||||
assert_eq!(message.message.text(), "abcdef");
|
|
||||||
assert!(message.message.tool_calls().next().is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_preserves_logprobs_delta() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::LogprobsDelta {
|
|
||||||
logprobs: Some(DecodedLogprobs {
|
|
||||||
positions: vec![DecodedPositionLogprobs {
|
|
||||||
entries: vec![DecodedTokenLogprob {
|
|
||||||
token_id: 0,
|
|
||||||
token: "a".to_string(),
|
|
||||||
logprob: -0.2,
|
|
||||||
rank: 1,
|
|
||||||
}],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
token_ids: vec![],
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
let events = tool_event_stream(
|
|
||||||
events,
|
|
||||||
Some(Box::new(FailingParser {
|
|
||||||
fail_next: false,
|
|
||||||
buffered: String::new(),
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
events,
|
|
||||||
vec![
|
|
||||||
AssistantEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
},
|
|
||||||
AssistantEvent::LogprobsDelta {
|
|
||||||
logprobs: Some(DecodedLogprobs {
|
|
||||||
positions: vec![DecodedPositionLogprobs {
|
|
||||||
entries: vec![DecodedTokenLogprob {
|
|
||||||
token_id: 0,
|
|
||||||
token: "a".to_string(),
|
|
||||||
logprob: -0.2,
|
|
||||||
rank: 1,
|
|
||||||
}],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
token_ids: vec![],
|
|
||||||
},
|
|
||||||
AssistantEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 0,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_rejects_interleaved_tool_indices() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "ignored".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 1,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let parser = ScriptedParser {
|
|
||||||
push_outputs: vec![ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("first".to_string()),
|
|
||||||
arguments: String::new(),
|
|
||||||
},
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 1,
|
|
||||||
name: None,
|
|
||||||
arguments: "{}".to_string(),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}],
|
|
||||||
finish_output: ToolParserOutput::default(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let err = tool_event_stream(events, Some(Box::new(parser)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.find_map(|output| output.err())
|
|
||||||
.expect("expected invariant error");
|
|
||||||
|
|
||||||
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "start".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "text".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "args".to_string(),
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let parser = ScriptedParser {
|
|
||||||
push_outputs: vec![
|
|
||||||
ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: None,
|
|
||||||
arguments: "}".to_string(),
|
|
||||||
}],
|
|
||||||
},
|
|
||||||
ToolParserOutput {
|
|
||||||
normal_text: "plain text".to_string(),
|
|
||||||
calls: Vec::new(),
|
|
||||||
},
|
|
||||||
ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("first".to_string()),
|
|
||||||
arguments: "{".to_string(),
|
|
||||||
}],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
finish_output: ToolParserOutput::default(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let err = tool_event_stream(events, Some(Box::new(parser)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.find_map(|output| output.err())
|
|
||||||
.expect("expected invariant error");
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
err,
|
|
||||||
Error::ToolCallStreamInvariant { message }
|
|
||||||
if message == "received arguments for tool index 0 before any tool-call start"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_stream_emits_start_and_args_for_terminal_text() {
|
|
||||||
let events = stream::iter(vec![
|
|
||||||
Ok(ContentEvent::Start {
|
|
||||||
prompt_token_ids: vec![1].into(),
|
|
||||||
prompt_logprobs: None,
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta: "ignored".to_string(),
|
|
||||||
}),
|
|
||||||
Ok(ContentEvent::Done {
|
|
||||||
usage: vllm_llm::TokenUsage {
|
|
||||||
prompt_token_count: 1,
|
|
||||||
output_token_count: 1,
|
|
||||||
cached_token_count: 0,
|
|
||||||
},
|
|
||||||
finish_reason: FinishReason::stop_eos(),
|
|
||||||
kv_transfer_params: None,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let parser = ScriptedParser {
|
|
||||||
push_outputs: vec![ToolParserOutput {
|
|
||||||
normal_text: String::new(),
|
|
||||||
calls: vec![
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 0,
|
|
||||||
name: Some("first".to_string()),
|
|
||||||
arguments: r#"{"a":1}"#.to_string(),
|
|
||||||
},
|
|
||||||
crate::parser::tool::ToolCallDelta {
|
|
||||||
tool_index: 1,
|
|
||||||
name: Some("second".to_string()),
|
|
||||||
arguments: r#"{"b":2}"#.to_string(),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}],
|
|
||||||
finish_output: ToolParserOutput::default(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let events = tool_event_stream(events, Some(Box::new(parser)))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.collect::<crate::Result<Vec<_>>>()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. }));
|
|
||||||
assert!(matches!(
|
|
||||||
events[2],
|
|
||||||
AssistantEvent::ToolCallArgumentsDelta { .. }
|
|
||||||
));
|
|
||||||
assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. }));
|
|
||||||
assert!(matches!(
|
|
||||||
events[4],
|
|
||||||
AssistantEvent::ToolCallArgumentsDelta { .. }
|
|
||||||
));
|
|
||||||
let collected = ChatEventStream::new(
|
|
||||||
"req_final_only".to_string(),
|
|
||||||
Box::pin(structured_chat_event_stream(
|
|
||||||
stream::iter(events.into_iter().map(Ok)),
|
|
||||||
true,
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
.collect_message()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let tool_calls = collected.message.tool_calls().collect::<Vec<_>>();
|
|
||||||
assert_eq!(tool_calls.len(), 2);
|
|
||||||
assert_eq!(tool_calls[0].name, "first");
|
|
||||||
assert_eq!(tool_calls[1].name, "second");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,582 @@
|
|||||||
|
//! Adapts decoded text updates into parsed assistant deltas.
|
||||||
|
//!
|
||||||
|
//! This stage sits between low-level token decoding and final block assembly.
|
||||||
|
//! It drives one unified parser that may emit normal text, reasoning text, or
|
||||||
|
//! tool-call deltas, then normalizes those parser events into internal
|
||||||
|
//! assistant events.
|
||||||
|
|
||||||
|
use asynk_strim_attr::{TryYielder, try_stream};
|
||||||
|
use futures::{StreamExt as _, pin_mut};
|
||||||
|
use thiserror_ext::AsReport;
|
||||||
|
use tracing::warn;
|
||||||
|
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||||
|
use vllm_text::output::DecodedTextEvent;
|
||||||
|
|
||||||
|
use crate::Result;
|
||||||
|
use crate::error::Error;
|
||||||
|
use crate::event::AssistantBlockKind;
|
||||||
|
use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
|
||||||
|
|
||||||
|
/// Per-stream unified parsing state.
|
||||||
|
struct UnifiedParserState {
|
||||||
|
/// Parser for the current request stream.
|
||||||
|
parser: Box<dyn UnifiedParser>,
|
||||||
|
/// Whether unified parsing has already failed for this stream.
|
||||||
|
parser_failed: bool,
|
||||||
|
/// The parser-local index of the currently open tool call, if any.
|
||||||
|
///
|
||||||
|
/// Supported parsers currently emit at most one active tool call at a time.
|
||||||
|
/// Change this to an indexed map if a model needs interleaved calls later.
|
||||||
|
open_call_index: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UnifiedParserState {
|
||||||
|
/// Create one fresh unified parsing state for a new streamed response.
|
||||||
|
fn new(parser: Box<dyn UnifiedParser>) -> Self {
|
||||||
|
Self {
|
||||||
|
parser,
|
||||||
|
parser_failed: false,
|
||||||
|
open_call_index: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initialize parser state once prompt token IDs are available.
|
||||||
|
fn initialize(&mut self, prompt_token_ids: &[u32]) {
|
||||||
|
if self.parser_failed {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.parser.initialize(prompt_token_ids) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error.as_report(),
|
||||||
|
"failed to initialize unified parser; falling back to plain text deltas"
|
||||||
|
);
|
||||||
|
self.parser_failed = true;
|
||||||
|
self.open_call_index = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert one decoded text delta into zero or more parsed assistant events.
|
||||||
|
fn process_delta(&mut self, delta: String) -> Result<Vec<AssistantEvent>> {
|
||||||
|
if self.parser_failed {
|
||||||
|
self.open_call_index = None;
|
||||||
|
return Ok(text_event(AssistantBlockKind::Text, delta).into_iter().collect());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
match self.parser.parse_into(&delta, &mut output) {
|
||||||
|
Ok(()) => {
|
||||||
|
let mut events = Vec::new();
|
||||||
|
self.process_parser_output(output, &mut events)?;
|
||||||
|
Ok(events)
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error.as_report(),
|
||||||
|
"unified parser failed; falling back to plain text deltas"
|
||||||
|
);
|
||||||
|
self.parser_failed = true;
|
||||||
|
|
||||||
|
let mut events = Vec::new();
|
||||||
|
self.process_parser_output(output, &mut events)?;
|
||||||
|
self.open_call_index = None;
|
||||||
|
|
||||||
|
let recovered = self.parser.reset();
|
||||||
|
if recovered.is_empty() && events.is_empty() {
|
||||||
|
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
|
||||||
|
} else {
|
||||||
|
push_text_delta(&mut events, AssistantBlockKind::Text, recovered);
|
||||||
|
}
|
||||||
|
Ok(events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flush parser state at end-of-stream and close any remaining open calls.
|
||||||
|
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
|
||||||
|
let mut events = Vec::new();
|
||||||
|
|
||||||
|
if self.parser_failed {
|
||||||
|
return Ok(events);
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.parser.finish() {
|
||||||
|
Ok(output) => self.process_parser_output(output, &mut events)?,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error.as_report(),
|
||||||
|
"unified parser finish failed; closing open parser state"
|
||||||
|
);
|
||||||
|
self.parser_failed = true;
|
||||||
|
self.open_call_index = None;
|
||||||
|
// TODO: should we reset and emit the buffered text?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(events)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply one parsed unified output to the current stream state.
|
||||||
|
fn process_parser_output(
|
||||||
|
&mut self,
|
||||||
|
output: UnifiedParserOutput,
|
||||||
|
events: &mut Vec<AssistantEvent>,
|
||||||
|
) -> Result<()> {
|
||||||
|
for event in output.events {
|
||||||
|
match event {
|
||||||
|
UnifiedParserEvent::Text(delta) => {
|
||||||
|
self.open_call_index = None;
|
||||||
|
push_text_delta(events, AssistantBlockKind::Text, delta);
|
||||||
|
}
|
||||||
|
UnifiedParserEvent::Reasoning(delta) => {
|
||||||
|
self.open_call_index = None;
|
||||||
|
push_text_delta(events, AssistantBlockKind::Reasoning, delta);
|
||||||
|
}
|
||||||
|
UnifiedParserEvent::ToolCall(item) => {
|
||||||
|
self.process_tool_item(item, events)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply one parsed tool-call delta emitted by the parser.
|
||||||
|
fn process_tool_item(
|
||||||
|
&mut self,
|
||||||
|
item: vllm_parser::tool::ToolCallDelta,
|
||||||
|
events: &mut Vec<AssistantEvent>,
|
||||||
|
) -> Result<()> {
|
||||||
|
if let Some(name) = item.name {
|
||||||
|
let is_new_tool = match self.open_call_index {
|
||||||
|
Some(open_call_index) => open_call_index != item.tool_index,
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
if is_new_tool {
|
||||||
|
let id = self
|
||||||
|
.parser
|
||||||
|
.tool_call_id(item.tool_index)
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_else(generate_tool_call_id);
|
||||||
|
self.open_call_index = Some(item.tool_index);
|
||||||
|
events.push(AssistantEvent::ToolCallStart { id, name });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if item.arguments.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let Some(open_call_index) = self.open_call_index else {
|
||||||
|
return Err(Error::ToolCallStreamInvariant {
|
||||||
|
message: format!(
|
||||||
|
"received arguments for tool index {} before any tool-call start",
|
||||||
|
item.tool_index
|
||||||
|
),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
if open_call_index != item.tool_index {
|
||||||
|
return Err(Error::ToolCallStreamInvariant {
|
||||||
|
message: format!(
|
||||||
|
"received arguments for tool index {} while tool index {} is open",
|
||||||
|
item.tool_index, open_call_index
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
events.push(AssistantEvent::ToolCallArgumentsDelta {
|
||||||
|
delta: item.arguments,
|
||||||
|
});
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build one plain text event if `delta` is non-empty.
|
||||||
|
fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
|
||||||
|
if delta.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(AssistantEvent::TextDelta { kind, delta })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push one plain text delta if it is non-empty.
|
||||||
|
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
|
||||||
|
if let Some(event) = text_event(kind, delta) {
|
||||||
|
events.push(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wrap one decoded-text stream into the internal unified assistant stream.
|
||||||
|
#[try_stream]
|
||||||
|
pub(crate) async fn unified_event_stream(
|
||||||
|
decoded_stream: impl DecodedTextEventStream,
|
||||||
|
parser: Box<dyn UnifiedParser>,
|
||||||
|
mut y: TryYielder<AssistantEvent, Error>,
|
||||||
|
) -> Result<()> {
|
||||||
|
pin_mut!(decoded_stream);
|
||||||
|
|
||||||
|
let mut state = UnifiedParserState::new(parser);
|
||||||
|
|
||||||
|
while let Some(event) = decoded_stream.next().await.transpose()? {
|
||||||
|
match event {
|
||||||
|
DecodedTextEvent::Start {
|
||||||
|
prompt_token_ids,
|
||||||
|
prompt_logprobs,
|
||||||
|
} => {
|
||||||
|
state.initialize(&prompt_token_ids);
|
||||||
|
y.yield_ok(AssistantEvent::Start {
|
||||||
|
prompt_token_ids,
|
||||||
|
prompt_logprobs,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
DecodedTextEvent::TextDelta {
|
||||||
|
delta,
|
||||||
|
token_ids,
|
||||||
|
logprobs,
|
||||||
|
finished,
|
||||||
|
} => {
|
||||||
|
for next in state.process_delta(delta)? {
|
||||||
|
y.yield_ok(next).await;
|
||||||
|
}
|
||||||
|
if logprobs.is_some() || !token_ids.is_empty() {
|
||||||
|
y.yield_ok(AssistantEvent::LogprobsDelta {
|
||||||
|
logprobs,
|
||||||
|
token_ids,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
if let Some(finished) = finished {
|
||||||
|
for next in state.finish()? {
|
||||||
|
y.yield_ok(next).await;
|
||||||
|
}
|
||||||
|
y.yield_ok(AssistantEvent::Done {
|
||||||
|
usage: finished.usage,
|
||||||
|
finish_reason: finished.finish_reason,
|
||||||
|
kv_transfer_params: finished.kv_transfer_params,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
|
||||||
|
use futures::{StreamExt as _, stream};
|
||||||
|
use vllm_parser::reasoning::ReasoningError;
|
||||||
|
use vllm_parser::tool::ToolCallDelta;
|
||||||
|
use vllm_parser::unified::{UnifiedParserError, UnifiedParserOutput};
|
||||||
|
|
||||||
|
use super::unified_event_stream;
|
||||||
|
use crate::event::AssistantBlockKind;
|
||||||
|
use crate::output::AssistantEvent;
|
||||||
|
|
||||||
|
enum ScriptedStep {
|
||||||
|
Output(UnifiedParserOutput),
|
||||||
|
Error {
|
||||||
|
committed: UnifiedParserOutput,
|
||||||
|
reset_text: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ScriptedParser {
|
||||||
|
steps: VecDeque<ScriptedStep>,
|
||||||
|
reset_text: String,
|
||||||
|
tool_call_id: Option<String>,
|
||||||
|
finish_error_reset_text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScriptedParser {
|
||||||
|
fn new(steps: impl IntoIterator<Item = ScriptedStep>) -> Self {
|
||||||
|
Self {
|
||||||
|
steps: steps.into_iter().collect(),
|
||||||
|
reset_text: String::new(),
|
||||||
|
tool_call_id: Some("call_test".to_string()),
|
||||||
|
finish_error_reset_text: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_finish_error(mut self, reset_text: &str) -> Self {
|
||||||
|
self.finish_error_reset_text = Some(reset_text.to_string());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl vllm_parser::unified::UnifiedParser for ScriptedParser {
|
||||||
|
fn create(
|
||||||
|
_tools: &[vllm_parser::tool::Tool],
|
||||||
|
_tokenizer: vllm_tokenizer::DynTokenizer,
|
||||||
|
) -> vllm_parser::unified::Result<Box<dyn vllm_parser::unified::UnifiedParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static,
|
||||||
|
{
|
||||||
|
unreachable!("ScriptedParser is constructed directly in tests")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_into(
|
||||||
|
&mut self,
|
||||||
|
_delta: &str,
|
||||||
|
output: &mut UnifiedParserOutput,
|
||||||
|
) -> vllm_parser::unified::Result<()> {
|
||||||
|
match self.steps.pop_front().expect("unexpected parser call") {
|
||||||
|
ScriptedStep::Output(next) => {
|
||||||
|
output.append(next);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
ScriptedStep::Error {
|
||||||
|
committed,
|
||||||
|
reset_text,
|
||||||
|
} => {
|
||||||
|
output.append(committed);
|
||||||
|
self.reset_text = reset_text;
|
||||||
|
Err(UnifiedParserError::Reasoning(
|
||||||
|
ReasoningError::MissingToken {
|
||||||
|
token: "<think>".to_string(),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
|
||||||
|
self.tool_call_id.as_deref()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self) -> vllm_parser::unified::Result<UnifiedParserOutput> {
|
||||||
|
if let Some(reset_text) = self.finish_error_reset_text.take() {
|
||||||
|
self.reset_text = reset_text;
|
||||||
|
return Err(UnifiedParserError::Reasoning(
|
||||||
|
ReasoningError::MissingToken {
|
||||||
|
token: "</tool_call>".to_string(),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(UnifiedParserOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) -> String {
|
||||||
|
std::mem::take(&mut self.reset_text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||||
|
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||||
|
delta: delta.to_string(),
|
||||||
|
token_ids: Vec::new(),
|
||||||
|
logprobs: None,
|
||||||
|
finished: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||||
|
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||||
|
delta: delta.to_string(),
|
||||||
|
token_ids: Vec::new(),
|
||||||
|
logprobs: None,
|
||||||
|
finished: Some(vllm_text::output::Finished {
|
||||||
|
usage: vllm_llm::TokenUsage::default(),
|
||||||
|
finish_reason: crate::FinishReason::Stop(None),
|
||||||
|
kv_transfer_params: None,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn collect(
|
||||||
|
parser: ScriptedParser,
|
||||||
|
events: Vec<vllm_text::output::DecodedTextEvent>,
|
||||||
|
) -> Vec<AssistantEvent> {
|
||||||
|
let stream = stream::iter(events.into_iter().map(Ok));
|
||||||
|
unified_event_stream(stream, Box::new(parser))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.collect::<crate::Result<Vec<_>>>()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(delta: &str) -> UnifiedParserOutput {
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
output.push_text(delta.to_string());
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reasoning(delta: &str) -> UnifiedParserOutput {
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
output.push_reasoning(delta.to_string());
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
|
||||||
|
UnifiedParserOutput {
|
||||||
|
events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
|
||||||
|
ToolCallDelta {
|
||||||
|
tool_index: 0,
|
||||||
|
name: Some(name.to_string()),
|
||||||
|
arguments: arguments.to_string(),
|
||||||
|
},
|
||||||
|
)],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn combined(first: UnifiedParserOutput, second: UnifiedParserOutput) -> UnifiedParserOutput {
|
||||||
|
let mut output = first;
|
||||||
|
output.append(second);
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_emits_reasoning_only_deltas() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Output(reasoning("thinking"))]),
|
||||||
|
vec![decoded_delta("raw")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Reasoning,
|
||||||
|
delta: "thinking".to_string(),
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_emits_tool_only_deltas() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Output(tool_call(
|
||||||
|
"get_weather",
|
||||||
|
r#"{"location":"Paris"}"#,
|
||||||
|
))]),
|
||||||
|
vec![decoded_delta("raw")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![
|
||||||
|
AssistantEvent::ToolCallStart {
|
||||||
|
id: "call_test".to_string(),
|
||||||
|
name: "get_weather".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::ToolCallArgumentsDelta {
|
||||||
|
delta: r#"{"location":"Paris"}"#.to_string(),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_emits_reasoning_followed_by_tool_call() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Output(combined(
|
||||||
|
reasoning("thinking"),
|
||||||
|
tool_call("get_weather", r#"{"location":"Paris"}"#),
|
||||||
|
))]),
|
||||||
|
vec![decoded_delta("raw")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![
|
||||||
|
AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Reasoning,
|
||||||
|
delta: "thinking".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::ToolCallStart {
|
||||||
|
id: "call_test".to_string(),
|
||||||
|
name: "get_weather".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::ToolCallArgumentsDelta {
|
||||||
|
delta: r#"{"location":"Paris"}"#.to_string(),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_emits_visible_text_followed_by_tool_call() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Output(combined(
|
||||||
|
text("visible "),
|
||||||
|
tool_call("get_weather", r#"{"location":"Paris"}"#),
|
||||||
|
))]),
|
||||||
|
vec![decoded_delta("raw")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![
|
||||||
|
AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Text,
|
||||||
|
delta: "visible ".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::ToolCallStart {
|
||||||
|
id: "call_test".to_string(),
|
||||||
|
name: "get_weather".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::ToolCallArgumentsDelta {
|
||||||
|
delta: r#"{"location":"Paris"}"#.to_string(),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_fallback_keeps_committed_output_and_disables_later_parsing() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Error {
|
||||||
|
committed: text("committed"),
|
||||||
|
reset_text: "buffered".to_string(),
|
||||||
|
}]),
|
||||||
|
vec![decoded_delta("bad"), decoded_delta("later")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![
|
||||||
|
AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Text,
|
||||||
|
delta: "committed".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Text,
|
||||||
|
delta: "buffered".to_string(),
|
||||||
|
},
|
||||||
|
AssistantEvent::TextDelta {
|
||||||
|
kind: AssistantBlockKind::Text,
|
||||||
|
delta: "later".to_string(),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unified_stream_finish_error_closes_parser_without_reset_text() {
|
||||||
|
let events = collect(
|
||||||
|
ScriptedParser::new([ScriptedStep::Output(UnifiedParserOutput::default())])
|
||||||
|
.with_finish_error("buffered"),
|
||||||
|
vec![finished_delta("")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events,
|
||||||
|
vec![AssistantEvent::Done {
|
||||||
|
usage: vllm_llm::TokenUsage::default(),
|
||||||
|
finish_reason: crate::FinishReason::Stop(None),
|
||||||
|
kv_transfer_params: None,
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,7 +2,6 @@ use std::pin::Pin;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
use subenum::subenum;
|
|
||||||
use trait_set::trait_set;
|
use trait_set::trait_set;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use vllm_llm::TokenUsage;
|
use vllm_llm::TokenUsage;
|
||||||
@@ -22,23 +21,19 @@ pub(crate) use harmony::validate_harmony_parser_overrides;
|
|||||||
|
|
||||||
/// Internal assistant event before final assembly.
|
/// Internal assistant event before final assembly.
|
||||||
///
|
///
|
||||||
/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content.
|
/// Unified parsing produces these events, and structured assembly consumes
|
||||||
/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants.
|
/// them to build public chat events.
|
||||||
#[subenum(ContentEvent)]
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub(crate) enum AssistantEvent {
|
pub(crate) enum AssistantEvent {
|
||||||
#[subenum(ContentEvent)]
|
|
||||||
Start {
|
Start {
|
||||||
prompt_token_ids: Arc<[u32]>,
|
prompt_token_ids: Arc<[u32]>,
|
||||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||||
},
|
},
|
||||||
#[subenum(ContentEvent)]
|
|
||||||
TextDelta {
|
TextDelta {
|
||||||
kind: AssistantBlockKind,
|
kind: AssistantBlockKind,
|
||||||
delta: String,
|
delta: String,
|
||||||
},
|
},
|
||||||
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
||||||
#[subenum(ContentEvent)]
|
|
||||||
LogprobsDelta {
|
LogprobsDelta {
|
||||||
logprobs: Option<DecodedLogprobs>,
|
logprobs: Option<DecodedLogprobs>,
|
||||||
token_ids: Vec<u32>,
|
token_ids: Vec<u32>,
|
||||||
@@ -48,7 +43,6 @@ pub(crate) enum AssistantEvent {
|
|||||||
/// A delta for the arguments of the currently open tool call. Must follow a
|
/// A delta for the arguments of the currently open tool call. Must follow a
|
||||||
/// `ToolCallStart`.
|
/// `ToolCallStart`.
|
||||||
ToolCallArgumentsDelta { delta: String },
|
ToolCallArgumentsDelta { delta: String },
|
||||||
#[subenum(ContentEvent)]
|
|
||||||
Done {
|
Done {
|
||||||
usage: TokenUsage,
|
usage: TokenUsage,
|
||||||
finish_reason: FinishReason,
|
finish_reason: FinishReason,
|
||||||
@@ -57,50 +51,6 @@ pub(crate) enum AssistantEvent {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ContentEvent {
|
|
||||||
/// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values
|
|
||||||
/// by treating all text as plain (non-reasoning) content.
|
|
||||||
fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec<Self> {
|
|
||||||
match event {
|
|
||||||
DecodedTextEvent::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
} => vec![Self::Start {
|
|
||||||
prompt_token_ids,
|
|
||||||
prompt_logprobs,
|
|
||||||
}],
|
|
||||||
DecodedTextEvent::TextDelta {
|
|
||||||
delta,
|
|
||||||
token_ids,
|
|
||||||
logprobs,
|
|
||||||
finished,
|
|
||||||
} => {
|
|
||||||
let mut events = Vec::new();
|
|
||||||
if !delta.is_empty() {
|
|
||||||
events.push(Self::TextDelta {
|
|
||||||
kind: AssistantBlockKind::Text,
|
|
||||||
delta,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if logprobs.is_some() || !token_ids.is_empty() {
|
|
||||||
events.push(Self::LogprobsDelta {
|
|
||||||
logprobs,
|
|
||||||
token_ids,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if let Some(finished) = finished {
|
|
||||||
events.push(Self::Done {
|
|
||||||
usage: finished.usage,
|
|
||||||
finish_reason: finished.finish_reason,
|
|
||||||
kv_transfer_params: finished.kv_transfer_params,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
events
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Boxed stream of decoded text events coming from [`vllm_text`].
|
/// Boxed stream of decoded text events coming from [`vllm_text`].
|
||||||
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
|
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
|
||||||
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
|
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
pub use vllm_reasoning_parser::{
|
pub use vllm_parser::reasoning::{
|
||||||
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
||||||
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
|
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
|
||||||
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
|
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
|
||||||
@@ -34,7 +34,7 @@ pub mod names {
|
|||||||
|
|
||||||
/// Constructor signature for one registered reasoning parser implementation.
|
/// Constructor signature for one registered reasoning parser implementation.
|
||||||
type ReasoningParserCreator =
|
type ReasoningParserCreator =
|
||||||
fn(DynTokenizer) -> vllm_reasoning_parser::Result<Box<dyn ReasoningParser>>;
|
fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>>;
|
||||||
|
|
||||||
/// Registry and model matcher for reasoning parsers.
|
/// Registry and model matcher for reasoning parsers.
|
||||||
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
|
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
|
||||||
|
|||||||
@@ -2,13 +2,12 @@
|
|||||||
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
pub use vllm_tool_parser::{
|
pub use vllm_parser::tool::{
|
||||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||||
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
||||||
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
|
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
|
||||||
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
|
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
|
||||||
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError,
|
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError,
|
||||||
ToolParserOutput,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::parser::ParserFactory;
|
use crate::parser::ParserFactory;
|
||||||
@@ -41,7 +40,7 @@ pub mod names {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Constructor signature for one registered tool parser implementation.
|
/// Constructor signature for one registered tool parser implementation.
|
||||||
type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>;
|
type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>>;
|
||||||
|
|
||||||
/// Registry and model matcher for tool parsers.
|
/// Registry and model matcher for tool parsers.
|
||||||
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
|
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use vllm_tool_parser::Result;
|
use vllm_parser::tool::{Result, ToolParserOutput};
|
||||||
|
|
||||||
use super::{ToolParser, ToolParserFactory, ToolParserOutput, names};
|
use super::{ToolParser, ToolParserFactory, names};
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::request::ChatTool;
|
use crate::request::ChatTool;
|
||||||
|
|
||||||
|
|||||||
@@ -4,9 +4,9 @@ use llm_multimodal::ImageDetail;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||||
|
pub use vllm_parser::tool::Tool as ChatTool;
|
||||||
pub use vllm_text::SamplingParams;
|
pub use vllm_text::SamplingParams;
|
||||||
use vllm_text::TextDecodeOptions;
|
use vllm_text::TextDecodeOptions;
|
||||||
pub use vllm_tool_parser::Tool as ChatTool;
|
|
||||||
|
|
||||||
use crate::AssistantMessageExt;
|
use crate::AssistantMessageExt;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
|
|||||||
@@ -1180,7 +1180,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
|
|||||||
),
|
),
|
||||||
request_output(
|
request_output(
|
||||||
"chat-tool",
|
"chat-tool",
|
||||||
bytes_to_token_ids(
|
bytes_with_special_stop_token(
|
||||||
b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>",
|
b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>",
|
||||||
),
|
),
|
||||||
Some(EngineCoreFinishReason::Stop),
|
Some(EngineCoreFinishReason::Stop),
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "vllm-tool-parser"
|
name = "vllm-parser"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
@@ -13,6 +13,7 @@ serde.workspace = true
|
|||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
thiserror-ext.workspace = true
|
thiserror-ext.workspace = true
|
||||||
|
vllm-tokenizer.workspace = true
|
||||||
winnow.workspace = true
|
winnow.workspace = true
|
||||||
xgrammar-structural-tag.workspace = true
|
xgrammar-structural-tag.workspace = true
|
||||||
|
|
||||||
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
|
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{DeepSeekV3ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
|
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{DeepSeekV31ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
+2
-2
@@ -1,8 +1,8 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{DeepSeekV32ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::feed_parser;
|
use utils::feed_parser;
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{Gemma4ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::feed_parser;
|
use utils::feed_parser;
|
||||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
|
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
|
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{KimiK2ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
|
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{Llama3JsonToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
|
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{MinimaxM2ToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
|
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{Qwen3CoderToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||||
use tool_parser::parsers::QwenParser as ExternalQwenParser;
|
use tool_parser::parsers::QwenParser as ExternalQwenParser;
|
||||||
use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
|
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||||
use vllm_tool_parser::{Qwen3XmlToolParser, Tool, ToolParser};
|
use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser};
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||||
@@ -3,8 +3,8 @@
|
|||||||
use futures::FutureExt as _;
|
use futures::FutureExt as _;
|
||||||
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
|
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
|
||||||
use tool_parser::traits::ToolParser as ExternalToolParser;
|
use tool_parser::traits::ToolParser as ExternalToolParser;
|
||||||
use vllm_tool_parser::test_utils::collect_stream;
|
use vllm_parser::tool::test_utils::collect_stream;
|
||||||
use vllm_tool_parser::{Tool, ToolParser};
|
use vllm_parser::tool::{Tool, ToolParser};
|
||||||
|
|
||||||
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
|
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
|
||||||
tools
|
tools
|
||||||
@@ -13,7 +13,7 @@ pyo3.workspace = true
|
|||||||
pythonize = { workspace = true, features = ["serde_json"] }
|
pythonize = { workspace = true, features = ["serde_json"] }
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror-ext.workspace = true
|
thiserror-ext.workspace = true
|
||||||
vllm-tool-parser.workspace = true
|
vllm-parser.workspace = true
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Thin PyO3 bindings for `vllm_tool_parser`.
|
//! Thin PyO3 bindings for `vllm_parser::tool`.
|
||||||
//!
|
//!
|
||||||
//! This crate exposes the Rust tool parser trait and data shapes to Python
|
//! This crate exposes the Rust tool parser trait and data shapes to Python
|
||||||
//! while keeping parser state, grammar, and schema-aware argument conversion in
|
//! while keeping parser state, grammar, and schema-aware argument conversion in
|
||||||
@@ -11,7 +11,7 @@ use pyo3::types::{PyAny, PyModule};
|
|||||||
use pythonize::{depythonize, pythonize};
|
use pythonize::{depythonize, pythonize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use thiserror_ext::AsReport as _;
|
use thiserror_ext::AsReport as _;
|
||||||
use vllm_tool_parser::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
use vllm_parser::tool::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
macro_rules! tool_parser_factory {
|
macro_rules! tool_parser_factory {
|
||||||
($($parser:ident),+ $(,)?) => {
|
($($parser:ident),+ $(,)?) => {
|
||||||
@@ -22,7 +22,7 @@ macro_rules! tool_parser_factory {
|
|||||||
match name {
|
match name {
|
||||||
$(
|
$(
|
||||||
stringify!($parser) => {
|
stringify!($parser) => {
|
||||||
<vllm_tool_parser::$parser as ToolParser>::create(tools)
|
<vllm_parser::tool::$parser as ToolParser>::create(tools)
|
||||||
}
|
}
|
||||||
)+
|
)+
|
||||||
_ => {
|
_ => {
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
//! Streaming parsers for chat completions.
|
||||||
|
|
||||||
|
pub mod reasoning;
|
||||||
|
pub mod tool;
|
||||||
|
pub mod unified;
|
||||||
@@ -119,7 +119,7 @@ mod tests {
|
|||||||
use vllm_tokenizer::Tokenizer;
|
use vllm_tokenizer::Tokenizer;
|
||||||
|
|
||||||
use super::Gemma4ReasoningParser;
|
use super::Gemma4ReasoningParser;
|
||||||
use crate::ReasoningParser;
|
use crate::reasoning::ReasoningParser;
|
||||||
|
|
||||||
struct FakeTokenizer;
|
struct FakeTokenizer;
|
||||||
|
|
||||||
+1
-1
@@ -49,7 +49,7 @@ mod tests {
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::SeedOssReasoningParser;
|
use super::SeedOssReasoningParser;
|
||||||
use crate::{ReasoningParser, tests::FakeTokenizer};
|
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn without_prompt_markers_expects_start_token() {
|
fn without_prompt_markers_expects_start_token() {
|
||||||
@@ -127,7 +127,7 @@ mod tests {
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::Step3p5ReasoningParser;
|
use super::Step3p5ReasoningParser;
|
||||||
use crate::{ReasoningParser, tests::FakeTokenizer};
|
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn picks_up_prompt_start_boundary() {
|
fn picks_up_prompt_start_boundary() {
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for DeepSeek V3.2 models.
|
/// Tool parser for DeepSeek V3.2 models.
|
||||||
///
|
///
|
||||||
@@ -67,8 +67,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::DeepSeekV32ToolParser;
|
use super::DeepSeekV32ToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for DeepSeek V4 models.
|
/// Tool parser for DeepSeek V4 models.
|
||||||
///
|
///
|
||||||
@@ -69,8 +69,8 @@ mod tests {
|
|||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use super::DeepSeekV4ToolParser;
|
use super::DeepSeekV4ToolParser;
|
||||||
use crate::test_utils::{collect_stream, test_tools};
|
use crate::tool::test_utils::{collect_stream, test_tools};
|
||||||
use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
|
use crate::tool::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
+1
-1
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
|
|||||||
use super::parameters::ToolSchemas;
|
use super::parameters::ToolSchemas;
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, ToolCallDelta, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParserOutput};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
mod deepseek_v32;
|
mod deepseek_v32;
|
||||||
mod deepseek_v4;
|
mod deepseek_v4;
|
||||||
+4
-4
@@ -1,5 +1,5 @@
|
|||||||
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
|
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
|
||||||
///
|
///
|
||||||
@@ -55,12 +55,12 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::DeepSeekV3ToolParser;
|
use super::DeepSeekV3ToolParser;
|
||||||
use crate::deepseek_json::{
|
use crate::tool::deepseek_json::{
|
||||||
TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, V3_ARGUMENT_END,
|
TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, V3_ARGUMENT_END,
|
||||||
V3_JSON_START,
|
V3_JSON_START,
|
||||||
};
|
};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn v3_tool_call(function_name: &str, arguments: &str) -> String {
|
fn v3_tool_call(function_name: &str, arguments: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
+4
-4
@@ -1,5 +1,5 @@
|
|||||||
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
|
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
|
||||||
///
|
///
|
||||||
@@ -51,11 +51,11 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::DeepSeekV31ToolParser;
|
use super::DeepSeekV31ToolParser;
|
||||||
use crate::deepseek_json::{
|
use crate::tool::deepseek_json::{
|
||||||
TOOL_CALL_END, TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START,
|
TOOL_CALL_END, TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START,
|
||||||
};
|
};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn v31_tool_call(function_name: &str, arguments: &str) -> String {
|
fn v31_tool_call(function_name: &str, arguments: &str) -> String {
|
||||||
format!("{TOOL_CALL_START}{function_name}{TOOL_CALL_SEPARATOR}{arguments}{TOOL_CALL_END}")
|
format!("{TOOL_CALL_START}{function_name}{TOOL_CALL_SEPARATOR}{arguments}{TOOL_CALL_END}")
|
||||||
@@ -6,7 +6,7 @@ pub type Result<T> = std::result::Result<T, ToolParserError>;
|
|||||||
|
|
||||||
/// Errors produced while creating or running tool parsers.
|
/// Errors produced while creating or running tool parsers.
|
||||||
#[derive(Debug, Error, Macro)]
|
#[derive(Debug, Error, Macro)]
|
||||||
#[thiserror_ext(macro(path = "crate::error"))]
|
#[thiserror_ext(macro(path = "crate::tool::error"))]
|
||||||
pub enum ToolParserError {
|
pub enum ToolParserError {
|
||||||
#[error("tool parser parsing failed: {message}")]
|
#[error("tool parser parsing failed: {message}")]
|
||||||
ParsingFailed { message: String },
|
ParsingFailed { message: String },
|
||||||
@@ -8,7 +8,7 @@ use winnow::token::{literal, take_till, take_until};
|
|||||||
|
|
||||||
use super::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_text_len};
|
use super::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_text_len};
|
||||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
const TOOL_CALL_START: &str = "<|tool_call>";
|
const TOOL_CALL_START: &str = "<|tool_call>";
|
||||||
const TOOL_CALL_END: &str = "<tool_call|>";
|
const TOOL_CALL_END: &str = "<tool_call|>";
|
||||||
@@ -428,7 +428,7 @@ mod tests {
|
|||||||
Gemma4ToolParser, ToolCallDelta, ToolParser, ToolParserOutput, gemma4_array_content,
|
Gemma4ToolParser, ToolCallDelta, ToolParser, ToolParserOutput, gemma4_array_content,
|
||||||
parse_gemma4_args,
|
parse_gemma4_args,
|
||||||
};
|
};
|
||||||
use crate::{Tool, ToolParserTestExt as _};
|
use crate::tool::{Tool, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn parse_gemma4_array(array: &str) -> super::Result<Vec<Value>> {
|
fn parse_gemma4_array(array: &str) -> super::Result<Vec<Value>> {
|
||||||
let mut input = array;
|
let mut input = array;
|
||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
use super::{GlmXmlToolParser, Separator};
|
use super::{GlmXmlToolParser, Separator};
|
||||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for GLM-4.5/4.6 MoE XML-style tool calls.
|
/// Tool parser for GLM-4.5/4.6 MoE XML-style tool calls.
|
||||||
///
|
///
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
use super::{GlmXmlToolParser, Separator};
|
use super::{GlmXmlToolParser, Separator};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
||||||
///
|
///
|
||||||
@@ -44,8 +44,8 @@ mod tests {
|
|||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use super::Glm47MoeToolParser;
|
use super::Glm47MoeToolParser;
|
||||||
use crate::ToolParserTestExt as _;
|
use crate::tool::ToolParserTestExt as _;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
|
|
||||||
fn glm47_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn glm47_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until, take_while};
|
|||||||
use super::parameters::ToolSchemas;
|
use super::parameters::ToolSchemas;
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, ToolCallDelta, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParserOutput};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
mod glm45_moe;
|
mod glm45_moe;
|
||||||
mod glm47_moe;
|
mod glm47_moe;
|
||||||
@@ -264,8 +264,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Glm45MoeToolParser;
|
use super::Glm45MoeToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn glm45_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn glm45_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
|
|||||||
use super::parameters::ToolSchemas;
|
use super::parameters::ToolSchemas;
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::{StructuralTagModel, Tool};
|
use crate::tool::{StructuralTagModel, Tool};
|
||||||
|
|
||||||
const TOOL_CALLS_START: &str = "<tool_calls>";
|
const TOOL_CALLS_START: &str = "<tool_calls>";
|
||||||
const TOOL_CALLS_END: &str = "</tool_calls>";
|
const TOOL_CALLS_END: &str = "</tool_calls>";
|
||||||
@@ -249,8 +249,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::{HyV3ToolParser, ToolParser};
|
use super::{HyV3ToolParser, ToolParser};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
+4
-4
@@ -8,10 +8,10 @@ use super::{
|
|||||||
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
|
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
|
||||||
tool_call_header_event,
|
tool_call_header_event,
|
||||||
};
|
};
|
||||||
use crate::utils::{
|
use crate::tool::utils::{
|
||||||
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
|
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
|
||||||
};
|
};
|
||||||
use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const TOOL_CALL_START: &str = "<tool_call>";
|
const TOOL_CALL_START: &str = "<tool_call>";
|
||||||
const TOOL_CALL_END: &str = "</tool_call>";
|
const TOOL_CALL_END: &str = "</tool_call>";
|
||||||
@@ -279,8 +279,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Granite4ToolParser;
|
use super::Granite4ToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn granite4_parse_complete_without_tool_call_keeps_text() {
|
fn granite4_parse_complete_without_tool_call_keeps_text() {
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "Hermes",
|
parser_name: "Hermes",
|
||||||
@@ -68,8 +68,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::HermesToolParser;
|
use super::HermesToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
||||||
format!(r#"<tool_call>{{"name":"{function_name}","arguments":{arguments}}}</tool_call>"#)
|
format!(r#"<tool_call>{{"name":"{function_name}","arguments":{arguments}}}</tool_call>"#)
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "InternLM2",
|
parser_name: "InternLM2",
|
||||||
@@ -123,8 +123,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Internlm2ToolParser;
|
use super::Internlm2ToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
const ACTION_START: &str = "<|action_start|><|plugin|>";
|
const ACTION_START: &str = "<|action_start|><|plugin|>";
|
||||||
const ACTION_END: &str = "<|action_end|>";
|
const ACTION_END: &str = "<|action_end|>";
|
||||||
@@ -8,8 +8,8 @@ use super::{
|
|||||||
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
|
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
|
||||||
argument_delta_event, tool_call_header_event,
|
argument_delta_event, tool_call_header_event,
|
||||||
};
|
};
|
||||||
use crate::utils::{JsonObjectScanState, parse_buffered_event};
|
use crate::tool::utils::{JsonObjectScanState, parse_buffered_event};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
enum LlamaJsonMode {
|
enum LlamaJsonMode {
|
||||||
@@ -256,8 +256,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Llama3JsonToolParser;
|
use super::Llama3JsonToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, parameters: &str) -> String {
|
fn build_tool_call(function_name: &str, parameters: &str) -> String {
|
||||||
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
|
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "Mistral",
|
parser_name: "Mistral",
|
||||||
@@ -61,8 +61,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::MistralToolParser;
|
use super::MistralToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
||||||
format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#)
|
format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#)
|
||||||
@@ -374,7 +374,7 @@ mod tests {
|
|||||||
use expect_test::expect;
|
use expect_test::expect;
|
||||||
|
|
||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::ToolParserOutput;
|
use crate::tool::ToolParserOutput;
|
||||||
|
|
||||||
const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "Delimited JSON",
|
parser_name: "Delimited JSON",
|
||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "Phi4Mini",
|
parser_name: "Phi4Mini",
|
||||||
@@ -69,8 +69,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Phi4MiniJsonToolParser;
|
use super::Phi4MiniJsonToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserTestExt as _};
|
||||||
|
|
||||||
/// Build one phi-4-mini tool-call object: `{"name":..,"<args_key>":<args>}`.
|
/// Build one phi-4-mini tool-call object: `{"name":..,"<args_key>":<args>}`.
|
||||||
fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String {
|
fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String {
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||||
parser_name: "Qwen XML",
|
parser_name: "Qwen XML",
|
||||||
@@ -70,8 +70,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::Qwen3XmlToolParser;
|
use super::Qwen3XmlToolParser;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
fn build_tool_call(function_name: &str, arguments: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
|
|||||||
|
|
||||||
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
|
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
|
||||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::{StructuralTagModel, Tool};
|
use crate::tool::{StructuralTagModel, Tool};
|
||||||
|
|
||||||
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
|
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
|
||||||
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
|
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
|
||||||
@@ -339,8 +339,8 @@ mod tests {
|
|||||||
KimiK2ToolParser, TOOL_CALL_ARGUMENT_START, TOOL_CALL_END, TOOL_CALL_START, TOOL_CALLS_END,
|
KimiK2ToolParser, TOOL_CALL_ARGUMENT_START, TOOL_CALL_END, TOOL_CALL_START, TOOL_CALLS_END,
|
||||||
TOOL_CALLS_START, ToolParser, tool_header,
|
TOOL_CALLS_START, ToolParser, tool_header,
|
||||||
};
|
};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, index: usize, arguments: &str) -> String {
|
fn build_tool_call(function_name: &str, index: usize, arguments: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
|
|||||||
use super::parameters::ToolSchemas;
|
use super::parameters::ToolSchemas;
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::{StructuralTagModel, Tool};
|
use crate::tool::{StructuralTagModel, Tool};
|
||||||
|
|
||||||
const TOOL_CALL_START: &str = "<minimax:tool_call>";
|
const TOOL_CALL_START: &str = "<minimax:tool_call>";
|
||||||
const TOOL_CALL_END: &str = "</minimax:tool_call>";
|
const TOOL_CALL_END: &str = "</minimax:tool_call>";
|
||||||
@@ -272,8 +272,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
|
use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
|
||||||
use crate::ToolParserTestExt as _;
|
use crate::tool::ToolParserTestExt as _;
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
|
|
||||||
fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
|
fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
|
||||||
let invokes = invokes
|
let invokes = invokes
|
||||||
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until};
|
|||||||
use super::parameters::{ParamElement, ParamInput, ToolSchemas};
|
use super::parameters::{ParamElement, ParamInput, ToolSchemas};
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
const NAMESPACE: &str = "]<]minimax[>[";
|
const NAMESPACE: &str = "]<]minimax[>[";
|
||||||
const TOOL_CALL_START: &str = "]<]minimax[>[<tool_call>";
|
const TOOL_CALL_START: &str = "]<]minimax[>[<tool_call>";
|
||||||
@@ -388,8 +388,8 @@ mod tests {
|
|||||||
ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser,
|
ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser,
|
||||||
TOOL_CALL_END, TOOL_CALL_START, ToolParser,
|
TOOL_CALL_END, TOOL_CALL_START, ToolParser,
|
||||||
};
|
};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{Tool, ToolParserTestExt as _};
|
use crate::tool::{Tool, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn element(name: &str, body: &str) -> String {
|
fn element(name: &str, body: &str) -> String {
|
||||||
format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>")
|
format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>")
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
//! Streaming tool parsers for chat completions.
|
//! Streaming tool parsers for chat completions.
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
mod error;
|
pub(crate) mod error;
|
||||||
mod deepseek_dsml;
|
mod deepseek_dsml;
|
||||||
mod deepseek_json;
|
pub(crate) mod deepseek_json;
|
||||||
mod gemma4;
|
mod gemma4;
|
||||||
mod glm_xml;
|
mod glm_xml;
|
||||||
mod hy_v3;
|
mod hy_v3;
|
||||||
@@ -15,7 +15,7 @@ mod parameters;
|
|||||||
mod qwen_coder;
|
mod qwen_coder;
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
pub mod test_utils;
|
pub mod test_utils;
|
||||||
mod utils;
|
pub(crate) mod utils;
|
||||||
|
|
||||||
use std::collections::{BTreeMap, btree_map};
|
use std::collections::{BTreeMap, btree_map};
|
||||||
|
|
||||||
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
|
|||||||
|
|
||||||
use serde_json::{Map, Number, Value};
|
use serde_json::{Map, Number, Value};
|
||||||
|
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
/// Normalized parameter schemas for all tools in one request.
|
/// Normalized parameter schemas for all tools in one request.
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
@@ -416,7 +416,7 @@ mod tests {
|
|||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use super::{ParamElement, ParamInput, ToolSchema, ToolSchemas};
|
use super::{ParamElement, ParamInput, ToolSchema, ToolSchemas};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
fn test_tool(name: &str, parameters: serde_json::Value) -> Tool {
|
fn test_tool(name: &str, parameters: serde_json::Value) -> Tool {
|
||||||
Tool {
|
Tool {
|
||||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, take_until};
|
|||||||
use super::parameters::ToolSchemas;
|
use super::parameters::ToolSchemas;
|
||||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||||
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::Tool;
|
use crate::tool::Tool;
|
||||||
|
|
||||||
const TOOL_CALL_START: &str = "<tool_call>";
|
const TOOL_CALL_START: &str = "<tool_call>";
|
||||||
const TOOL_CALL_END: &str = "</tool_call>";
|
const TOOL_CALL_END: &str = "</tool_call>";
|
||||||
@@ -241,8 +241,8 @@ mod tests {
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
|
|
||||||
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
|
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
|
||||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||||
use crate::{ToolParserOutput, ToolParserTestExt as _};
|
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
|
||||||
|
|
||||||
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||||
let params = params
|
let params = params
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use super::{ToolParser, ToolParserOutput};
|
use super::{ToolParser, ToolParserOutput};
|
||||||
use crate::{Tool, ToolParserTestExt as _};
|
use crate::tool::{Tool, ToolParserTestExt as _};
|
||||||
|
|
||||||
/// Build a reusable set of function tools for parser unit tests.
|
/// Build a reusable set of function tools for parser unit tests.
|
||||||
pub fn test_tools() -> Vec<Tool> {
|
pub fn test_tools() -> Vec<Tool> {
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use super::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
use super::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||||
use crate::ToolParserTestExt as _;
|
use crate::tool::ToolParserTestExt as _;
|
||||||
|
|
||||||
struct DefaultParser;
|
struct DefaultParser;
|
||||||
|
|
||||||
@@ -0,0 +1,346 @@
|
|||||||
|
//! Adapter that combines reasoning and tool parsers.
|
||||||
|
|
||||||
|
use vllm_tokenizer::DynTokenizer;
|
||||||
|
|
||||||
|
use crate::reasoning::ReasoningParser;
|
||||||
|
use crate::tool::{StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
|
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||||
|
|
||||||
|
/// Unified parser that composes existing reasoning and tool parsers.
|
||||||
|
pub struct CombinedParser {
|
||||||
|
reasoning: Option<Box<dyn ReasoningParser>>,
|
||||||
|
tool: Option<Box<dyn ToolParser>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CombinedParser {
|
||||||
|
/// Create a combined parser from optional reasoning and tool parsers.
|
||||||
|
pub fn new(
|
||||||
|
reasoning: Option<Box<dyn ReasoningParser>>,
|
||||||
|
tool: Option<Box<dyn ToolParser>>,
|
||||||
|
) -> Self {
|
||||||
|
Self { reasoning, tool }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a text-only combined parser.
|
||||||
|
pub fn plain_text_only() -> Self {
|
||||||
|
Self {
|
||||||
|
reasoning: None,
|
||||||
|
tool: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_tool(&mut self, content: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||||
|
let Some(tool) = self.tool.as_mut() else {
|
||||||
|
output.push_text(content.to_string());
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
|
||||||
|
// Preserve any tool output that was already produced before the error.
|
||||||
|
let mut tool_output = ToolParserOutput::default();
|
||||||
|
let result = tool.parse_into(content, &mut tool_output);
|
||||||
|
output.append_tool_output(tool_output);
|
||||||
|
result?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush_tool(&mut self) -> Result<UnifiedParserOutput> {
|
||||||
|
let Some(tool) = self.tool.as_mut() else {
|
||||||
|
return Ok(UnifiedParserOutput::default());
|
||||||
|
};
|
||||||
|
|
||||||
|
let output = tool.finish()?;
|
||||||
|
let mut unified = UnifiedParserOutput::default();
|
||||||
|
unified.append_tool_output(output);
|
||||||
|
Ok(unified)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UnifiedParser for CombinedParser {
|
||||||
|
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static,
|
||||||
|
{
|
||||||
|
Err(UnifiedParserError::CombinedParserConstructor)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||||
|
if let Some(reasoning) = self.reasoning.as_mut() {
|
||||||
|
reasoning.initialize(prompt_token_ids)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn preserve_special_tokens(&self) -> bool {
|
||||||
|
self.reasoning.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
||||||
|
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||||
|
self.tool.as_ref().and_then(|parser| parser.structural_tag_model())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||||
|
self.tool.as_ref().and_then(|parser| parser.tool_call_id(tool_index))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||||
|
let Some(reasoning) = self.reasoning.as_mut() else {
|
||||||
|
return self.parse_tool(delta, output);
|
||||||
|
};
|
||||||
|
|
||||||
|
let reasoning_delta = reasoning.push(delta)?;
|
||||||
|
if let Some(reasoning) = reasoning_delta.reasoning {
|
||||||
|
output.push_reasoning(reasoning);
|
||||||
|
}
|
||||||
|
if let Some(content) = reasoning_delta.content {
|
||||||
|
self.parse_tool(&content, output)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
if let Some(reasoning) = self.reasoning.as_mut() {
|
||||||
|
let reasoning_delta = reasoning.finish()?;
|
||||||
|
if let Some(reasoning) = reasoning_delta.reasoning {
|
||||||
|
output.push_reasoning(reasoning);
|
||||||
|
}
|
||||||
|
if let Some(content) = reasoning_delta.content {
|
||||||
|
self.parse_tool(&content, &mut output)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output.append(self.flush_tool()?);
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) -> String {
|
||||||
|
self.tool.as_mut().map_or_else(String::new, |parser| parser.reset())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use vllm_tokenizer::Tokenizer;
|
||||||
|
|
||||||
|
use super::CombinedParser;
|
||||||
|
use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
|
||||||
|
use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
|
||||||
|
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||||
|
|
||||||
|
struct FakeTokenizer;
|
||||||
|
|
||||||
|
impl Tokenizer for FakeTokenizer {
|
||||||
|
fn encode(
|
||||||
|
&self,
|
||||||
|
text: &str,
|
||||||
|
_add_special_tokens: bool,
|
||||||
|
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||||
|
Ok(text.chars().map(u32::from).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode(
|
||||||
|
&self,
|
||||||
|
token_ids: &[u32],
|
||||||
|
_skip_special_tokens: bool,
|
||||||
|
) -> vllm_tokenizer::Result<String> {
|
||||||
|
Ok(token_ids
|
||||||
|
.iter()
|
||||||
|
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||||
|
match token {
|
||||||
|
"<think>" => Some(1),
|
||||||
|
"</think>" => Some(2),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_tools() -> Vec<Tool> {
|
||||||
|
vec![Tool {
|
||||||
|
name: "get_weather".to_string(),
|
||||||
|
description: None,
|
||||||
|
parameters: serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"location": { "type": "string" }
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
strict: None,
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect(parser: &mut dyn UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput {
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
for chunk in chunks {
|
||||||
|
parser.parse_into(chunk, &mut output).unwrap();
|
||||||
|
}
|
||||||
|
output.append(parser.finish().unwrap());
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PreserveReasoningParser;
|
||||||
|
|
||||||
|
impl ReasoningParser for PreserveReasoningParser {
|
||||||
|
fn create(
|
||||||
|
_tokenizer: vllm_tokenizer::DynTokenizer,
|
||||||
|
) -> crate::reasoning::Result<Box<dyn ReasoningParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static,
|
||||||
|
{
|
||||||
|
Ok(Box::new(Self))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn preserve_special_tokens(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(&mut self, delta: &str) -> crate::reasoning::Result<ReasoningDelta> {
|
||||||
|
Ok(ReasoningDelta {
|
||||||
|
reasoning: None,
|
||||||
|
content: Some(delta.to_string()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PreserveToolParser;
|
||||||
|
|
||||||
|
impl ToolParser for PreserveToolParser {
|
||||||
|
fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static,
|
||||||
|
{
|
||||||
|
Ok(Box::new(Self))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn preserve_special_tokens(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_into(
|
||||||
|
&mut self,
|
||||||
|
chunk: &str,
|
||||||
|
output: &mut crate::tool::ToolParserOutput,
|
||||||
|
) -> crate::tool::Result<()> {
|
||||||
|
output.normal_text.push_str(chunk);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
|
||||||
|
Ok(crate::tool::ToolParserOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) -> String {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PartialThenErrorToolParser;
|
||||||
|
|
||||||
|
impl ToolParser for PartialThenErrorToolParser {
|
||||||
|
fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static,
|
||||||
|
{
|
||||||
|
Ok(Box::new(Self))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_into(
|
||||||
|
&mut self,
|
||||||
|
_chunk: &str,
|
||||||
|
output: &mut crate::tool::ToolParserOutput,
|
||||||
|
) -> crate::tool::Result<()> {
|
||||||
|
output.normal_text.push_str("committed");
|
||||||
|
Err(crate::tool::ToolParserError::ParsingFailed {
|
||||||
|
message: "synthetic failure".to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
|
||||||
|
Ok(crate::tool::ToolParserOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) -> String {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn combined_parser_emits_reasoning_and_text() {
|
||||||
|
let tokenizer = Arc::new(FakeTokenizer);
|
||||||
|
let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
|
||||||
|
let mut parser = CombinedParser::new(Some(reasoning), None);
|
||||||
|
|
||||||
|
let output = collect(&mut parser, &["<think>work</think>answer"]);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
output.events,
|
||||||
|
vec![
|
||||||
|
UnifiedParserEvent::Reasoning("work".to_string()),
|
||||||
|
UnifiedParserEvent::Text("answer".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn combined_parser_emits_tool_calls_from_visible_content() {
|
||||||
|
let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap();
|
||||||
|
let mut parser = CombinedParser::new(None, Some(tool));
|
||||||
|
assert!(matches!(
|
||||||
|
parser.structural_tag_model(),
|
||||||
|
Some(crate::tool::StructuralTagModel::Qwen3)
|
||||||
|
));
|
||||||
|
|
||||||
|
let output = collect(
|
||||||
|
&mut parser,
|
||||||
|
&[r#"<tool_call>
|
||||||
|
{"name":"get_weather","arguments":{"location":"Paris"}}
|
||||||
|
</tool_call>"#],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
output.events,
|
||||||
|
vec![
|
||||||
|
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
|
||||||
|
tool_index: 0,
|
||||||
|
name: Some("get_weather".to_string()),
|
||||||
|
arguments: String::new(),
|
||||||
|
}),
|
||||||
|
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
|
||||||
|
tool_index: 0,
|
||||||
|
name: None,
|
||||||
|
arguments: r#"{"location":"Paris"}"#.to_string(),
|
||||||
|
}),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn combined_parser_preserves_tool_output_on_parse_error() {
|
||||||
|
let mut parser = CombinedParser::new(None, Some(Box::new(PartialThenErrorToolParser)));
|
||||||
|
let mut output = UnifiedParserOutput::default();
|
||||||
|
|
||||||
|
let error = parser.parse_into("bad", &mut output).unwrap_err();
|
||||||
|
|
||||||
|
assert!(matches!(error, crate::unified::UnifiedParserError::Tool(_)));
|
||||||
|
assert_eq!(
|
||||||
|
output.events,
|
||||||
|
vec![UnifiedParserEvent::Text("committed".to_string())]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn combined_parser_preserves_special_tokens_when_either_inner_parser_needs_it() {
|
||||||
|
let mut parser = CombinedParser::new(Some(Box::new(PreserveReasoningParser)), None);
|
||||||
|
assert!(parser.preserve_special_tokens());
|
||||||
|
|
||||||
|
parser = CombinedParser::new(None, Some(Box::new(PreserveToolParser)));
|
||||||
|
assert!(parser.preserve_special_tokens());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
//! Unified parser interface for reasoning and tool-call deltas.
|
||||||
|
|
||||||
|
mod combined;
|
||||||
|
|
||||||
|
use thiserror::Error;
|
||||||
|
use vllm_tokenizer::DynTokenizer;
|
||||||
|
|
||||||
|
pub use combined::CombinedParser;
|
||||||
|
|
||||||
|
use crate::reasoning::ReasoningError;
|
||||||
|
use crate::tool::{StructuralTagModel, Tool, ToolCallDelta, ToolParserError, ToolParserOutput};
|
||||||
|
|
||||||
|
/// Result alias for unified parser operations.
|
||||||
|
pub type Result<T> = std::result::Result<T, UnifiedParserError>;
|
||||||
|
|
||||||
|
/// One parsed event emitted by a unified parser.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum UnifiedParserEvent {
|
||||||
|
/// Normal assistant-visible text.
|
||||||
|
Text(String),
|
||||||
|
/// Reasoning text hidden from the normal content stream.
|
||||||
|
Reasoning(String),
|
||||||
|
/// A tool-call update extracted from visible assistant text.
|
||||||
|
ToolCall(ToolCallDelta),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of advancing unified parsing with one assistant-text input.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct UnifiedParserOutput {
|
||||||
|
/// Ordered parser events committed by this input.
|
||||||
|
pub events: Vec<UnifiedParserEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UnifiedParserOutput {
|
||||||
|
/// Append one visible text event if `delta` is non-empty.
|
||||||
|
pub fn push_text(&mut self, delta: String) {
|
||||||
|
if delta.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.events.push(UnifiedParserEvent::Text(delta));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append one reasoning text event if `delta` is non-empty.
|
||||||
|
pub fn push_reasoning(&mut self, delta: String) {
|
||||||
|
if delta.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.events.push(UnifiedParserEvent::Reasoning(delta));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append parsed tool parser output as unified events.
|
||||||
|
pub fn append_tool_output(&mut self, output: ToolParserOutput) {
|
||||||
|
// TODO: make ToolParserOutput carry ordered events and remove this text-first flattening.
|
||||||
|
self.push_text(output.normal_text);
|
||||||
|
self.events.extend(output.calls.into_iter().map(UnifiedParserEvent::ToolCall));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append another parser output onto this one.
|
||||||
|
pub fn append(&mut self, mut other: Self) {
|
||||||
|
self.events.append(&mut other.events);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Incremental parser that extracts reasoning and tool-call events from assistant output.
|
||||||
|
pub trait UnifiedParser: Send {
|
||||||
|
/// Construct a boxed parser instance for one request stream.
|
||||||
|
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
|
||||||
|
where
|
||||||
|
Self: Sized + 'static;
|
||||||
|
|
||||||
|
/// Initialize parser state from prompt token IDs before output deltas arrive.
|
||||||
|
fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return whether decoded output must preserve tokenizer special tokens.
|
||||||
|
fn preserve_special_tokens(&self) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the xgrammar structural-tag model used for strict tool calling.
|
||||||
|
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the parser-provided ID for a tool call by index, if the model emitted one.
|
||||||
|
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Feed one decoded text delta into the parser, appending committed output into `output`.
|
||||||
|
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()>;
|
||||||
|
|
||||||
|
/// Flush any buffered parser state at end of stream.
|
||||||
|
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||||
|
Ok(UnifiedParserOutput::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clear parser state and return currently uncommitted buffered text.
|
||||||
|
fn reset(&mut self) -> String {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Errors produced while creating or running unified parsers.
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum UnifiedParserError {
|
||||||
|
#[error("combined parser is constructed from split parser instances")]
|
||||||
|
CombinedParserConstructor,
|
||||||
|
#[error(transparent)]
|
||||||
|
Reasoning(#[from] ReasoningError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Tool(#[from] ToolParserError),
|
||||||
|
}
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "vllm-reasoning-parser"
|
|
||||||
version.workspace = true
|
|
||||||
edition.workspace = true
|
|
||||||
license.workspace = true
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
thiserror.workspace = true
|
|
||||||
vllm-tokenizer.workspace = true
|
|
||||||
|
|
||||||
[lints]
|
|
||||||
workspace = true
|
|
||||||
+1
-1
@@ -27,7 +27,7 @@ def rust_extensions(*, optional: bool = False) -> list[RustExtension]:
|
|||||||
),
|
),
|
||||||
RustExtension(
|
RustExtension(
|
||||||
target="vllm._rust_tool_parser",
|
target="vllm._rust_tool_parser",
|
||||||
path="rust/src/tool-parser/python/Cargo.toml",
|
path="rust/src/parser/python/Cargo.toml",
|
||||||
features=["pyo3/abi3-py38"],
|
features=["pyo3/abi3-py38"],
|
||||||
binding=Binding.PyO3,
|
binding=Binding.PyO3,
|
||||||
optional=optional,
|
optional=optional,
|
||||||
|
|||||||
Reference in New Issue
Block a user