[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "vllm-reasoning-parser"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,45 @@
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
/// Reasoning parser for Cohere Command models that use explicit START/END tags.
|
||||
pub struct CohereCmdReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
|
||||
impl CohereCmdReasoningParser {
|
||||
/// Create a Cohere Command parser backed by the shared delimited state
|
||||
/// machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(
|
||||
tokenizer,
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
false,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for CohereCmdReasoningParser {
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self::new(tokenizer)?))
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.push(delta))
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
/// Reasoning parser for DeepSeek R1 style outputs.
|
||||
///
|
||||
/// DeepSeek R1 may begin generating directly inside a reasoning span and only
|
||||
/// emit the closing `</think>` delimiter, so the no-boundary fallback defaults
|
||||
/// to `in_reasoning = true`.
|
||||
pub struct DeepSeekR1ReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
|
||||
impl DeepSeekR1ReasoningParser {
|
||||
/// Create a DeepSeek R1 parser backed by the shared delimited state
|
||||
/// machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", true)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for DeepSeekR1ReasoningParser {
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self::new(tokenizer)?))
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.push(delta))
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
use vllm_tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use super::{ReasoningDelta, ReasoningError, Result};
|
||||
|
||||
/// Shared incremental state machine for tag-delimited reasoning protocols.
|
||||
///
|
||||
/// This helper is intentionally not a public parser type. Model-family parser
|
||||
/// wrappers own one `DelimitedReasoningParser` internally and expose the
|
||||
/// request-facing [`super::ReasoningParser`] trait.
|
||||
///
|
||||
/// The shared state machine stays generic by deriving its initial
|
||||
/// `current_in_reasoning` state from the prompt token boundary instead of
|
||||
/// hardcoding model-family conventions. That means families with the same
|
||||
/// delimiters can often reuse this implementation even if their chat templates
|
||||
/// prefill different prompts.
|
||||
pub(crate) struct DelimitedReasoningParser {
|
||||
tokenizer: DynTokenizer,
|
||||
current_in_reasoning: bool,
|
||||
buffer: String,
|
||||
start_token: String,
|
||||
end_token: String,
|
||||
start_token_id: u32,
|
||||
end_token_id: u32,
|
||||
default_in_reasoning: bool,
|
||||
}
|
||||
|
||||
impl DelimitedReasoningParser {
|
||||
/// Create one delimited parser state machine.
|
||||
///
|
||||
/// `default_in_reasoning` is only used when prompt initialization sees no
|
||||
/// reasoning boundary token at all. If the prompt contains either the
|
||||
/// start or end delimiter, that prompt boundary always wins.
|
||||
pub(crate) fn new(
|
||||
tokenizer: DynTokenizer,
|
||||
start_token: &'static str,
|
||||
end_token: &'static str,
|
||||
default_in_reasoning: bool,
|
||||
) -> Result<Self> {
|
||||
let start_token_id =
|
||||
tokenizer.token_to_id(start_token).ok_or_else(|| ReasoningError::MissingToken {
|
||||
token: start_token.to_string(),
|
||||
})?;
|
||||
let end_token_id =
|
||||
tokenizer.token_to_id(end_token).ok_or_else(|| ReasoningError::MissingToken {
|
||||
token: end_token.to_string(),
|
||||
})?;
|
||||
|
||||
Ok(Self {
|
||||
tokenizer,
|
||||
current_in_reasoning: default_in_reasoning,
|
||||
buffer: String::new(),
|
||||
start_token: start_token.to_string(),
|
||||
end_token: end_token.to_string(),
|
||||
start_token_id,
|
||||
end_token_id,
|
||||
default_in_reasoning,
|
||||
})
|
||||
}
|
||||
|
||||
/// Initialize the starting state from prompt token IDs.
|
||||
pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) {
|
||||
self.current_in_reasoning = last_reasoning_boundary(
|
||||
prompt_token_ids,
|
||||
self.start_token_id,
|
||||
self.end_token_id,
|
||||
self.tokenizer.as_ref(),
|
||||
)
|
||||
.unwrap_or(self.default_in_reasoning);
|
||||
}
|
||||
|
||||
/// Parse one decoded text delta and return its reasoning/content split.
|
||||
pub(crate) fn push(&mut self, delta: &str) -> ReasoningDelta {
|
||||
self.buffer.push_str(delta);
|
||||
|
||||
let partial_suffix_len = self.partial_suffix_len(&self.buffer);
|
||||
let stable_len = self.buffer.len() - partial_suffix_len;
|
||||
let pending_suffix = self.buffer.split_off(stable_len);
|
||||
let stable_text = std::mem::replace(&mut self.buffer, pending_suffix);
|
||||
|
||||
self.parse_stable_text(&stable_text)
|
||||
}
|
||||
|
||||
/// Flush any buffered partial delimiter suffix at end of stream.
|
||||
pub(crate) fn finish(&mut self) -> ReasoningDelta {
|
||||
let stable_text = std::mem::take(&mut self.buffer);
|
||||
self.parse_stable_text(&stable_text)
|
||||
}
|
||||
|
||||
/// Parse text that is known not to end with a partial delimiter suffix.
|
||||
fn parse_stable_text(&mut self, mut stable: &str) -> ReasoningDelta {
|
||||
let mut delta = ReasoningDelta::default();
|
||||
|
||||
while !stable.is_empty() {
|
||||
if self.current_in_reasoning {
|
||||
if let Some(end_idx) = stable.find(&self.end_token) {
|
||||
delta.push_reasoning(&stable[..end_idx]);
|
||||
stable = &stable[end_idx + self.end_token.len()..];
|
||||
self.current_in_reasoning = false;
|
||||
} else {
|
||||
delta.push_reasoning(stable);
|
||||
break;
|
||||
}
|
||||
} else if let Some(start_idx) = stable.find(&self.start_token) {
|
||||
delta.push_content(&stable[..start_idx]);
|
||||
stable = &stable[start_idx + self.start_token.len()..];
|
||||
self.current_in_reasoning = true;
|
||||
} else {
|
||||
delta.push_content(stable);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delta
|
||||
}
|
||||
|
||||
/// Return the longest trailing suffix that could still complete a
|
||||
/// delimiter.
|
||||
fn partial_suffix_len(&self, text: &str) -> usize {
|
||||
let mut best = 0;
|
||||
for idx in text.char_indices().map(|(idx, _)| idx).skip(1) {
|
||||
let suffix = &text[idx..];
|
||||
if self.start_token.starts_with(suffix) && self.start_token != suffix {
|
||||
best = best.max(text.len() - idx);
|
||||
}
|
||||
if self.end_token.starts_with(suffix) && self.end_token != suffix {
|
||||
best = best.max(text.len() - idx);
|
||||
}
|
||||
}
|
||||
|
||||
if self.start_token.starts_with(text) && self.start_token != text {
|
||||
best = best.max(text.len());
|
||||
}
|
||||
if self.end_token.starts_with(text) && self.end_token != text {
|
||||
best = best.max(text.len());
|
||||
}
|
||||
|
||||
best
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine the reasoning state implied by the last prompt boundary, if any.
|
||||
fn last_reasoning_boundary(
|
||||
prompt_token_ids: &[u32],
|
||||
start_token_id: u32,
|
||||
end_token_id: u32,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Option<bool> {
|
||||
for token_id in prompt_token_ids.iter().rev() {
|
||||
if *token_id == start_token_id {
|
||||
return Some(true);
|
||||
}
|
||||
if *token_id == end_token_id {
|
||||
return Some(false);
|
||||
}
|
||||
if tokenizer.is_special_id(*token_id) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
const THOUGHT_PREFIX: &str = "thought\n";
|
||||
|
||||
/// Reasoning parser for Google Gemma4 thinking models.
|
||||
///
|
||||
/// Gemma4 emits reasoning inside `<|channel> ... <channel|>` spans and adds a
|
||||
/// structural `thought\n` label at the beginning of the reasoning channel.
|
||||
/// This parser keeps the delimiter handling in the shared delimited parser and
|
||||
/// only layers on Gemma4-specific request adjustment plus prefix stripping.
|
||||
///
|
||||
/// Original Python implementation:
|
||||
/// <https://github.com/vllm-project/vllm/blob/18b1c77211d8f6fe800bcfb89524d2b598708032/vllm/reasoning/gemma4_reasoning_parser.py#L23>
|
||||
pub struct Gemma4ReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
reasoning_text: String,
|
||||
prefix_stripped: bool,
|
||||
}
|
||||
|
||||
impl Gemma4ReasoningParser {
|
||||
/// Create a Gemma4 parser.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<|channel>", "<channel|>", false)?,
|
||||
reasoning_text: String::new(),
|
||||
prefix_stripped: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Apply Gemma4's `thought\n` stripping rule to one reasoning delta.
|
||||
///
|
||||
/// Early reasoning text is buffered until we can decide whether it begins
|
||||
/// with the structural channel label.
|
||||
fn strip_thought_prefix(&mut self, reasoning: &str) -> Option<String> {
|
||||
if self.prefix_stripped {
|
||||
return Some(reasoning.to_string());
|
||||
}
|
||||
|
||||
self.reasoning_text.push_str(reasoning);
|
||||
|
||||
if self.reasoning_text.starts_with(THOUGHT_PREFIX) {
|
||||
let prefix_len = THOUGHT_PREFIX.len();
|
||||
let previous_len = self.reasoning_text.len() - reasoning.len();
|
||||
if previous_len >= prefix_len {
|
||||
self.reasoning_text.clear();
|
||||
self.prefix_stripped = true;
|
||||
return Some(reasoning.to_string());
|
||||
}
|
||||
|
||||
let prefix_chars_in_delta = prefix_len - previous_len;
|
||||
let stripped = &reasoning[prefix_chars_in_delta.min(reasoning.len())..];
|
||||
if stripped.is_empty() {
|
||||
if self.reasoning_text.len() >= prefix_len {
|
||||
self.reasoning_text.clear();
|
||||
self.prefix_stripped = true;
|
||||
}
|
||||
return None;
|
||||
}
|
||||
|
||||
self.reasoning_text.clear();
|
||||
self.prefix_stripped = true;
|
||||
return Some(stripped.to_string());
|
||||
}
|
||||
|
||||
if THOUGHT_PREFIX.starts_with(&self.reasoning_text) {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.prefix_stripped = true;
|
||||
Some(std::mem::take(&mut self.reasoning_text))
|
||||
}
|
||||
|
||||
/// Apply Gemma4-specific reasoning post-processing to one parsed delta.
|
||||
fn post_process(&mut self, mut result: ReasoningDelta) -> ReasoningDelta {
|
||||
if let Some(reasoning) = result.reasoning.take() {
|
||||
result.reasoning =
|
||||
self.strip_thought_prefix(&reasoning).filter(|text| !text.is_empty());
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for Gemma4ReasoningParser {
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self::new(tokenizer)?))
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids);
|
||||
self.reasoning_text.clear();
|
||||
self.prefix_stripped = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
|
||||
let result = self.inner.push(delta);
|
||||
Ok(self.post_process(result))
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
let result = self.inner.finish();
|
||||
Ok(self.post_process(result))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::Gemma4ReasoningParser;
|
||||
use crate::ReasoningParser;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|channel>" => Some(1000),
|
||||
"<channel|>" => Some(1001),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_streaming(output: &[&str]) -> (Option<String>, Option<String>) {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = Gemma4ReasoningParser::new(tokenizer).unwrap();
|
||||
let mut reasoning = String::new();
|
||||
let mut content = String::new();
|
||||
|
||||
for delta in output {
|
||||
let result = parser.push(delta).unwrap();
|
||||
if let Some(next) = result.reasoning {
|
||||
reasoning.push_str(&next);
|
||||
}
|
||||
if let Some(next) = result.content {
|
||||
content.push_str(&next);
|
||||
}
|
||||
}
|
||||
|
||||
let final_delta = parser.finish().unwrap();
|
||||
if let Some(next) = final_delta.reasoning {
|
||||
reasoning.push_str(&next);
|
||||
}
|
||||
if let Some(next) = final_delta.content {
|
||||
content.push_str(&next);
|
||||
}
|
||||
|
||||
(
|
||||
(!reasoning.is_empty()).then_some(reasoning),
|
||||
(!content.is_empty()).then_some(content),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemma4_reasoning_streaming_handles_channel_delimited_outputs() {
|
||||
let cases = [
|
||||
(
|
||||
"no_reasoning",
|
||||
vec!["This is content"],
|
||||
None,
|
||||
Some("This is content"),
|
||||
),
|
||||
(
|
||||
"reasoning_and_content",
|
||||
vec!["<|channel>This is a reasoning section<channel|>This is the rest"],
|
||||
Some("This is a reasoning section"),
|
||||
Some("This is the rest"),
|
||||
),
|
||||
(
|
||||
"complete_reasoning",
|
||||
vec!["<|channel>This is a reasoning section<channel|>"],
|
||||
Some("This is a reasoning section"),
|
||||
None,
|
||||
),
|
||||
(
|
||||
"multiple_lines",
|
||||
vec!["<|channel>This\nThat<channel|>This is the rest\nThat"],
|
||||
Some("This\nThat"),
|
||||
Some("This is the rest\nThat"),
|
||||
),
|
||||
(
|
||||
"no_end",
|
||||
vec!["<|channel>This is a reasoning section"],
|
||||
Some("This is a reasoning section"),
|
||||
None,
|
||||
),
|
||||
("empty", vec![""], None, None),
|
||||
(
|
||||
"newline_around_reasoning",
|
||||
vec!["Before\n<|channel>This is a reasoning section<channel|>\nThis is the rest"],
|
||||
Some("This is a reasoning section"),
|
||||
Some("Before\n\nThis is the rest"),
|
||||
),
|
||||
(
|
||||
"thought_prefix",
|
||||
vec!["<|channel>thought\nActual reasoning here<channel|>Final answer"],
|
||||
Some("Actual reasoning here"),
|
||||
Some("Final answer"),
|
||||
),
|
||||
(
|
||||
"thought_prefix_only",
|
||||
vec!["<|channel>thought\n<channel|>"],
|
||||
None,
|
||||
None,
|
||||
),
|
||||
(
|
||||
"thought_prefix_multiline",
|
||||
vec!["<|channel>thought\nLine1\nLine2<channel|>Answer"],
|
||||
Some("Line1\nLine2"),
|
||||
Some("Answer"),
|
||||
),
|
||||
(
|
||||
"thought_prefix_diverge",
|
||||
vec!["<|channel>thousand reasons<channel|>Done"],
|
||||
Some("thousand reasons"),
|
||||
Some("Done"),
|
||||
),
|
||||
];
|
||||
|
||||
for (name, output, expected_reasoning, expected_content) in cases {
|
||||
let (reasoning, content) = run_streaming(&output);
|
||||
assert_eq!(reasoning.as_deref(), expected_reasoning, "{name}");
|
||||
assert_eq!(content.as_deref(), expected_content, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemma4_strips_thought_prefix_even_when_split_across_deltas() {
|
||||
let (reasoning, content) =
|
||||
run_streaming(&["<|channel>thou", "ght", "\nabc", "<channel|>done"]);
|
||||
assert_eq!(reasoning.as_deref(), Some("abc"));
|
||||
assert_eq!(content.as_deref(), Some("done"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemma4_preserves_special_tokens() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let parser = Gemma4ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
assert!(parser.preserve_special_tokens());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
/// Reasoning parser for legacy Kimi models that use Unicode thinking tags.
|
||||
pub struct KimiReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
|
||||
impl KimiReasoningParser {
|
||||
/// Create a Kimi parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷", false)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for KimiReasoningParser {
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self::new(tokenizer)?))
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.push(delta))
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! Streaming reasoning parsers for chat completions.
|
||||
//!
|
||||
//! The key design choice here is that parser initialization prefers the
|
||||
//! *actual rendered prompt state* over model-family conventions. When a stream
|
||||
//! starts, each parser receives the prompt token IDs and inspects the last
|
||||
//! reasoning boundary that is already present in the prompt. In practice this
|
||||
//! is a more faithful signal than hardcoding assumptions such as "this model
|
||||
//! always starts in reasoning" or "this model always emits `<think>` itself".
|
||||
//!
|
||||
//! That prompt-first initialization lets multiple model families share the
|
||||
//! same incremental parser implementation even when older Python parsers split
|
||||
//! them apart. If two families use the same textual delimiters and differ
|
||||
//! mostly in how their chat templates prefill `<think>` / `</think>`, they can
|
||||
//! usually reuse one parser here because the prompt token IDs already tell us
|
||||
//! which state the stream is entering with.
|
||||
|
||||
mod cohere_cmd;
|
||||
mod deepseek_r1;
|
||||
mod delimited;
|
||||
mod gemma4;
|
||||
mod kimi;
|
||||
mod qwen3;
|
||||
|
||||
use thiserror::Error;
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
pub use self::cohere_cmd::CohereCmdReasoningParser;
|
||||
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
|
||||
pub(crate) use self::delimited::DelimitedReasoningParser;
|
||||
pub use self::gemma4::Gemma4ReasoningParser;
|
||||
pub use self::kimi::KimiReasoningParser;
|
||||
pub use self::qwen3::Qwen3ReasoningParser;
|
||||
|
||||
/// DeepSeek V3 currently shares the standard `<think>...</think>` parser.
|
||||
pub type DeepSeekV3ReasoningParser = Qwen3ReasoningParser;
|
||||
/// DeepSeek V4 currently shares the standard `<think>...</think>` parser.
|
||||
pub type DeepSeekV4ReasoningParser = Qwen3ReasoningParser;
|
||||
/// GLM45 currently shares the standard `<think>...</think>` parser.
|
||||
pub type Glm45ReasoningParser = Qwen3ReasoningParser;
|
||||
/// Kimi K2 currently shares the standard `<think>...</think>` parser.
|
||||
// TODO: kimi k2 may implicitly end reasoning by starting a tool call section
|
||||
// using <|tool_calls_section_begin|>, we should support that.
|
||||
pub type KimiK2ReasoningParser = Qwen3ReasoningParser;
|
||||
/// MiniMax M2 currently shares the standard `<think>...</think>` parser.
|
||||
pub type MiniMaxM2ReasoningParser = Qwen3ReasoningParser;
|
||||
/// Nemotron V3 currently shares the standard `<think>...</think>` parser.
|
||||
pub type NemotronV3ReasoningParser = Qwen3ReasoningParser;
|
||||
/// Step3 currently shares the standard `<think>...</think>` parser.
|
||||
pub type Step3ReasoningParser = Qwen3ReasoningParser;
|
||||
|
||||
/// Result alias for reasoning parser operations.
|
||||
pub type Result<T> = std::result::Result<T, ReasoningError>;
|
||||
|
||||
/// One parsed streaming delta split into reasoning and visible content.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct ReasoningDelta {
|
||||
pub reasoning: Option<String>,
|
||||
pub content: Option<String>,
|
||||
}
|
||||
|
||||
impl ReasoningDelta {
|
||||
/// Return true when this delta carries neither reasoning nor content text.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.reasoning.is_none() && self.content.is_none()
|
||||
}
|
||||
|
||||
/// Append text to the reasoning portion, creating it on first use.
|
||||
pub(crate) fn push_reasoning(&mut self, text: &str) {
|
||||
if text.is_empty() {
|
||||
return;
|
||||
}
|
||||
match &mut self.reasoning {
|
||||
Some(existing) => existing.push_str(text),
|
||||
None => self.reasoning = Some(text.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append text to the visible content portion, creating it on first use.
|
||||
pub(crate) fn push_content(&mut self, text: &str) {
|
||||
if text.is_empty() {
|
||||
return;
|
||||
}
|
||||
match &mut self.content {
|
||||
Some(existing) => existing.push_str(text),
|
||||
None => self.content = Some(text.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Incremental parser that splits decoded text deltas into reasoning and
|
||||
/// content.
|
||||
pub trait ReasoningParser: Send {
|
||||
/// Construct a boxed parser instance for one request stream.
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static;
|
||||
|
||||
/// Initialize parser state from prompt token IDs before output deltas
|
||||
/// arrive.
|
||||
fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return whether decoded output must preserve tokenizer special tokens.
|
||||
///
|
||||
/// Some model families emit reasoning sentinels as special tokens. Those
|
||||
/// parsers need `skip_special_tokens = false` while parsing is enabled.
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Feed one decoded text delta into the parser.
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta>;
|
||||
|
||||
/// Flush any buffered partial delimiter state at end of stream.
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
Ok(ReasoningDelta::default())
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors produced while creating or running reasoning parsers.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ReasoningError {
|
||||
#[error("tokenizer is missing reasoning delimiter token `{token}`")]
|
||||
MissingToken { token: String },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,43 @@
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
/// Reasoning parser for the Qwen3/Qwen3.5 family.
|
||||
///
|
||||
/// This parser uses standard `<think>...</think>` delimiters and defaults to
|
||||
/// waiting for an explicit start token when prompt initialization finds no
|
||||
/// reasoning boundary.
|
||||
pub struct Qwen3ReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
|
||||
impl Qwen3ReasoningParser {
|
||||
/// Create a Qwen3 parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for Qwen3ReasoningParser {
|
||||
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self::new(tokenizer)?))
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.push(delta))
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ReasoningDelta> {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::{
|
||||
DeepSeekR1ReasoningParser, DelimitedReasoningParser, Qwen3ReasoningParser, ReasoningParser,
|
||||
};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<think>" => Some(1),
|
||||
"</think>" => Some(2),
|
||||
"<|START_THINKING|>" => Some(3),
|
||||
"<|END_THINKING|>" => Some(4),
|
||||
"◁think▷" => Some(5),
|
||||
"◁/think▷" => Some(6),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == 7
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_content_only_stream() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
parser.push("plain content").content.as_deref(),
|
||||
Some("plain content")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_single_chunk_with_reasoning_and_content() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
let delta = parser.push("<think>reason</think>answer");
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_partial_tokens_across_chunks() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
assert!(parser.push("<thi").is_empty());
|
||||
let delta = parser.push("nk>reason</think>answer");
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_finish_flushes_buffer() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
parser.initialize(&[1]);
|
||||
|
||||
let delta = parser.push("unfinished</thi");
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));
|
||||
let final_delta = parser.finish();
|
||||
assert_eq!(final_delta.reasoning.as_deref(), Some("</thi"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(delta.content.as_deref(), Some("reason</think>answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_prompt_end_marker_starts_in_content() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[2]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_tolerates_old_and_new_formats() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
|
||||
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
|
||||
let old = old_parser.push("<think>reason</think>answer").unwrap();
|
||||
assert_eq!(old.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(old.content.as_deref(), Some("answer"));
|
||||
|
||||
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
new_parser.initialize(&[1]).unwrap();
|
||||
let new = new_parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(new.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(new.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[1, 7]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[2, 7]).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(delta.content.as_deref(), Some("answer"));
|
||||
}
|
||||
Reference in New Issue
Block a user