[Rust Frontend] Use token-attributed text in reasoning and unified parsers (#54884)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-04 02:05:05 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8bf39632b8
commit 21a2211040
22 changed files with 1012 additions and 350 deletions
+5 -2
View File
@@ -9,7 +9,7 @@ use vllm_parser::tool::{
use vllm_parser::unified::{
UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput,
};
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::{DecodedText, Tokenizer};
/// Tokenizer stub used by unified-parser benchmarks.
struct BenchTokenizer;
@@ -99,7 +99,10 @@ impl<T: UnifiedParser> ToolParser for UnifiedToolParserAdapter<T> {
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
let mut unified_output = UnifiedParserOutput::default();
let result = self.inner.parse_into(chunk, &mut unified_output).map_err(map_unified_error);
let result = self
.inner
.parse_into(DecodedText::unattributed(chunk), &mut unified_output)
.map_err(map_unified_error);
append_unified_output(unified_output, output)?;
result
}