[Rust Frontend] Attribute decoded text to tokens (#52910)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-01 22:25:31 +00:00
committed by GitHub
parent 259a209bfa
commit b911fe85c4
17 changed files with 1182 additions and 320 deletions
+14 -13
View File
@@ -13,7 +13,7 @@ use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
use vllm_text::output::DecodedTextEvent;
use vllm_text::output::{DecodedTextEvent, SampledDelta};
use crate::Result;
use crate::error::Error;
@@ -237,12 +237,15 @@ pub(crate) async fn unified_event_stream(
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
} => {
for next in state.process_delta(delta)? {
for next in state.process_delta(decoded.text)? {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
@@ -375,24 +378,22 @@ mod tests {
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
decoded: vllm_text::DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: None,
}
}
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(vllm_text::output::Finished {
decoded: vllm_text::DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: Some(Box::new(vllm_text::output::Finished {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}
}
+7 -4
View File
@@ -11,7 +11,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
use futures::StreamExt as _;
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
use openai_harmony::{HarmonyEncoding, StreamableParser};
use vllm_text::output::DecodedTextEvent;
use vllm_text::output::{DecodedTextEvent, SampledDelta};
use crate::Result as ChatResult;
use crate::error::{Error, Result};
@@ -345,9 +345,12 @@ async fn harmony_assistant_event_stream(
.await;
}
DecodedTextEvent::TextDelta {
delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here
token_ids,
logprobs,
decoded: _, // harmony takes raw token IDs as input, so we ignore decoded text here
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
} => {
for event in state.process_token_ids(&token_ids)? {
+30 -49
View File
@@ -6,7 +6,9 @@ use std::sync::Arc;
use futures::executor::block_on;
use futures::{TryStreamExt as _, stream};
use openai_harmony::chat::{Message, Role};
use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
use vllm_text::output::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedText, DecodedTextEvent, Finished, SampledDelta,
};
use super::*;
use crate::output::ChatOutputProcessor;
@@ -59,6 +61,21 @@ fn finished() -> Finished {
}
}
fn decoded_tokens(
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText::default(),
sampled: SampledDelta {
token_ids,
logprobs,
},
finished: finished.map(Box::new),
}
}
async fn collect_events(
processor: HarmonyChatOutputProcessor,
events: Vec<DecodedTextEvent>,
@@ -96,12 +113,7 @@ fn interrupted_final_message_is_preserved() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..tokens.len() - 1].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
],
));
@@ -135,12 +147,7 @@ fn eos_flush_preserves_trailing_replacement_text() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -157,12 +164,7 @@ fn interrupted_analysis_message_is_preserved() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..tokens.len() - 1].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
],
));
@@ -196,12 +198,7 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -225,12 +222,7 @@ fn multiple_messages_get_newline_separators() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -256,18 +248,8 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..midpoint].to_vec(),
logprobs: None,
finished: None,
},
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[midpoint..].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..midpoint].to_vec(), None, None),
decoded_tokens(tokens[midpoint..].to_vec(), None, Some(finished())),
],
));
@@ -303,14 +285,13 @@ fn semantic_events_precede_same_update_logprobs() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: Some(DecodedLogprobs {
decoded_tokens(
tokens,
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs { entries: vec![] }],
}),
finished: Some(finished()),
},
Some(finished()),
),
],
));
+13 -12
View File
@@ -784,15 +784,14 @@ fn decoded_completion_stream(
if chunks.is_empty() {
events.push({
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(Finished {
decoded: vllm_text::DecodedText::default(),
sampled: vllm_text::SampledDelta::default(),
finished: Some(Box::new(Finished {
usage: Default::default(),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}
});
} else {
@@ -805,10 +804,12 @@ fn decoded_completion_stream(
ec_transfer_params: None,
});
events.push(DecodedTextEvent::TextDelta {
delta: chunk.delta,
token_ids: chunk.token_ids,
logprobs: None,
finished,
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
sampled: vllm_text::SampledDelta {
token_ids: chunk.token_ids,
logprobs: None,
},
finished: finished.map(Box::new),
});
}
}
@@ -864,7 +865,7 @@ fn incremental_decode_chunks(
for token_id in chunk_token_ids.iter().copied() {
decoder.push_token(token_id)?;
while let Some(chunk) = decoder.next_chunk() {
delta.push_str(&chunk);
delta.push_str(&chunk.text);
}
}
chunks.push(DecodedCompletionChunk {
@@ -876,10 +877,10 @@ fn incremental_decode_chunks(
let (last_chunk, _) = decoder.flush(None)?;
if let Some(last_chunk) = last_chunk {
if let Some(delta) = chunks.last_mut() {
delta.delta.push_str(&last_chunk);
delta.delta.push_str(&last_chunk.text);
} else {
chunks.push(DecodedCompletionChunk {
delta: last_chunk,
delta: last_chunk.text,
token_ids: Vec::new(),
});
}