[Rust Frontend] Attribute decoded text to tokens (#52910)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -13,7 +13,7 @@ use futures::{StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport;
|
||||
use tracing::warn;
|
||||
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
use vllm_text::output::{DecodedTextEvent, SampledDelta};
|
||||
|
||||
use crate::Result;
|
||||
use crate::error::Error;
|
||||
@@ -237,12 +237,15 @@ pub(crate) async fn unified_event_stream(
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
for next in state.process_delta(delta)? {
|
||||
for next in state.process_delta(decoded.text)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
if logprobs.is_some() || !token_ids.is_empty() {
|
||||
@@ -375,24 +378,22 @@ mod tests {
|
||||
|
||||
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
delta: delta.to_string(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
delta: delta.to_string(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(vllm_text::output::Finished {
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: Some(Box::new(vllm_text::output::Finished {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::StreamExt as _;
|
||||
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
|
||||
use openai_harmony::{HarmonyEncoding, StreamableParser};
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
use vllm_text::output::{DecodedTextEvent, SampledDelta};
|
||||
|
||||
use crate::Result as ChatResult;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -345,9 +345,12 @@ async fn harmony_assistant_event_stream(
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded: _, // harmony takes raw token IDs as input, so we ignore decoded text here
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
for event in state.process_token_ids(&token_ids)? {
|
||||
|
||||
@@ -6,7 +6,9 @@ use std::sync::Arc;
|
||||
use futures::executor::block_on;
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use openai_harmony::chat::{Message, Role};
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
|
||||
use vllm_text::output::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedText, DecodedTextEvent, Finished, SampledDelta,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::output::ChatOutputProcessor;
|
||||
@@ -59,6 +61,21 @@ fn finished() -> Finished {
|
||||
}
|
||||
}
|
||||
|
||||
fn decoded_tokens(
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText::default(),
|
||||
sampled: SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_events(
|
||||
processor: HarmonyChatOutputProcessor,
|
||||
events: Vec<DecodedTextEvent>,
|
||||
@@ -96,12 +113,7 @@ fn interrupted_final_message_is_preserved() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -135,12 +147,7 @@ fn eos_flush_preserves_trailing_replacement_text() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -157,12 +164,7 @@ fn interrupted_analysis_message_is_preserved() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -196,12 +198,7 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -225,12 +222,7 @@ fn multiple_messages_get_newline_separators() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -256,18 +248,8 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..midpoint].to_vec(),
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
},
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[midpoint..].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..midpoint].to_vec(), None, None),
|
||||
decoded_tokens(tokens[midpoint..].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -303,14 +285,13 @@ fn semantic_events_precede_same_update_logprobs() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
decoded_tokens(
|
||||
tokens,
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs { entries: vec![] }],
|
||||
}),
|
||||
finished: Some(finished()),
|
||||
},
|
||||
Some(finished()),
|
||||
),
|
||||
],
|
||||
));
|
||||
|
||||
|
||||
@@ -784,15 +784,14 @@ fn decoded_completion_stream(
|
||||
if chunks.is_empty() {
|
||||
events.push({
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
decoded: vllm_text::DecodedText::default(),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: Default::default(),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}
|
||||
});
|
||||
} else {
|
||||
@@ -805,10 +804,12 @@ fn decoded_completion_stream(
|
||||
ec_transfer_params: None,
|
||||
});
|
||||
events.push(DecodedTextEvent::TextDelta {
|
||||
delta: chunk.delta,
|
||||
token_ids: chunk.token_ids,
|
||||
logprobs: None,
|
||||
finished,
|
||||
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
|
||||
sampled: vllm_text::SampledDelta {
|
||||
token_ids: chunk.token_ids,
|
||||
logprobs: None,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -864,7 +865,7 @@ fn incremental_decode_chunks(
|
||||
for token_id in chunk_token_ids.iter().copied() {
|
||||
decoder.push_token(token_id)?;
|
||||
while let Some(chunk) = decoder.next_chunk() {
|
||||
delta.push_str(&chunk);
|
||||
delta.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
chunks.push(DecodedCompletionChunk {
|
||||
@@ -876,10 +877,10 @@ fn incremental_decode_chunks(
|
||||
let (last_chunk, _) = decoder.flush(None)?;
|
||||
if let Some(last_chunk) = last_chunk {
|
||||
if let Some(delta) = chunks.last_mut() {
|
||||
delta.delta.push_str(&last_chunk);
|
||||
delta.delta.push_str(&last_chunk.text);
|
||||
} else {
|
||||
chunks.push(DecodedCompletionChunk {
|
||||
delta: last_chunk,
|
||||
delta: last_chunk.text,
|
||||
token_ids: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user