[Rust Frontend] Attribute decoded text to tokens (#52910)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-01 22:25:31 +00:00
committed by GitHub
parent 259a209bfa
commit b911fe85c4
17 changed files with 1182 additions and 320 deletions
+9 -6
View File
@@ -14,7 +14,7 @@ use tracing::{Span, info, info_span, warn};
use tracing_futures::Instrument as _;
use uuid::Uuid;
use vllm_llm::current_unix_timestamp_secs;
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
use vllm_text::{DecodedTextEvent, Prompt, SampledDelta, TextOutputStreamExt as _, TextRequest};
use super::convert::{self, ResponseOpts};
use super::{InferenceServer, pb};
@@ -264,17 +264,20 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
})
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
}) => Ok(pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&delta,
&decoded.text,
&token_ids,
logprobs.as_ref(),
finished.as_ref(),
finished.as_deref(),
&response_opts,
)),
}),
@@ -23,7 +23,7 @@ use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::tokenizer::Tokenizer;
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, SampledDelta, TextOutputStream,
TextOutputStreamExt as _, TextRequest,
};
@@ -317,11 +317,15 @@ async fn completion_chunk_stream(
}
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
}) => {
let delta = decoded.text;
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
// The one generated token is only used to drive the engine to a finished event,
// so hide its delta and forward only the terminal finish/usage metadata.
@@ -561,8 +565,8 @@ mod tests {
use itertools::Itertools as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
DecodedTokenLogprob, FinishReason, Finished,
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedText,
DecodedTextEvent, DecodedTokenLogprob, FinishReason, Finished, SampledDelta,
};
use super::{
@@ -570,6 +574,22 @@ mod tests {
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
};
fn decoded_delta(
text: impl Into<String>,
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText::unattributed(text),
sampled: SampledDelta {
token_ids,
logprobs,
},
finished: finished.map(Box::new),
}
}
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
Arc::new(StreamResponseEnvelope::new(
"cmpl-1".to_string(),
@@ -619,10 +639,10 @@ mod tests {
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "h".to_string(),
token_ids: vec![b'h' as u32],
logprobs: Some(DecodedLogprobs {
Ok(decoded_delta(
"h",
vec![b'h' as u32],
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
@@ -640,12 +660,12 @@ mod tests {
],
}],
}),
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: vec![b'!' as u32],
logprobs: Some(DecodedLogprobs {
None,
)),
Ok(decoded_delta(
"",
vec![b'!' as u32],
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
@@ -663,7 +683,7 @@ mod tests {
],
}],
}),
finished: Some(Finished {
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 5,
output_token_count: 2,
@@ -675,7 +695,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -773,11 +793,11 @@ mod tests {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -787,7 +807,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let response = super::collect_completion(
@@ -825,11 +845,11 @@ mod tests {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
@@ -839,7 +859,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let response = super::collect_completion(
@@ -880,11 +900,11 @@ mod tests {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -894,7 +914,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -952,11 +972,11 @@ mod tests {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
@@ -966,7 +986,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -1026,11 +1046,11 @@ mod tests {
}],
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -1040,7 +1060,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(