[Rust Frontend] Attribute decoded text to tokens (#52910)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -14,7 +14,7 @@ use tracing::{Span, info, info_span, warn};
|
||||
use tracing_futures::Instrument as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_llm::current_unix_timestamp_secs;
|
||||
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
|
||||
use vllm_text::{DecodedTextEvent, Prompt, SampledDelta, TextOutputStreamExt as _, TextRequest};
|
||||
|
||||
use super::convert::{self, ResponseOpts};
|
||||
use super::{InferenceServer, pb};
|
||||
@@ -264,17 +264,20 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
})
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&delta,
|
||||
&decoded.text,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_ref(),
|
||||
finished.as_deref(),
|
||||
&response_opts,
|
||||
)),
|
||||
}),
|
||||
|
||||
@@ -23,7 +23,7 @@ use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, SampledDelta, TextOutputStream,
|
||||
TextOutputStreamExt as _, TextRequest,
|
||||
};
|
||||
|
||||
@@ -317,11 +317,15 @@ async fn completion_chunk_stream(
|
||||
}
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
}) => {
|
||||
let delta = decoded.text;
|
||||
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
|
||||
// The one generated token is only used to drive the engine to a finished event,
|
||||
// so hide its delta and forward only the terminal finish/usage metadata.
|
||||
@@ -561,8 +565,8 @@ mod tests {
|
||||
use itertools::Itertools as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
|
||||
DecodedTokenLogprob, FinishReason, Finished,
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedText,
|
||||
DecodedTextEvent, DecodedTokenLogprob, FinishReason, Finished, SampledDelta,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -570,6 +574,22 @@ mod tests {
|
||||
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
fn decoded_delta(
|
||||
text: impl Into<String>,
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText::unattributed(text),
|
||||
sampled: SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"cmpl-1".to_string(),
|
||||
@@ -619,10 +639,10 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "h".to_string(),
|
||||
token_ids: vec![b'h' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
Ok(decoded_delta(
|
||||
"h",
|
||||
vec![b'h' as u32],
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
@@ -640,12 +660,12 @@ mod tests {
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: vec![b'!' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
None,
|
||||
)),
|
||||
Ok(decoded_delta(
|
||||
"",
|
||||
vec![b'!' as u32],
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
@@ -663,7 +683,7 @@ mod tests {
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
@@ -675,7 +695,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -773,11 +793,11 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -787,7 +807,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
@@ -825,11 +845,11 @@ mod tests {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
@@ -839,7 +859,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
@@ -880,11 +900,11 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -894,7 +914,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -952,11 +972,11 @@ mod tests {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
@@ -966,7 +986,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -1026,11 +1046,11 @@ mod tests {
|
||||
}],
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -1040,7 +1060,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
|
||||
Reference in New Issue
Block a user