[Bugfix][Rust Frontend] Always include token_ids in completion stream choices (#60286)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: devshah-cohere <dev.shah@cohere.com>
This commit is contained in:
devshah-cohere
2026-10-08 19:11:55 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent ba77c4c130
commit b49c921a27
@@ -271,6 +271,13 @@ async fn completion_chunk_stream(
macro_rules! yield_chunk {
($chunk:expr) => {{
let mut chunk = $chunk;
if return_token_ids {
for choice in &mut chunk.choices {
if choice.token_ids.is_none() {
choice.token_ids = Some(Vec::new());
}
}
}
if include_continuous_usage {
chunk.usage = Some(continuous_usage.to_usage());
}
@@ -787,6 +794,70 @@ mod tests {
}
}
#[tokio::test]
async fn completion_stream_serializes_requested_token_ids_on_every_choice() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(decoded_delta(
"ok",
vec![7, 8],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
let chunks = completion_chunk_stream(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
return_token_ids: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await;
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
let payloads: Vec<_> = chunks
.iter()
.map(|chunk| serde_json::to_value(chunk).expect("chunk should serialize"))
.collect();
assert_eq!(payloads.len(), 3);
assert_eq!(
payloads[0]["choices"][0]["prompt_token_ids"],
serde_json::json!([1, 2])
);
assert_eq!(
payloads[0]["choices"][0]["token_ids"],
serde_json::json!([])
);
assert_eq!(
payloads[1]["choices"][0]["token_ids"],
serde_json::json!([7, 8])
);
assert_eq!(
payloads[2]["choices"][0]["token_ids"],
serde_json::json!([])
);
}
#[tokio::test]
async fn collect_completion_hides_internal_prompt_only_token() {
let stream = stream::iter(vec![