[Rust Frontend] Box rarely-set fields of per-token output types (#60588)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 11:01:16 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 1a3a7a8f01
commit 9a585c62aa
16 changed files with 80 additions and 64 deletions
+2 -2
View File
@@ -406,8 +406,8 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> Result<pb::FinishIn
num_output_tokens: finished.usage.output_token_count as u32,
finish_reason,
stop_reason,
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct),
kv_transfer_params: finished.kv_transfer_params.as_deref().and_then(json_to_proto_struct),
ec_transfer_params: finished.ec_transfer_params.as_deref().and_then(json_to_proto_struct),
num_cached_tokens: Some(finished.usage.cached_token_count as u32),
})
}
+2 -2
View File
@@ -255,8 +255,8 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
let finish_info = vllm_text::Finished {
usage: collected.usage,
finish_reason: collected.finish_reason,
kv_transfer_params: collected.kv_transfer_params,
ec_transfer_params: collected.ec_transfer_params,
kv_transfer_params: collected.kv_transfer_params.map(Box::new),
ec_transfer_params: collected.ec_transfer_params.map(Box::new),
sampling_mask: collected.sampling_mask,
};
@@ -176,7 +176,7 @@ async fn generate_chunk_stream(
match next {
Ok(output) => {
if let Some(metrics) = output.spec_decode_metrics {
spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(metrics));
spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(*metrics));
}
if prompt_tokens.is_none()
&& let Some(info) = output.prompt_info.as_ref()
@@ -632,11 +632,11 @@ mod tests {
}),
Ok(GenerateOutput {
request_id: String::new(),
prompt_info: Some(GeneratePromptInfo {
prompt_info: Some(Box::new(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22_u32]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
})),
token_ids: vec![33],
logprobs: None,
finish_reason: Some(FinishReason::stop_eos()),
@@ -716,10 +716,12 @@ mod tests {
) -> GenerateOutput {
GenerateOutput {
request_id: String::new(),
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
prompt_info: prompt_token_ids.map(|ids| {
Box::new(GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
})
}),
token_ids,
logprobs: None,
@@ -817,7 +819,7 @@ mod tests {
..Default::default()
};
let mut output = stream_output(None, Vec::new(), Some(FinishReason::stop_eos()));
output.spec_decode_metrics = Some(metrics.clone());
output.spec_decode_metrics = Some(Box::new(metrics.clone()));
for include_usage in [false, true] {
let chunks: Vec<_> = generate_chunk_stream(
+2 -2
View File
@@ -120,8 +120,8 @@ fn request_output_with_logprobs_and_kv(
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
finish_reason,
stop_reason,
kv_transfer_params,
ec_transfer_params,
kv_transfer_params: kv_transfer_params.map(Box::new),
ec_transfer_params: ec_transfer_params.map(Box::new),
..Default::default()
}
}