[Rust Frontend] Box rarely-set fields of per-token output types (#60588)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1a3a7a8f01
commit
9a585c62aa
@@ -406,8 +406,8 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> Result<pb::FinishIn
|
||||
num_output_tokens: finished.usage.output_token_count as u32,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
kv_transfer_params: finished.kv_transfer_params.as_deref().and_then(json_to_proto_struct),
|
||||
ec_transfer_params: finished.ec_transfer_params.as_deref().and_then(json_to_proto_struct),
|
||||
num_cached_tokens: Some(finished.usage.cached_token_count as u32),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -255,8 +255,8 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
let finish_info = vllm_text::Finished {
|
||||
usage: collected.usage,
|
||||
finish_reason: collected.finish_reason,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
ec_transfer_params: collected.ec_transfer_params,
|
||||
kv_transfer_params: collected.kv_transfer_params.map(Box::new),
|
||||
ec_transfer_params: collected.ec_transfer_params.map(Box::new),
|
||||
sampling_mask: collected.sampling_mask,
|
||||
};
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ async fn generate_chunk_stream(
|
||||
match next {
|
||||
Ok(output) => {
|
||||
if let Some(metrics) = output.spec_decode_metrics {
|
||||
spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(metrics));
|
||||
spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(*metrics));
|
||||
}
|
||||
if prompt_tokens.is_none()
|
||||
&& let Some(info) = output.prompt_info.as_ref()
|
||||
@@ -632,11 +632,11 @@ mod tests {
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
request_id: String::new(),
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_info: Some(Box::new(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22_u32]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
})),
|
||||
token_ids: vec![33],
|
||||
logprobs: None,
|
||||
finish_reason: Some(FinishReason::stop_eos()),
|
||||
@@ -716,10 +716,12 @@ mod tests {
|
||||
) -> GenerateOutput {
|
||||
GenerateOutput {
|
||||
request_id: String::new(),
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
prompt_info: prompt_token_ids.map(|ids| {
|
||||
Box::new(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
})
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -817,7 +819,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let mut output = stream_output(None, Vec::new(), Some(FinishReason::stop_eos()));
|
||||
output.spec_decode_metrics = Some(metrics.clone());
|
||||
output.spec_decode_metrics = Some(Box::new(metrics.clone()));
|
||||
|
||||
for include_usage in [false, true] {
|
||||
let chunks: Vec<_> = generate_chunk_stream(
|
||||
|
||||
@@ -120,8 +120,8 @@ fn request_output_with_logprobs_and_kv(
|
||||
new_prompt_logprobs_tensors: new_prompt_logprobs_tensors.map(MaybeWireLogprobs::Direct),
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
kv_transfer_params: kv_transfer_params.map(Box::new),
|
||||
ec_transfer_params: ec_transfer_params.map(Box::new),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user