[Rust Frontend] Box rarely-set fields of per-token output types (#60588)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1a3a7a8f01
commit
9a585c62aa
@@ -47,10 +47,12 @@ pub struct Finished {
|
||||
pub usage: TokenUsage,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
/// Kept boxed as received from `vllm-llm`, so the chat events built from
|
||||
/// this stay small without boxing again.
|
||||
pub kv_transfer_params: Option<Box<serde_json::Value>>,
|
||||
/// Connector-specific encoder cache transfer parameters for disaggregated
|
||||
/// serving.
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
pub ec_transfer_params: Option<Box<serde_json::Value>>,
|
||||
/// Sampling support sets aligned with all generated token positions.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
}
|
||||
|
||||
@@ -101,8 +101,10 @@ impl<T: TextOutputStream> T {
|
||||
let mut collected = collected.unwrap();
|
||||
collected.finish_reason = finished.finish_reason;
|
||||
collected.usage = finished.usage;
|
||||
collected.kv_transfer_params = finished.kv_transfer_params;
|
||||
collected.ec_transfer_params = finished.ec_transfer_params;
|
||||
collected.kv_transfer_params =
|
||||
finished.kv_transfer_params.map(|value| *value);
|
||||
collected.ec_transfer_params =
|
||||
finished.ec_transfer_params.map(|value| *value);
|
||||
collected.sampling_mask = finished.sampling_mask;
|
||||
return Ok(collected);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user