[Rust Frontend] Box rarely-set fields of per-token output types (#60588)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 11:01:16 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 1a3a7a8f01
commit 9a585c62aa
16 changed files with 80 additions and 64 deletions
+4 -2
View File
@@ -47,10 +47,12 @@ pub struct Finished {
pub usage: TokenUsage,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
/// Kept boxed as received from `vllm-llm`, so the chat events built from
/// this stay small without boxing again.
pub kv_transfer_params: Option<Box<serde_json::Value>>,
/// Connector-specific encoder cache transfer parameters for disaggregated
/// serving.
pub ec_transfer_params: Option<serde_json::Value>,
pub ec_transfer_params: Option<Box<serde_json::Value>>,
/// Sampling support sets aligned with all generated token positions.
pub sampling_mask: Option<SamplingMask>,
}
+4 -2
View File
@@ -101,8 +101,10 @@ impl<T: TextOutputStream> T {
let mut collected = collected.unwrap();
collected.finish_reason = finished.finish_reason;
collected.usage = finished.usage;
collected.kv_transfer_params = finished.kv_transfer_params;
collected.ec_transfer_params = finished.ec_transfer_params;
collected.kv_transfer_params =
finished.kv_transfer_params.map(|value| *value);
collected.ec_transfer_params =
finished.ec_transfer_params.map(|value| *value);
collected.sampling_mask = finished.sampling_mask;
return Ok(collected);
}