[Rust Frontend] Box rarely-set fields of per-token output types (#60588)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1a3a7a8f01
commit
9a585c62aa
@@ -82,6 +82,10 @@ pub struct EngineCoreEvent {
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
|
||||
///
|
||||
/// Fields that are set on few outputs (first or final output, or opt-in
|
||||
/// features) are boxed: outputs are moved by value through the dispatcher and
|
||||
/// the request stream once per engine step, so their inline size matters.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutput {
|
||||
pub request_id: String,
|
||||
@@ -95,7 +99,7 @@ pub struct EngineCoreOutput {
|
||||
#[serde(default)]
|
||||
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
|
||||
#[serde(default)]
|
||||
pub pooling_output: Option<OpaqueValue>,
|
||||
pub pooling_output: Option<Box<OpaqueValue>>,
|
||||
#[serde(default)]
|
||||
pub finish_reason: Option<EngineCoreFinishReason>,
|
||||
#[serde(default)]
|
||||
@@ -103,17 +107,17 @@ pub struct EngineCoreOutput {
|
||||
#[serde(default)]
|
||||
pub events: Option<Vec<EngineCoreEvent>>,
|
||||
#[serde(default)]
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
pub kv_transfer_params: Option<Box<serde_json::Value>>,
|
||||
#[serde(default)]
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
pub ec_transfer_params: Option<Box<serde_json::Value>>,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<OpaqueValue>,
|
||||
pub trace_headers: Option<Box<OpaqueValue>>,
|
||||
/// Breakdown of the scheduled prefill computation, set on the first output
|
||||
/// of a newly scheduled prefill and elided for subsequent decode outputs.
|
||||
#[serde(default)]
|
||||
pub prefill_stats: Option<PrefillStats>,
|
||||
pub prefill_stats: Option<Box<PrefillStats>>,
|
||||
#[serde(default)]
|
||||
pub routed_experts: Option<OpaqueValue>,
|
||||
pub routed_experts: Option<Box<OpaqueValue>>,
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
@@ -133,11 +137,11 @@ pub struct EngineCoreOutput {
|
||||
/// Per-request speculative-decoding acceptance metrics, set on the final
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
pub spec_decode_metrics: Option<Box<RequestSpecDecodeMetrics>>,
|
||||
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
|
||||
/// the first output of a request that asked for them.
|
||||
#[serde(default)]
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
pub prompt_token_id_logprobs: Option<Box<WireNdArray>>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
|
||||
Reference in New Issue
Block a user