Signed-off-by: William Zhao <williamzh@meta.com> Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cyrus Leung
Claude
parent
f4e9bb01de
commit
3962042304
@@ -112,6 +112,17 @@ pub struct EngineCoreOutput {
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
/// Multi-modal hashes the engine could not find in its receiver cache,
|
||||
/// because the frontend sent `data: None` for an item the engine had
|
||||
/// already evicted. Non-empty makes the output retryable: a frontend that
|
||||
/// keeps a metadata-only shadow of the engine cache drops these entries and
|
||||
/// resends the request with the item data attached.
|
||||
///
|
||||
/// TODO: always `None` here, since this client has no multi-modal processor
|
||||
/// cache and always sends item data inline. Act on it once the Rust
|
||||
/// frontend grows one.
|
||||
#[serde(default)]
|
||||
pub mm_cache_miss_hashes: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
@@ -372,18 +383,9 @@ mod tests {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![42],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
stop_reason: Some(StopReason::Text("stop".to_string())),
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
@@ -437,6 +439,7 @@ mod tests {
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -223,18 +223,8 @@ fn request_output(
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -97,6 +97,7 @@ class EngineCoreOutput(
|
||||
prefill_stats: object | None = None
|
||||
routed_experts: object | None = None
|
||||
num_nans_in_logits: int = 0
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
|
||||
Reference in New Issue
Block a user