[Bugfix][V1][Multimodal] Recover from P0/P1 processor cache drift (#46747) (#46747)

Signed-off-by: William Zhao <williamzh@meta.com>
Signed-off-by: WillZZZy <93454470+WillZZZy@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
WillZZZy
2026-08-12 12:06:57 +08:00
committed by GitHub
co-authored by Cyrus Leung Claude
parent f4e9bb01de
commit 3962042304
13 changed files with 236 additions and 113 deletions
@@ -112,6 +112,17 @@ pub struct EngineCoreOutput {
/// Number of NaNs seen in logits. Values above zero indicate corruption.
#[serde(default)]
pub num_nans_in_logits: u32,
/// Multi-modal hashes the engine could not find in its receiver cache,
/// because the frontend sent `data: None` for an item the engine had
/// already evicted. Non-empty makes the output retryable: a frontend that
/// keeps a metadata-only shadow of the engine cache drops these entries and
/// resends the request with the item data attached.
///
/// TODO: always `None` here, since this client has no multi-modal processor
/// cache and always sends item data inline. Act on it once the Rust
/// frontend grows one.
#[serde(default)]
pub mm_cache_miss_hashes: Option<Vec<String>>,
}
impl EngineCoreOutput {
@@ -372,18 +383,9 @@ mod tests {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![42],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: Some(EngineCoreFinishReason::Length),
stop_reason: Some(StopReason::Text("stop".to_string())),
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
@@ -437,6 +439,7 @@ mod tests {
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
},
],
scheduler_stats: None,
@@ -223,18 +223,8 @@ fn request_output(
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
ec_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
..Default::default()
}
}
@@ -2607,6 +2597,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
},
],
scheduler_stats: None,
@@ -97,6 +97,7 @@ class EngineCoreOutput(
prefill_stats: object | None = None
routed_experts: object | None = None
num_nans_in_logits: int = 0
mm_cache_miss_hashes: list[str] | None = None
class EngineCoreOutputs(