[Rust Frontend] Add external→internal request-id map for abort() (#45137)
Signed-off-by: Sahil Singh <sahiilsiingh37@gmail.com>
This commit is contained in:
@@ -554,6 +554,142 @@ async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_co
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn abort_resolves_external_request_id_to_internal_before_reaching_engine() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-abort".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-abort"));
|
||||
assert!(request.request_id.starts_with("req-abort-"));
|
||||
assert_ne!(request.request_id, "req-abort");
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(&request.request_id, vec![7], None)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
// The abort frame must carry the internal engine id, not the
|
||||
// external "req-abort" id the caller aborted by.
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
assert_eq!(aborted_ids, vec![request.request_id]);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-abort", 4)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
assert_ne!(internal_id, "req-abort");
|
||||
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![7]);
|
||||
|
||||
// Abort by the external id; engine-core only knows the internal id.
|
||||
llm.abort(&["req-abort".to_string()]).await.unwrap();
|
||||
|
||||
// The consumer stream is finalized locally with a clean abort terminal
|
||||
// rather than hanging or surfacing as RequestStreamClosed. The engine sends
|
||||
// no final output for a client abort, so this output is synthesized.
|
||||
let terminal = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
|
||||
assert!(terminal.token_ids.is_empty());
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
drop(stream);
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn abort_by_external_id_aborts_all_internal_requests() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-abort-many".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add_1 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_1[0].as_ref(), &[0x00]);
|
||||
let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
|
||||
|
||||
let add_2 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_2[0].as_ref(), &[0x00]);
|
||||
let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
|
||||
|
||||
assert_eq!(request_1.external_req_id.as_deref(), Some("req-dup-abort"));
|
||||
assert_eq!(request_2.external_req_id.as_deref(), Some("req-dup-abort"));
|
||||
assert_ne!(request_1.request_id, request_2.request_id);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![
|
||||
request_output(&request_1.request_id, vec![7], None),
|
||||
request_output(&request_2.request_id, vec![8], None),
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
// A single abort by the shared external id must abort both
|
||||
// internal engine ids it expanded into.
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let mut aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
aborted_ids.sort();
|
||||
let mut expected = vec![request_1.request_id, request_2.request_id];
|
||||
expected.sort();
|
||||
assert_eq!(aborted_ids, expected);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream_1 = llm.generate(sample_generate_request("req-dup-abort", 4)).await.unwrap();
|
||||
let mut stream_2 = llm.generate(sample_generate_request("req-dup-abort", 4)).await.unwrap();
|
||||
assert_ne!(stream_1.request_id(), stream_2.request_id());
|
||||
|
||||
assert_eq!(stream_1.next().await.unwrap().unwrap().token_ids, vec![7]);
|
||||
assert_eq!(stream_2.next().await.unwrap().unwrap().token_ids, vec![8]);
|
||||
|
||||
llm.abort(&["req-dup-abort".to_string()]).await.unwrap();
|
||||
|
||||
// Both internal requests the external id expanded into are finalized with a
|
||||
// clean abort terminal.
|
||||
for stream in [&mut stream_1, &mut stream_2] {
|
||||
let terminal = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
|
||||
assert!(stream.next().await.is_none());
|
||||
}
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
drop(stream_1);
|
||||
drop(stream_2);
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user