diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 2a3f41ffa4..ca44f59c93 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -391,13 +391,14 @@ pub fn to_sequence_output( logprobs: Option<&DecodedLogprobs>, finished: Option<&Finished>, opts: &ResponseOpts, -) -> pb::SequenceOutput { +) -> Result { + let finish_info = finished.map(|f| to_finish_info(f, token_ids)).transpose()?; let (lp_values, rank_values, candidates) = match logprobs { Some(lp) if opts.output_logprobs => output_logprobs_to_proto(lp), _ => (vec![], vec![], vec![]), }; - pb::SequenceOutput { + Ok(pb::SequenceOutput { index: 0, // TODO: multi-sequence (n > 1) not supported text: if opts.output_text { delta.to_string() @@ -413,11 +414,11 @@ pub fn to_sequence_output( logprobs: lp_values, ranks: rank_values, candidate_tokens: candidates, - finish_info: finished.map(|f| to_finish_info(f, token_ids)), - } + finish_info, + }) } -fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo { +fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> Result { use pb::finish_info::FinishReason as PbFinishReason; let (finish_reason, stop_reason) = match &finished.finish_reason { @@ -438,18 +439,17 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo { (PbFinishReason::Stop as i32, sr) } FinishReason::Length => (PbFinishReason::Length as i32, None), - FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => { - (PbFinishReason::Aborted as i32, None) - } + FinishReason::Error => return Err(Status::internal("engine failed during generation")), + FinishReason::Abort | FinishReason::Repetition(_) => (PbFinishReason::Aborted as i32, None), }; - pb::FinishInfo { + Ok(pb::FinishInfo { num_output_tokens: finished.usage.output_token_count as u32, finish_reason, stop_reason, kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct), ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct), - } + }) } // ======================================================================================== @@ -709,7 +709,7 @@ mod tests { let fin = finished(FinishReason::Stop(None)); let token_ids = [1_u32, 2, 3, 151643]; - let info = to_finish_info(&fin, &token_ids); + let info = to_finish_info(&fin, &token_ids).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Stop as i32); assert_eq!(info.stop_reason, Some(PbStopReason::EosTokenId(151643))); @@ -719,7 +719,7 @@ mod tests { fn eos_stop_with_empty_token_ids_leaves_stop_reason_unset() { let fin = finished(FinishReason::Stop(None)); - let info = to_finish_info(&fin, &[]); + let info = to_finish_info(&fin, &[]).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Stop as i32); assert_eq!(info.stop_reason, None); @@ -730,7 +730,7 @@ mod tests { let fin = finished(FinishReason::Stop(Some(StopReason::TokenId(42)))); // Terminal token list should be ignored when an explicit stop reason is // present. - let info = to_finish_info(&fin, &[7, 42]); + let info = to_finish_info(&fin, &[7, 42]).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Stop as i32); assert_eq!(info.stop_reason, Some(PbStopReason::StopTokenId(42))); @@ -740,7 +740,7 @@ mod tests { fn explicit_stop_string_is_preserved() { let fin = finished(FinishReason::Stop(Some(StopReason::Text("".into())))); - let info = to_finish_info(&fin, &[1, 2, 3]); + let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Stop as i32); assert_eq!( @@ -753,7 +753,7 @@ mod tests { fn length_finish_has_no_stop_reason() { let fin = finished(FinishReason::Length); - let info = to_finish_info(&fin, &[1, 2, 3]); + let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Length as i32); assert_eq!(info.stop_reason, None); @@ -763,7 +763,7 @@ mod tests { fn abort_finish_is_mapped_to_aborted() { let fin = finished(FinishReason::Abort); - let info = to_finish_info(&fin, &[]); + let info = to_finish_info(&fin, &[]).expect("finish info"); assert_eq!(info.finish_reason, PbFinishReason::Aborted as i32); assert_eq!(info.stop_reason, None); @@ -778,7 +778,8 @@ mod tests { ..Default::default() }; - let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts); + let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts) + .expect("sequence output"); let finish = out.finish_info.expect("finish_info should be present"); assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32); diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index f864cbde59..7051558cd8 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -241,12 +241,6 @@ impl pb::inference_server::Inference for InferenceServiceImpl { .instrument(request_span.clone()) .await .map_err(|error| log_text_error(&request_span, started_at, "collection", error))?; - info!( - parent: &request_span, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "gRPC inference request completed" - ); - // Build the single aggregated response. let prompt_info = convert::to_prompt_info( &collected.prompt_token_ids, @@ -267,6 +261,11 @@ impl pb::inference_server::Inference for InferenceServiceImpl { collected.logprobs.as_ref(), Some(&finish_info), &response_opts, + )?; + info!( + parent: &request_span, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "gRPC inference request completed" ); Ok(Response::new(pb::GenerateResponse { @@ -331,19 +330,21 @@ impl pb::inference_server::Inference for InferenceServiceImpl { logprobs, }, finished, - }) => Ok(pb::GenerateResponse { + }) => convert::to_sequence_output( + &decoded.text, + &token_ids, + logprobs.as_ref(), + finished.as_deref(), + &response_opts, + ) + .map(|outputs| pb::GenerateResponse { prompt_info: None, - outputs: Some(convert::to_sequence_output( - &decoded.text, - &token_ids, - logprobs.as_ref(), - finished.as_deref(), - &response_opts, - )), + outputs: Some(outputs), }), }; - if tx.send(response).await.is_err() { + let failed = response.is_err(); + if tx.send(response).await.is_err() || failed { break; } } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index ea4364480c..59403277ee 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -680,6 +680,65 @@ async fn unary_generate_returns_collected_text() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn engine_error_finish_returns_internal_for_unary_and_streaming() { + for streaming in [false, true] { + for prefix in [vec![], vec![b'h' as u32]] { + let mut output_specs = Vec::new(); + if !prefix.is_empty() { + output_specs.push((prefix.clone(), None)); + } + output_specs.push((vec![], Some(EngineCoreFinishReason::Error))); + let (mut client, server_task, engine_task) = + grpc_test_server(b"engine-grpc-error", output_specs).await; + let request = pb::GenerateRequest { + request_id: "test-engine-error".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())), + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 10, + ..Default::default() + }), + response: Some(pb::ResponseOptions { + output_token_ids: true, + ..Default::default() + }), + ..Default::default() + }; + + let status = if streaming { + let mut stream = + client.generate_stream(request).await.expect("start stream").into_inner(); + let mut received_tokens = Vec::new(); + let status = loop { + match stream.message().await { + Ok(Some(response)) => { + if let Some(output) = response.outputs { + assert!( + output.finish_info.is_none(), + "error became a finish event" + ); + received_tokens.extend(output.token_ids); + } + } + Ok(None) => panic!("engine error ended as a successful stream"), + Err(status) => break status, + } + }; + assert_eq!(received_tokens, prefix); + status + } else { + client.generate(request).await.expect_err("engine error must fail the RPC") + }; + assert_eq!(status.code(), tonic::Code::Internal); + + engine_task.await.expect("mock engine task"); + server_task.abort(); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_with_token_ids_prompt() {