[Rust Frontend][gRPC] Surface engine generation errors (#56405)

Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Alec
2026-09-11 15:31:55 -07:00
committed by GitHub
co-authored by Codex
parent bbaa1b9340
commit 988d9b6777
3 changed files with 93 additions and 32 deletions
+18 -17
View File
@@ -391,13 +391,14 @@ pub fn to_sequence_output(
logprobs: Option<&DecodedLogprobs>,
finished: Option<&Finished>,
opts: &ResponseOpts,
) -> pb::SequenceOutput {
) -> Result<pb::SequenceOutput, Status> {
let finish_info = finished.map(|f| to_finish_info(f, token_ids)).transpose()?;
let (lp_values, rank_values, candidates) = match logprobs {
Some(lp) if opts.output_logprobs => output_logprobs_to_proto(lp),
_ => (vec![], vec![], vec![]),
};
pb::SequenceOutput {
Ok(pb::SequenceOutput {
index: 0, // TODO: multi-sequence (n > 1) not supported
text: if opts.output_text {
delta.to_string()
@@ -413,11 +414,11 @@ pub fn to_sequence_output(
logprobs: lp_values,
ranks: rank_values,
candidate_tokens: candidates,
finish_info: finished.map(|f| to_finish_info(f, token_ids)),
}
finish_info,
})
}
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> Result<pb::FinishInfo, Status> {
use pb::finish_info::FinishReason as PbFinishReason;
let (finish_reason, stop_reason) = match &finished.finish_reason {
@@ -438,18 +439,17 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
(PbFinishReason::Stop as i32, sr)
}
FinishReason::Length => (PbFinishReason::Length as i32, None),
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => {
(PbFinishReason::Aborted as i32, None)
}
FinishReason::Error => return Err(Status::internal("engine failed during generation")),
FinishReason::Abort | FinishReason::Repetition(_) => (PbFinishReason::Aborted as i32, None),
};
pb::FinishInfo {
Ok(pb::FinishInfo {
num_output_tokens: finished.usage.output_token_count as u32,
finish_reason,
stop_reason,
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct),
}
})
}
// ========================================================================================
@@ -709,7 +709,7 @@ mod tests {
let fin = finished(FinishReason::Stop(None));
let token_ids = [1_u32, 2, 3, 151643];
let info = to_finish_info(&fin, &token_ids);
let info = to_finish_info(&fin, &token_ids).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, Some(PbStopReason::EosTokenId(151643)));
@@ -719,7 +719,7 @@ mod tests {
fn eos_stop_with_empty_token_ids_leaves_stop_reason_unset() {
let fin = finished(FinishReason::Stop(None));
let info = to_finish_info(&fin, &[]);
let info = to_finish_info(&fin, &[]).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, None);
@@ -730,7 +730,7 @@ mod tests {
let fin = finished(FinishReason::Stop(Some(StopReason::TokenId(42))));
// Terminal token list should be ignored when an explicit stop reason is
// present.
let info = to_finish_info(&fin, &[7, 42]);
let info = to_finish_info(&fin, &[7, 42]).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(info.stop_reason, Some(PbStopReason::StopTokenId(42)));
@@ -740,7 +740,7 @@ mod tests {
fn explicit_stop_string_is_preserved() {
let fin = finished(FinishReason::Stop(Some(StopReason::Text("</stop>".into()))));
let info = to_finish_info(&fin, &[1, 2, 3]);
let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(
@@ -753,7 +753,7 @@ mod tests {
fn length_finish_has_no_stop_reason() {
let fin = finished(FinishReason::Length);
let info = to_finish_info(&fin, &[1, 2, 3]);
let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Length as i32);
assert_eq!(info.stop_reason, None);
@@ -763,7 +763,7 @@ mod tests {
fn abort_finish_is_mapped_to_aborted() {
let fin = finished(FinishReason::Abort);
let info = to_finish_info(&fin, &[]);
let info = to_finish_info(&fin, &[]).expect("finish info");
assert_eq!(info.finish_reason, PbFinishReason::Aborted as i32);
assert_eq!(info.stop_reason, None);
@@ -778,7 +778,8 @@ mod tests {
..Default::default()
};
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts);
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts)
.expect("sequence output");
let finish = out.finish_info.expect("finish_info should be present");
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
+16 -15
View File
@@ -241,12 +241,6 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
.instrument(request_span.clone())
.await
.map_err(|error| log_text_error(&request_span, started_at, "collection", error))?;
info!(
parent: &request_span,
elapsed_ms = started_at.elapsed().as_millis() as u64,
"gRPC inference request completed"
);
// Build the single aggregated response.
let prompt_info = convert::to_prompt_info(
&collected.prompt_token_ids,
@@ -267,6 +261,11 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
collected.logprobs.as_ref(),
Some(&finish_info),
&response_opts,
)?;
info!(
parent: &request_span,
elapsed_ms = started_at.elapsed().as_millis() as u64,
"gRPC inference request completed"
);
Ok(Response::new(pb::GenerateResponse {
@@ -331,19 +330,21 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
logprobs,
},
finished,
}) => Ok(pb::GenerateResponse {
}) => convert::to_sequence_output(
&decoded.text,
&token_ids,
logprobs.as_ref(),
finished.as_deref(),
&response_opts,
)
.map(|outputs| pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&decoded.text,
&token_ids,
logprobs.as_ref(),
finished.as_deref(),
&response_opts,
)),
outputs: Some(outputs),
}),
};
if tx.send(response).await.is_err() {
let failed = response.is_err();
if tx.send(response).await.is_err() || failed {
break;
}
}
+59
View File
@@ -680,6 +680,65 @@ async fn unary_generate_returns_collected_text() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn engine_error_finish_returns_internal_for_unary_and_streaming() {
for streaming in [false, true] {
for prefix in [vec![], vec![b'h' as u32]] {
let mut output_specs = Vec::new();
if !prefix.is_empty() {
output_specs.push((prefix.clone(), None));
}
output_specs.push((vec![], Some(EngineCoreFinishReason::Error)));
let (mut client, server_task, engine_task) =
grpc_test_server(b"engine-grpc-error", output_specs).await;
let request = pb::GenerateRequest {
request_id: "test-engine-error".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
response: Some(pb::ResponseOptions {
output_token_ids: true,
..Default::default()
}),
..Default::default()
};
let status = if streaming {
let mut stream =
client.generate_stream(request).await.expect("start stream").into_inner();
let mut received_tokens = Vec::new();
let status = loop {
match stream.message().await {
Ok(Some(response)) => {
if let Some(output) = response.outputs {
assert!(
output.finish_info.is_none(),
"error became a finish event"
);
received_tokens.extend(output.token_ids);
}
}
Ok(None) => panic!("engine error ended as a successful stream"),
Err(status) => break status,
}
};
assert_eq!(received_tokens, prefix);
status
} else {
client.generate(request).await.expect_err("engine error must fail the RPC")
};
assert_eq!(status.code(), tonic::Code::Internal);
engine_task.await.expect("mock engine task");
server_task.abort();
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_with_token_ids_prompt() {