[Rust Frontend][gRPC] Surface engine generation errors (#56405)
Signed-off-by: Alec Flowers <aflowers@nvidia.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -391,13 +391,14 @@ pub fn to_sequence_output(
|
||||
logprobs: Option<&DecodedLogprobs>,
|
||||
finished: Option<&Finished>,
|
||||
opts: &ResponseOpts,
|
||||
) -> pb::SequenceOutput {
|
||||
) -> Result<pb::SequenceOutput, Status> {
|
||||
let finish_info = finished.map(|f| to_finish_info(f, token_ids)).transpose()?;
|
||||
let (lp_values, rank_values, candidates) = match logprobs {
|
||||
Some(lp) if opts.output_logprobs => output_logprobs_to_proto(lp),
|
||||
_ => (vec![], vec![], vec![]),
|
||||
};
|
||||
|
||||
pb::SequenceOutput {
|
||||
Ok(pb::SequenceOutput {
|
||||
index: 0, // TODO: multi-sequence (n > 1) not supported
|
||||
text: if opts.output_text {
|
||||
delta.to_string()
|
||||
@@ -413,11 +414,11 @@ pub fn to_sequence_output(
|
||||
logprobs: lp_values,
|
||||
ranks: rank_values,
|
||||
candidate_tokens: candidates,
|
||||
finish_info: finished.map(|f| to_finish_info(f, token_ids)),
|
||||
}
|
||||
finish_info,
|
||||
})
|
||||
}
|
||||
|
||||
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
|
||||
fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> Result<pb::FinishInfo, Status> {
|
||||
use pb::finish_info::FinishReason as PbFinishReason;
|
||||
|
||||
let (finish_reason, stop_reason) = match &finished.finish_reason {
|
||||
@@ -438,18 +439,17 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
|
||||
(PbFinishReason::Stop as i32, sr)
|
||||
}
|
||||
FinishReason::Length => (PbFinishReason::Length as i32, None),
|
||||
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => {
|
||||
(PbFinishReason::Aborted as i32, None)
|
||||
}
|
||||
FinishReason::Error => return Err(Status::internal("engine failed during generation")),
|
||||
FinishReason::Abort | FinishReason::Repetition(_) => (PbFinishReason::Aborted as i32, None),
|
||||
};
|
||||
|
||||
pb::FinishInfo {
|
||||
Ok(pb::FinishInfo {
|
||||
num_output_tokens: finished.usage.output_token_count as u32,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
ec_transfer_params: finished.ec_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
@@ -709,7 +709,7 @@ mod tests {
|
||||
let fin = finished(FinishReason::Stop(None));
|
||||
let token_ids = [1_u32, 2, 3, 151643];
|
||||
|
||||
let info = to_finish_info(&fin, &token_ids);
|
||||
let info = to_finish_info(&fin, &token_ids).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, Some(PbStopReason::EosTokenId(151643)));
|
||||
@@ -719,7 +719,7 @@ mod tests {
|
||||
fn eos_stop_with_empty_token_ids_leaves_stop_reason_unset() {
|
||||
let fin = finished(FinishReason::Stop(None));
|
||||
|
||||
let info = to_finish_info(&fin, &[]);
|
||||
let info = to_finish_info(&fin, &[]).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
@@ -730,7 +730,7 @@ mod tests {
|
||||
let fin = finished(FinishReason::Stop(Some(StopReason::TokenId(42))));
|
||||
// Terminal token list should be ignored when an explicit stop reason is
|
||||
// present.
|
||||
let info = to_finish_info(&fin, &[7, 42]);
|
||||
let info = to_finish_info(&fin, &[7, 42]).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(info.stop_reason, Some(PbStopReason::StopTokenId(42)));
|
||||
@@ -740,7 +740,7 @@ mod tests {
|
||||
fn explicit_stop_string_is_preserved() {
|
||||
let fin = finished(FinishReason::Stop(Some(StopReason::Text("</stop>".into()))));
|
||||
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]);
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(
|
||||
@@ -753,7 +753,7 @@ mod tests {
|
||||
fn length_finish_has_no_stop_reason() {
|
||||
let fin = finished(FinishReason::Length);
|
||||
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]);
|
||||
let info = to_finish_info(&fin, &[1, 2, 3]).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Length as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
@@ -763,7 +763,7 @@ mod tests {
|
||||
fn abort_finish_is_mapped_to_aborted() {
|
||||
let fin = finished(FinishReason::Abort);
|
||||
|
||||
let info = to_finish_info(&fin, &[]);
|
||||
let info = to_finish_info(&fin, &[]).expect("finish info");
|
||||
|
||||
assert_eq!(info.finish_reason, PbFinishReason::Aborted as i32);
|
||||
assert_eq!(info.stop_reason, None);
|
||||
@@ -778,7 +778,8 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts);
|
||||
let out = to_sequence_output("hello", &[10, 20, 30], None, Some(&fin), &opts)
|
||||
.expect("sequence output");
|
||||
|
||||
let finish = out.finish_info.expect("finish_info should be present");
|
||||
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
|
||||
|
||||
@@ -241,12 +241,6 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
.map_err(|error| log_text_error(&request_span, started_at, "collection", error))?;
|
||||
info!(
|
||||
parent: &request_span,
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
"gRPC inference request completed"
|
||||
);
|
||||
|
||||
// Build the single aggregated response.
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
&collected.prompt_token_ids,
|
||||
@@ -267,6 +261,11 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
collected.logprobs.as_ref(),
|
||||
Some(&finish_info),
|
||||
&response_opts,
|
||||
)?;
|
||||
info!(
|
||||
parent: &request_span,
|
||||
elapsed_ms = started_at.elapsed().as_millis() as u64,
|
||||
"gRPC inference request completed"
|
||||
);
|
||||
|
||||
Ok(Response::new(pb::GenerateResponse {
|
||||
@@ -331,19 +330,21 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
}) => convert::to_sequence_output(
|
||||
&decoded.text,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_deref(),
|
||||
&response_opts,
|
||||
)
|
||||
.map(|outputs| pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&decoded.text,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_deref(),
|
||||
&response_opts,
|
||||
)),
|
||||
outputs: Some(outputs),
|
||||
}),
|
||||
};
|
||||
|
||||
if tx.send(response).await.is_err() {
|
||||
let failed = response.is_err();
|
||||
if tx.send(response).await.is_err() || failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -680,6 +680,65 @@ async fn unary_generate_returns_collected_text() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn engine_error_finish_returns_internal_for_unary_and_streaming() {
|
||||
for streaming in [false, true] {
|
||||
for prefix in [vec![], vec![b'h' as u32]] {
|
||||
let mut output_specs = Vec::new();
|
||||
if !prefix.is_empty() {
|
||||
output_specs.push((prefix.clone(), None));
|
||||
}
|
||||
output_specs.push((vec![], Some(EngineCoreFinishReason::Error)));
|
||||
let (mut client, server_task, engine_task) =
|
||||
grpc_test_server(b"engine-grpc-error", output_specs).await;
|
||||
let request = pb::GenerateRequest {
|
||||
request_id: "test-engine-error".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
response: Some(pb::ResponseOptions {
|
||||
output_token_ids: true,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let status = if streaming {
|
||||
let mut stream =
|
||||
client.generate_stream(request).await.expect("start stream").into_inner();
|
||||
let mut received_tokens = Vec::new();
|
||||
let status = loop {
|
||||
match stream.message().await {
|
||||
Ok(Some(response)) => {
|
||||
if let Some(output) = response.outputs {
|
||||
assert!(
|
||||
output.finish_info.is_none(),
|
||||
"error became a finish event"
|
||||
);
|
||||
received_tokens.extend(output.token_ids);
|
||||
}
|
||||
}
|
||||
Ok(None) => panic!("engine error ended as a successful stream"),
|
||||
Err(status) => break status,
|
||||
}
|
||||
};
|
||||
assert_eq!(received_tokens, prefix);
|
||||
status
|
||||
} else {
|
||||
client.generate(request).await.expect_err("engine error must fail the RPC")
|
||||
};
|
||||
assert_eq!(status.code(), tonic::Code::Internal);
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_with_token_ids_prompt() {
|
||||
|
||||
Reference in New Issue
Block a user