[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Connor Carpenter
2026-08-10 15:05:24 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3c6b2e9c08
commit fa722b9f01
19 changed files with 215 additions and 60 deletions
+8
View File
@@ -457,6 +457,7 @@ impl SharedRuntimeArgs {
coordinator_address: Option<String>,
engine_start_index: u32,
engine_count: usize,
data_parallel_size: usize,
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
@@ -474,6 +475,7 @@ impl SharedRuntimeArgs {
engine_count,
ready_timeout,
},
data_parallel_size,
coordinator_mode: match coordinator_address {
Some(address) => CoordinatorMode::External { address },
None => CoordinatorMode::None,
@@ -530,6 +532,7 @@ impl SharedRuntimeArgs {
local_input_address,
local_output_address,
},
data_parallel_size: engine_count,
coordinator_mode: CoordinatorMode::MaybeInProc,
model: self.model,
served_model_name: self.served_model_name,
@@ -661,6 +664,9 @@ pub struct FrontendArgs {
/// Total number of data-parallel engines expected for this frontend.
#[arg(long, default_value_t = 1)]
pub engine_count: usize,
/// Deployment-wide configured data-parallel size. Defaults to engine count.
#[arg(long)]
pub data_parallel_size: Option<usize>,
/// Shared frontend arguments as one JSON object.
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
@@ -670,6 +676,7 @@ pub struct FrontendArgs {
impl FrontendArgs {
/// Convert the CLI arguments into the OpenAI server's runtime config.
pub fn into_config(self) -> Config {
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
self.runtime.into_bootstrapped_config(
self.listen_fd,
self.input_address,
@@ -677,6 +684,7 @@ impl FrontendArgs {
self.coordinator_address,
self.engine_start_index,
self.engine_count,
data_parallel_size,
)
}
}
+9
View File
@@ -785,6 +785,7 @@ fn frontend_args_accept_json() {
),
engine_start_index: 0,
engine_count: 1,
data_parallel_size: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
engine_ready_timeout_secs: 600,
@@ -846,6 +847,8 @@ fn frontend_args_json_applies_defaults() {
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--engine-count",
"4",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
@@ -861,6 +864,7 @@ fn frontend_args_json_applies_defaults() {
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
assert_eq!(args.runtime.max_logprobs, None);
assert_eq!(args.runtime.shutdown_timeout, 0);
assert_eq!(args.into_config().data_parallel_size, 4);
}
#[test]
@@ -1517,6 +1521,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
"<ipc output>",
),
},
data_parallel_size: 4,
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
@@ -1602,6 +1607,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
local_input_address: None,
local_output_address: None,
},
data_parallel_size: 4,
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
@@ -1685,6 +1691,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
"3",
"--engine-count",
"1",
"--data-parallel-size",
"4",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
@@ -1704,6 +1712,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
engine_count: 1,
ready_timeout: 600s,
},
data_parallel_size: 4,
coordinator_mode: External {
address: "tcp://127.0.0.1:7000",
},
@@ -451,15 +451,6 @@ impl EngineCoreClient {
.world_size
}
/// Return the data parallel size from the parallel config, if available.
pub fn data_parallel_size(&self) -> u64 {
self.engines
.first()
.expect("engine core client requires at least one engine")
.ready_response
.data_parallel_size
}
/// Get the model name associated with this client used for metrics
/// labeling.
pub fn model_name(&self) -> &str {
+22 -18
View File
@@ -161,15 +161,16 @@ impl RequestRegistry {
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
if let Some(rank) = data_parallel_rank {
// Route to the engine at the specified rank index.
let engine_id = EngineId::from_engine_index(rank);
return self
.routing_per_engine
.contains_key(&engine_id)
.then_some(engine_id)
let engine_id = u16::try_from(rank).ok().map(EngineId::from_engine_index);
return engine_id
.filter(|engine_id| self.routing_per_engine.contains_key(engine_id))
.ok_or_else(|| Error::InvalidDataParallelRank {
rank,
num_engines: self.routing_per_engine.len() as u32,
connected_ranks: self
.routing_per_engine
.keys()
.filter_map(EngineId::engine_index)
.collect(),
});
}
@@ -343,6 +344,9 @@ impl RequestRegistry {
}
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
let Ok(engine_index) = u16::try_from(engine_index) else {
return false;
};
let engine_id = EngineId::from_engine_index(engine_index);
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
return false;
@@ -840,26 +844,26 @@ mod tests {
error,
crate::error::Error::InvalidDataParallelRank {
rank: 2,
num_engines: 2,
}
connected_ranks,
} if connected_ranks == vec![0, 1]
));
}
#[test]
fn register_with_rank_on_single_engine_only_accepts_zero() {
let engine_0 = EngineId::from_engine_index(0);
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
fn register_with_rank_uses_global_engine_identity() {
let engine_3 = EngineId::from_engine_index(3);
let mut registry = RequestRegistry::new(&[connected_engine(engine_3.clone())]);
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
assert_eq!(chosen, engine_0);
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(3)).unwrap();
assert_eq!(chosen, engine_3);
let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
let error = registry.register("req-bad".to_string(), None, Some(0)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
rank: 1,
num_engines: 1,
}
rank: 0,
connected_ranks,
} if connected_ranks == vec![3]
));
}
+7 -2
View File
@@ -68,8 +68,13 @@ pub enum Error {
ControlClosed { message: String },
#[error("request `{request_id}` is already in flight")]
DuplicateRequestId { request_id: String },
#[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
InvalidDataParallelRank { rank: u32, num_engines: u32 },
#[error(
"data parallel rank {rank} is not connected to this frontend; connected ranks: {connected_ranks:?}"
)]
InvalidDataParallelRank {
rank: u32,
connected_ranks: Vec<u32>,
},
#[error("engine-core output dispatcher closed: {message}")]
DispatcherClosed { message: String },
#[error("engine-core client is closed: {message}")]
+22 -4
View File
@@ -66,8 +66,8 @@ impl EngineId {
/// Construct an engine id from the Python-compatible engine index encoding
/// (two-byte little-endian).
pub fn from_engine_index(value: u32) -> Self {
Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes()))
pub fn from_engine_index(value: u16) -> Self {
Self(Bytes::copy_from_slice(&value.to_le_bytes()))
}
}
@@ -336,6 +336,22 @@ pub async fn connect_bootstrapped(
if engine_count == 0 {
bail_unexpected_handshake_message!("expected engine_count >= 1");
}
let engine_start_index =
u16::try_from(engine_start_index).map_err(|_| Error::UnexpectedHandshakeMessage {
message: "engine_start_index exceeds the two-byte engine identity limit".to_string(),
})?;
let engine_end_index =
usize::from(engine_start_index).checked_add(engine_count).ok_or_else(|| {
Error::UnexpectedHandshakeMessage {
message: "engine_start_index + engine_count overflows".to_string(),
}
})?;
if engine_end_index > usize::from(u16::MAX) + 1 {
return Err(Error::UnexpectedHandshakeMessage {
message: "engine_start_index + engine_count exceeds the two-byte engine identity limit"
.to_string(),
});
}
let mut input_socket = RouterSocket::new();
let input_address = input_socket.bind(input_address).await?.to_string();
@@ -345,8 +361,10 @@ pub async fn connect_bootstrapped(
let engines = wait_for_input_registrations(
&mut input_socket,
(0..engine_count)
.map(|offset| EngineId::from_engine_index(engine_start_index + offset as u32)),
(0..engine_count).map(|offset| {
let offset = u16::try_from(offset).expect("validated engine offset fits u16");
EngineId::from_engine_index(engine_start_index + offset)
}),
ready_timeout,
)
.await?;
+7 -8
View File
@@ -48,10 +48,13 @@ pub struct Opt {
}
/// Run one mock engine until shutdown or transport failure.
async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) -> Result<()> {
async fn run_engine(engine_index: usize, opt: Opt, shutdown: CancellationToken) -> Result<()> {
let engine_index =
u16::try_from(engine_index).context("mock engine index exceeds two-byte identity limit")?;
let engine_id = EngineId::from_engine_index(engine_index);
let MockEngineSockets { data_sockets, .. } = connect_to_frontend(
&opt.handshake_address,
EngineId::from_engine_index(engine_index),
engine_id,
MockEngineConfig::default(),
)
.await
@@ -71,7 +74,7 @@ async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) ->
));
// Engine loop: input_rx -> engine logic -> output_tx
let mut engine_loop = tokio::spawn(engine::run_engine_loop(
engine_index,
u32::from(engine_index),
opt,
input_rx,
output_tx,
@@ -111,11 +114,7 @@ pub async fn run(opt: Opt, shutdown: CancellationToken) -> Result<()> {
let mut engines = JoinSet::new();
for engine_index in 0..opt.engine_count {
engines.spawn(run_engine(
engine_index as u32,
opt.clone(),
shutdown.clone(),
));
engines.spawn(run_engine(engine_index, opt.clone(), shutdown.clone()));
}
tokio::select! {
@@ -58,6 +58,7 @@ async fn main() -> Result<()> {
local_input_address: None,
local_output_address: None,
},
data_parallel_size: args.engine_count,
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
served_model_name: vec![],
+42
View File
@@ -161,6 +161,8 @@ impl TlsConfig {
pub struct Config {
/// Frontend-to-engine transport setup.
pub transport_mode: TransportMode,
/// Deployment-wide data-parallel size retained by the frontend.
pub data_parallel_size: usize,
/// Requested frontend-side coordinator behavior.
pub coordinator_mode: CoordinatorMode,
/// Backend model identifier used for engine-core loading.
@@ -237,6 +239,46 @@ impl Config {
max_logprobs
);
}
if self.data_parallel_size == 0 {
bail!("data parallel size must be at least 1");
}
if self.data_parallel_size > usize::from(u16::MAX) + 1 {
bail!(
"data parallel size ({}) exceeds the two-byte engine identity limit",
self.data_parallel_size
);
}
match &self.transport_mode {
TransportMode::HandshakeOwner { engine_count, .. } => {
if *engine_count != self.data_parallel_size {
bail!(
"managed frontend engine count ({engine_count}) must equal data parallel size ({})",
self.data_parallel_size
);
}
}
TransportMode::Bootstrapped {
engine_start_index,
engine_count,
..
} => {
if *engine_count == 0 {
bail!("engine count must be at least 1");
}
let engine_start_index = usize::try_from(*engine_start_index)
.map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?;
let engine_end_index =
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
anyhow::anyhow!("engine start index + engine count overflows")
})?;
if engine_end_index > self.data_parallel_size {
bail!(
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}",
self.data_parallel_size
);
}
}
}
Ok(())
}
+1 -1
View File
@@ -31,7 +31,7 @@ impl ControlServiceImpl {
pb::ParallelismInfo {
tensor_parallel_size: ready.tensor_parallel_size,
pipeline_parallel_size: ready.pipeline_parallel_size,
data_parallel_size: ready.data_parallel_size.min(u64::from(u32::MAX)) as u32,
data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32,
data_parallel_rank: ready.data_parallel_rank,
decode_context_parallel_size: ready.decode_context_parallel_size,
}
+25 -2
View File
@@ -22,6 +22,8 @@ use crate::state::AppState;
pub(crate) type InferenceGrpcService = InferenceServer<InferenceServiceImpl>;
const DATA_PARALLEL_RANK_METADATA_KEY: &str = "x-data-parallel-rank";
/// gRPC inference service backed by the shared application state.
pub struct InferenceServiceImpl {
state: Arc<AppState>,
@@ -41,6 +43,7 @@ impl InferenceServiceImpl {
async fn prepare_request(
&self,
mut proto_request: pb::GenerateRequest,
data_parallel_rank: Option<u32>,
stream: bool,
rpc: &'static str,
) -> Result<PreparedGrpcRequest, Status> {
@@ -62,6 +65,7 @@ impl InferenceServiceImpl {
rpc,
%model,
media_count,
?data_parallel_rank,
);
info!(parent: &request_span, "gRPC inference request received");
@@ -70,6 +74,7 @@ impl InferenceServiceImpl {
let mut text_request =
convert::to_text_request(proto_request, stream, self.state.served_model_names())?;
text_request.arrival_time = Some(arrival_time);
text_request.data_parallel_rank = data_parallel_rank;
let media = convert::media_parts_from_request(media)?;
if !media.is_empty() {
@@ -115,6 +120,20 @@ impl InferenceServiceImpl {
}
}
fn data_parallel_rank_from_metadata(
request: &Request<pb::GenerateRequest>,
) -> Result<Option<u32>, Status> {
let Some(value) = request.metadata().get(DATA_PARALLEL_RANK_METADATA_KEY) else {
return Ok(None);
};
let value = value.to_str().map_err(|_| {
Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer")
})?;
value.trim().parse::<u32>().map(Some).map_err(|_| {
Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer")
})
}
#[tonic::async_trait]
impl pb::inference_server::Inference for InferenceServiceImpl {
type GenerateStreamStream =
@@ -125,13 +144,14 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
&self,
request: Request<pb::GenerateRequest>,
) -> Result<Response<pb::GenerateResponse>, Status> {
let data_parallel_rank = data_parallel_rank_from_metadata(&request)?;
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let PreparedGrpcRequest {
text_request,
request_span,
started_at,
} = self.prepare_request(proto_req, false, "Generate").await?;
} = self.prepare_request(proto_req, data_parallel_rank, false, "Generate").await?;
let stream = self
.state
@@ -186,13 +206,16 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
&self,
request: Request<pb::GenerateRequest>,
) -> Result<Response<Self::GenerateStreamStream>, Status> {
let data_parallel_rank = data_parallel_rank_from_metadata(&request)?;
let proto_req = request.into_inner();
let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref());
let PreparedGrpcRequest {
text_request,
request_span,
started_at,
} = self.prepare_request(proto_req, true, "GenerateStream").await?;
} = self
.prepare_request(proto_req, data_parallel_rank, true, "GenerateStream")
.await?;
let stream = self
.state
+37 -5
View File
@@ -843,6 +843,37 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_unconnected_data_parallel_rank_returns_invalid_argument() {
let (mut client, server_task, _engine_task) = grpc_test_server(
EngineId::from_engine_index(3),
default_stream_output_specs(),
)
.await;
let mut request = tonic::Request::new(pb::GenerateRequest {
request_id: "test-unconnected-dp-rank".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
..Default::default()
});
request.metadata_mut().insert(
"x-data-parallel-rank",
"0".parse().expect("valid metadata value"),
);
let status = client
.generate(request)
.await
.expect_err("rank 0 should not select globally ranked engine 3");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("connected ranks: [3]"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
@@ -1523,7 +1554,9 @@ async fn control_aggregates_multi_engine_capacity() {
ready_1.data_parallel_rank = 1;
let engine_tasks = [ready_0, ready_1].map(|ready| {
let engine_id = EngineId::from_engine_index(ready.data_parallel_rank);
let engine_id = EngineId::from_engine_index(
ready.data_parallel_rank.try_into().expect("test rank fits engine identity"),
);
MockEngineTask::new(spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id,
@@ -1551,10 +1584,8 @@ async fn control_aggregates_multi_engine_capacity() {
Llm::new(client),
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
);
let service = ControlServiceImpl::new(Arc::new(AppState::new(
vec!["test-model".to_string()],
chat,
)));
let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4);
let service = ControlServiceImpl::new(Arc::new(state));
let server = pb::control_server::Control::get_server_info(
&service,
@@ -1565,6 +1596,7 @@ async fn control_aggregates_multi_engine_capacity() {
.into_inner();
assert_eq!(server.max_model_len, 4_096);
assert_eq!(server.total_kv_blocks, 30);
assert_eq!(server.parallelism.unwrap().data_parallel_size, 4);
drop(engine_tasks);
}
+1
View File
@@ -141,6 +141,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.with_model_path(config.model.clone())
.with_api_server_options(config.api_server_options)
.with_server_info(ServerInfoSnapshot::from_config(config))
.with_data_parallel_size(config.data_parallel_size)
.with_api_keys(config.api_keys.clone())
.with_cors(config.cors.clone())
.with_profiler(config.profiler.clone()),
+9 -9
View File
@@ -725,11 +725,11 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
/// stays alive for the duration of the test.
async fn test_dev_mode_app_with_ready(
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
data_parallel_size: usize,
) -> (axum::Router, MockEngineTask) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-world-size".to_vec();
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id.clone(),
@@ -750,10 +750,10 @@ async fn test_dev_mode_app_with_ready(
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
let app = build_router_with_dev_mode(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
Arc::new(
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
.with_data_parallel_size(data_parallel_size),
),
true,
);
(app, engine_task)
@@ -6313,7 +6313,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() {
max_model_len: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await;
let (completion_status, completion_json) = post_json(
&mut app,
@@ -6569,10 +6569,10 @@ async fn world_size_endpoint_is_dev_mode_only() {
async fn world_size_includes_data_parallelism_by_default() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
data_parallel_size: 1,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
let response = app
.call(
@@ -6598,7 +6598,7 @@ async fn world_size_excludes_data_parallelism_when_include_dp_false() {
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
let response = app
.call(
+1 -1
View File
@@ -47,7 +47,7 @@ pub async fn get_world_size(
let ws = client.world_size();
let world_size = if params.include_dp {
let dp = client.data_parallel_size();
let dp = state.data_parallel_size() as u64;
ws * dp
} else {
ws
+15
View File
@@ -40,6 +40,8 @@ pub struct AppState {
pub cors: CorsConfig,
/// Runtime server information returned by `/server_info`, when available.
server_info: Option<ServerInfoSnapshot>,
/// Deployment-wide data-parallel size retained by the frontend.
data_parallel_size: usize,
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
api_key_hashes: Vec<ApiKeyHash>,
/// Number of in-flight inference requests currently owned by this frontend.
@@ -69,12 +71,14 @@ impl AppState {
!served_model_names.is_empty(),
"served_model_names must not be empty"
);
let data_parallel_size = chat.engine_core_client().engine_count();
Self {
served_model_names,
chat,
api_server_options: ApiServerOptions::default(),
cors: CorsConfig::default(),
server_info: None,
data_parallel_size,
api_key_hashes: Vec::new(),
server_load: AtomicU64::new(0),
lora_manager: LoraManager::new(),
@@ -114,6 +118,17 @@ impl AppState {
self
}
/// Set the deployment-wide data-parallel size reported by frontend APIs.
pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self {
self.data_parallel_size = size;
self
}
/// Return the deployment-wide data-parallel size.
pub(crate) fn data_parallel_size(&self) -> usize {
self.data_parallel_size
}
/// Build a `/server_info` response payload.
pub(crate) fn server_info_response(
&self,
+4 -1
View File
@@ -62,7 +62,10 @@ impl Error {
| Self::InvalidRepetitionDetection { .. }
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true,
| Self::Llm(LlmError::EmptyPromptTokenIds { .. })
| Self::Llm(LlmError::EngineCoreClient(
EngineCoreError::InvalidDataParallelRank { .. },
)) => true,
_ => false,
}
}