[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ffdde24f54
commit
35e920d671
@@ -87,8 +87,9 @@ fn build_request(
|
||||
}
|
||||
|
||||
async fn wait_for_final_output(
|
||||
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
|
||||
stream: vllm_engine_core_client::EngineCoreOutputStream,
|
||||
) -> Result<EngineCoreStreamOutput> {
|
||||
let mut stream = stream.into_outputs();
|
||||
while let Some(output) = stream.next().await {
|
||||
let output = output.context("failed to receive engine-core output")?;
|
||||
if output.finished() {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -25,10 +26,11 @@ use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
pub(crate) mod imp;
|
||||
mod output_channel;
|
||||
mod state;
|
||||
mod stream;
|
||||
|
||||
pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
|
||||
pub use stream::{EngineCoreOutputStream, EngineCoreStreamDelivery, EngineCoreStreamOutput};
|
||||
|
||||
/// How the frontend acquires its request/response transport with Python
|
||||
/// `EngineCoreProc`s.
|
||||
@@ -614,6 +616,12 @@ fn validate_lora_capabilities(engines: &[ConnectedEngine]) -> Result<()> {
|
||||
impl EngineCoreClient {
|
||||
/// Add a new request to the engine and return a per-request raw output
|
||||
/// stream.
|
||||
///
|
||||
/// With `sampling_params.stream_interval` above one, outputs after the
|
||||
/// first are delivered to the stream in batches of at least that many new
|
||||
/// tokens, and the terminal output is delivered immediately together with
|
||||
/// any held-back outputs. The stream still yields every raw output in
|
||||
/// order; batching only reduces how often the consuming task is woken.
|
||||
pub async fn call(&self, mut req: EngineCoreRequest) -> Result<EngineCoreOutputStream> {
|
||||
req.client_index = self.config.client_index;
|
||||
req.validate()?;
|
||||
@@ -628,8 +636,15 @@ impl EngineCoreClient {
|
||||
let request_id = req.request_id.clone();
|
||||
let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone());
|
||||
let data_parallel_rank = req.data_parallel_rank;
|
||||
let (engine_id, rx) =
|
||||
self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
|
||||
let stream_interval = (req.sampling_params.as_ref())
|
||||
.and_then(|params| params.stream_interval)
|
||||
.unwrap_or(NonZeroU32::MIN);
|
||||
let (engine_id, rx) = self.inner.register_request(
|
||||
request_id.clone(),
|
||||
lora_name,
|
||||
data_parallel_rank,
|
||||
stream_interval,
|
||||
)?;
|
||||
|
||||
// Construct the output stream first before actually sending the request to the engine.
|
||||
// This ensures that cancelling the future will properly clean up the request registration
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
@@ -15,7 +16,9 @@ use tracing::{debug, info, trace, warn};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::RouterSendHalf;
|
||||
|
||||
use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, UtilityRegistry};
|
||||
use crate::client::state::{
|
||||
OutputReceiver, OutputSender, RequestRegistry, UtilityReceiver, UtilityRegistry,
|
||||
};
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
@@ -111,12 +114,13 @@ impl ClientInner {
|
||||
request_id: String,
|
||||
lora_name: Option<String>,
|
||||
data_parallel_rank: Option<u32>,
|
||||
stream_interval: NonZeroU32,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
registry.register(request_id, lora_name, data_parallel_rank)
|
||||
registry.register(request_id, lora_name, data_parallel_rank, stream_interval)
|
||||
}
|
||||
|
||||
/// Allocate the next utility `call_id` and register its waiting receiver.
|
||||
@@ -159,13 +163,15 @@ impl ClientInner {
|
||||
Ok(registry.abortable_request_ids(request_ids))
|
||||
}
|
||||
|
||||
/// Obtain stream senders for a whole engine output batch with one registry
|
||||
/// lock acquisition.
|
||||
pub fn take_senders_for_outputs<'a>(
|
||||
/// Send a whole engine output batch to the request streams with one
|
||||
/// registry lock acquisition. Returns the outputs whose requests are no
|
||||
/// longer tracked.
|
||||
pub fn send_outputs(
|
||||
&self,
|
||||
outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
|
||||
) -> Vec<Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>>> {
|
||||
self.request_reg.lock().senders_for_outputs(outputs)
|
||||
outputs: impl IntoIterator<Item = EngineCoreStreamOutput>,
|
||||
) -> Vec<EngineCoreStreamOutput> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
outputs.into_iter().filter_map(|output| registry.send_output(output)).collect()
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
@@ -173,7 +179,7 @@ impl ClientInner {
|
||||
pub fn finish_requests<'a>(
|
||||
&self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
) -> Vec<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
|
||||
) -> Vec<OutputSender> {
|
||||
self.request_reg.lock().finish_many(request_ids)
|
||||
}
|
||||
|
||||
@@ -214,7 +220,7 @@ impl ClientInner {
|
||||
|
||||
// Notify all ongoing requests that the client is closed.
|
||||
for sender in request_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
sender.send_error(Error::Shared(persistent_error.clone()));
|
||||
}
|
||||
for sender in utility_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
@@ -413,6 +419,16 @@ pub(crate) async fn run_abort_loop(
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of tokens one output accounts for in `vllm:iteration_tokens_total`.
|
||||
fn output_iteration_tokens(output: &EngineCoreOutput) -> u64 {
|
||||
let mut tokens = output.new_token_ids.len() as u64;
|
||||
if let Some(prefill) = &output.prefill_stats {
|
||||
// The engine emits prefill_stats once, on the first output.
|
||||
tokens += u64::from(prefill.num_computed_tokens);
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
/// Background loop that listens for engine-core outputs and dispatches them to
|
||||
/// the corresponding request streams based on their `request_id`.
|
||||
pub(crate) async fn run_output_dispatcher_loop(
|
||||
@@ -435,29 +451,20 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
match outputs {
|
||||
EngineCoreOutputs::RequestBatch(batch) => {
|
||||
let has_outputs = !batch.outputs.is_empty();
|
||||
let mut iteration_tokens = 0_u64;
|
||||
let senders = inner.take_senders_for_outputs(&batch.outputs);
|
||||
for (output, sender) in batch.outputs.into_iter().zip(senders) {
|
||||
let request_id = output.request_id.clone();
|
||||
let Some(sender) = sender else {
|
||||
debug!(request_id, "dropping output for inactive request");
|
||||
continue;
|
||||
};
|
||||
|
||||
iteration_tokens += output.new_token_ids.len() as u64;
|
||||
if let Some(prefill) = &output.prefill_stats {
|
||||
// The engine emits prefill_stats once, on the first output.
|
||||
iteration_tokens += u64::from(prefill.num_computed_tokens);
|
||||
}
|
||||
|
||||
let wrapped_output = EngineCoreStreamOutput {
|
||||
engine_index: batch.engine_index,
|
||||
timestamp: batch.timestamp,
|
||||
output,
|
||||
};
|
||||
if sender.send(Ok(wrapped_output)).is_err() {
|
||||
debug!(request_id, "request output stream receiver dropped");
|
||||
}
|
||||
let mut iteration_tokens: u64 =
|
||||
batch.outputs.iter().map(output_iteration_tokens).sum();
|
||||
let (engine_index, timestamp) = (batch.engine_index, batch.timestamp);
|
||||
let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput {
|
||||
engine_index,
|
||||
timestamp,
|
||||
output,
|
||||
});
|
||||
for output in inner.send_outputs(outputs) {
|
||||
debug!(
|
||||
request_id = output.request_id,
|
||||
"dropping output for inactive request"
|
||||
);
|
||||
iteration_tokens -= output_iteration_tokens(&output);
|
||||
}
|
||||
|
||||
if has_outputs
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Per-request output channel between the output dispatcher and
|
||||
//! [`EngineCoreOutputStream`](crate::EngineCoreOutputStream).
|
||||
//!
|
||||
//! With a stream interval above one, the sender holds back the outputs after
|
||||
//! the first one until they carry at least that many new tokens, then delivers
|
||||
//! them in one channel message, which the stream yields as one
|
||||
//! [`EngineCoreStreamDelivery`](crate::EngineCoreStreamDelivery). The request
|
||||
//! task is then woken once per delivery instead of once per engine step.
|
||||
//!
|
||||
//! Outputs are never merged, reordered, or dropped here; the consumer merges a
|
||||
//! delivery into one output (see `vllm-llm`). Merging here would hide the raw
|
||||
//! outputs that request metrics observe one by one (per-step timestamps for
|
||||
//! inter-token latency, prefill stats, lifecycle events), and would have to
|
||||
//! combine wire-level fields such as undecoded logprobs tensors and opaque
|
||||
//! values.
|
||||
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::client::stream::{EngineCoreStreamDelivery as Delivery, EngineCoreStreamOutput};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Create the output channel for one request.
|
||||
pub(crate) fn output_channel(stream_interval: NonZeroU32) -> (OutputSender, OutputReceiver) {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let sender = OutputSender {
|
||||
tx,
|
||||
stream_interval: stream_interval.get() as usize,
|
||||
delivered_first: false,
|
||||
buffered: Vec::new(),
|
||||
buffered_tokens: 0,
|
||||
};
|
||||
(sender, rx)
|
||||
}
|
||||
|
||||
/// Receiving half of a request's output channel.
|
||||
pub(crate) type OutputReceiver = mpsc::UnboundedReceiver<Result<Delivery>>;
|
||||
|
||||
/// Sending half of a request's output channel, owned by its registry entry.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct OutputSender {
|
||||
tx: mpsc::UnboundedSender<Result<Delivery>>,
|
||||
/// Minimum number of new tokens per delivery after the first one.
|
||||
stream_interval: usize,
|
||||
/// Whether the first output has been delivered.
|
||||
delivered_first: bool,
|
||||
/// Outputs held back since the last delivery.
|
||||
buffered: Vec<EngineCoreStreamOutput>,
|
||||
/// Number of new tokens in `buffered`.
|
||||
buffered_tokens: usize,
|
||||
}
|
||||
|
||||
impl OutputSender {
|
||||
/// Send one output, or hold it back until the held-back outputs reach the
|
||||
/// stream interval. The first output and the terminal output are always
|
||||
/// delivered immediately, the latter together with any held-back outputs.
|
||||
pub fn send(&mut self, output: EngineCoreStreamOutput) {
|
||||
let finished = output.finished();
|
||||
if self.buffered.is_empty()
|
||||
&& (finished || !self.delivered_first || self.stream_interval == 1)
|
||||
{
|
||||
self.delivered_first = true;
|
||||
self.deliver(Delivery::One(output));
|
||||
return;
|
||||
}
|
||||
|
||||
self.buffered_tokens += output.new_token_ids.len();
|
||||
self.buffered.push(output);
|
||||
if finished || self.buffered_tokens >= self.stream_interval {
|
||||
self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/// Deliver the held-back outputs, if any.
|
||||
pub fn flush(&mut self) {
|
||||
if self.buffered.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.buffered_tokens = 0;
|
||||
let outputs = std::mem::take(&mut self.buffered);
|
||||
self.deliver(Delivery::Many(outputs));
|
||||
}
|
||||
|
||||
/// Deliver the held-back outputs, then terminate the stream with `error`.
|
||||
pub fn send_error(mut self, error: Error) {
|
||||
self.flush();
|
||||
let _ = self.tx.send(Err(error));
|
||||
}
|
||||
|
||||
fn deliver(&self, delivery: Delivery) {
|
||||
if let Err(mpsc::error::SendError(Ok(delivery))) = self.tx.send(Ok(delivery)) {
|
||||
debug!(
|
||||
request_id = delivery.first().request_id,
|
||||
"request output stream receiver dropped"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OutputSender {
|
||||
fn drop(&mut self) {
|
||||
// Every path that ends a live request delivers held-back outputs first
|
||||
// (terminal output, `flush`, or `send_error`). Outputs left here are lost.
|
||||
if !self.buffered.is_empty() && !self.tx.is_closed() {
|
||||
warn!(
|
||||
request_id = self.buffered[0].request_id,
|
||||
count = self.buffered.len(),
|
||||
"dropping request output sender with held-back outputs"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use expect_test::expect;
|
||||
use tokio::sync::mpsc::error::TryRecvError;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
|
||||
fn output(token_ids: &[u32]) -> EngineCoreStreamOutput {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
output: EngineCoreOutput {
|
||||
request_id: "req".to_string(),
|
||||
new_token_ids: token_ids.to_vec(),
|
||||
..EngineCoreOutput::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn finished(token_ids: &[u32]) -> EngineCoreStreamOutput {
|
||||
let mut output = output(token_ids);
|
||||
output.output.finish_reason = Some(EngineCoreFinishReason::Length);
|
||||
output
|
||||
}
|
||||
|
||||
fn channel(stream_interval: u32) -> (OutputSender, OutputReceiver) {
|
||||
output_channel(NonZeroU32::new(stream_interval).unwrap())
|
||||
}
|
||||
|
||||
/// Render a delivery as the token ids of its outputs.
|
||||
fn token_ids(delivery: Delivery) -> Vec<Vec<u32>> {
|
||||
delivery.into_iter().map(|output| output.output.new_token_ids).collect()
|
||||
}
|
||||
|
||||
/// Drain all pending channel messages, rendering each delivery as the
|
||||
/// token ids of its outputs.
|
||||
fn deliveries(receiver: &mut OutputReceiver) -> Vec<Vec<Vec<u32>>> {
|
||||
let mut deliveries = Vec::new();
|
||||
loop {
|
||||
match receiver.try_recv() {
|
||||
Ok(Ok(delivery)) => deliveries.push(token_ids(delivery)),
|
||||
Ok(Err(error)) => panic!("unexpected error delivery: {error:?}"),
|
||||
Err(TryRecvError::Empty | TryRecvError::Disconnected) => return deliveries,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_interval_one_delivers_every_output() {
|
||||
let (mut sender, mut receiver) = channel(1);
|
||||
sender.send(output(&[1]));
|
||||
sender.send(output(&[2, 3]));
|
||||
sender.send(finished(&[4]));
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
[
|
||||
[
|
||||
1,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
2,
|
||||
3,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
4,
|
||||
],
|
||||
],
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&deliveries(&mut receiver));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_interval_batches_outputs_after_the_first() {
|
||||
let (mut sender, mut receiver) = channel(3);
|
||||
// The first output is delivered immediately; later ones are held back
|
||||
// until they carry at least 3 tokens, counted across multi-token
|
||||
// outputs. The terminal output flushes whatever is held back.
|
||||
for token_ids in [&[1][..], &[2], &[3], &[4, 5], &[6], &[7, 8, 9, 10], &[11]] {
|
||||
sender.send(output(token_ids));
|
||||
}
|
||||
sender.send(finished(&[12]));
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
[
|
||||
[
|
||||
1,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
2,
|
||||
],
|
||||
[
|
||||
3,
|
||||
],
|
||||
[
|
||||
4,
|
||||
5,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
6,
|
||||
],
|
||||
[
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
11,
|
||||
],
|
||||
[
|
||||
12,
|
||||
],
|
||||
],
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&deliveries(&mut receiver));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_output_without_held_back_outputs_is_delivered_alone() {
|
||||
let (mut sender, mut receiver) = channel(2);
|
||||
sender.send(output(&[1]));
|
||||
sender.send(output(&[2, 3]));
|
||||
sender.send(finished(&[]));
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
[
|
||||
[
|
||||
1,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
2,
|
||||
3,
|
||||
],
|
||||
],
|
||||
[
|
||||
[],
|
||||
],
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&deliveries(&mut receiver));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_delivers_held_back_outputs() {
|
||||
let (mut sender, mut receiver) = channel(4);
|
||||
sender.send(output(&[1]));
|
||||
sender.send(output(&[2]));
|
||||
sender.flush();
|
||||
sender.flush();
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
[
|
||||
[
|
||||
1,
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
2,
|
||||
],
|
||||
],
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&deliveries(&mut receiver));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_follows_held_back_outputs() {
|
||||
let (mut sender, mut receiver) = channel(4);
|
||||
sender.send(output(&[1]));
|
||||
sender.send(output(&[2]));
|
||||
sender.send_error(Error::EngineCoreDead);
|
||||
|
||||
let mut items = Vec::new();
|
||||
while let Ok(item) = receiver.try_recv() {
|
||||
items.push(item.map(|delivery| token_ids(delivery).len()).ok());
|
||||
}
|
||||
// One delivery for the first output, one for the held-back output,
|
||||
// then the error.
|
||||
assert_eq!(items, [Some(1), Some(1), None]);
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,15 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tokio::sync::oneshot;
|
||||
use tracing::trace;
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::client::output_channel::output_channel;
|
||||
pub(crate) use crate::client::output_channel::{OutputReceiver, OutputSender};
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
|
||||
@@ -15,8 +18,6 @@ use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
|
||||
pub type OutputReceiver = mpsc::UnboundedReceiver<Result<EngineCoreStreamOutput>>;
|
||||
pub type UtilitySender = oneshot::Sender<Result<UtilityOutput>>;
|
||||
pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
|
||||
|
||||
@@ -137,18 +138,22 @@ impl RequestRegistry {
|
||||
/// When `data_parallel_rank` is provided, the request is routed directly to
|
||||
/// the engine at that rank index, bypassing load balancing. Otherwise
|
||||
/// the engine with the lowest routing score is chosen.
|
||||
///
|
||||
/// `stream_interval` sets how many new tokens the channel batches into one
|
||||
/// delivery after the first output; see [`OutputSender::send`].
|
||||
pub fn register(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
lora_name: Option<String>,
|
||||
data_parallel_rank: Option<u32>,
|
||||
stream_interval: NonZeroU32,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
if self.requests.contains_key(&request_id) {
|
||||
return Err(Error::DuplicateRequestId { request_id });
|
||||
}
|
||||
|
||||
let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let (tx, rx) = output_channel(stream_interval);
|
||||
let lora = lora_name.map(|adapter_name| LoraRequestState {
|
||||
adapter_name,
|
||||
phase: LoraPhase::Waiting,
|
||||
@@ -213,17 +218,30 @@ impl RequestRegistry {
|
||||
by_engine
|
||||
}
|
||||
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
|
||||
self.apply_lora_events(output);
|
||||
/// Send one output to its request stream. If it indicates the request is
|
||||
/// finished, the request will be removed from the registry. Returns the
|
||||
/// output back if its request is no longer tracked.
|
||||
///
|
||||
/// The send happens under the registry lock: the sender is stateful (see
|
||||
/// [`OutputSender::send`]) and owned by the registry entry, and sending on
|
||||
/// its unbounded channel never blocks.
|
||||
pub fn send_output(
|
||||
&mut self,
|
||||
output: EngineCoreStreamOutput,
|
||||
) -> Option<EngineCoreStreamOutput> {
|
||||
self.apply_lora_events(&output.output);
|
||||
if output.finished() {
|
||||
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
|
||||
match self.remove(output.request_id.as_str()) {
|
||||
Some((mut sender, _)) => sender.send(output),
|
||||
None => return Some(output),
|
||||
}
|
||||
} else {
|
||||
self.requests
|
||||
.get(output.request_id.as_str())
|
||||
.map(|tracked| tracked.sender.clone())
|
||||
match self.requests.get_mut(output.request_id.as_str()) {
|
||||
Some(tracked) => tracked.sender.send(output),
|
||||
None => return Some(output),
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Advance the request's LoRA scheduling phase from the engine-core events
|
||||
@@ -267,17 +285,8 @@ impl RequestRegistry {
|
||||
(running, waiting)
|
||||
}
|
||||
|
||||
/// Obtain stream senders for a whole engine output batch under one
|
||||
/// registry lock. Finished outputs are removed before returning.
|
||||
pub fn senders_for_outputs<'a>(
|
||||
&mut self,
|
||||
outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
|
||||
) -> Vec<Option<OutputSender>> {
|
||||
outputs.into_iter().map(|output| self.sender_for_output(output)).collect()
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
/// their stream senders.
|
||||
/// their stream senders after delivering any held-back outputs.
|
||||
pub fn finish_many<'a>(
|
||||
&mut self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
@@ -285,6 +294,10 @@ impl RequestRegistry {
|
||||
request_ids
|
||||
.into_iter()
|
||||
.filter_map(|request_id| self.remove(request_id.as_str()).map(|tracked| tracked.0))
|
||||
.map(|mut sender| {
|
||||
sender.flush();
|
||||
sender
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -325,7 +338,7 @@ impl RequestRegistry {
|
||||
) -> Vec<String> {
|
||||
let mut aborted = Vec::new();
|
||||
for request_id in request_ids {
|
||||
let Some((sender, engine_id)) = self.remove(request_id) else {
|
||||
let Some((mut sender, engine_id)) = self.remove(request_id) else {
|
||||
continue;
|
||||
};
|
||||
let output = EngineCoreStreamOutput {
|
||||
@@ -337,7 +350,7 @@ impl RequestRegistry {
|
||||
..EngineCoreOutput::default()
|
||||
},
|
||||
};
|
||||
let _ = sender.send(Ok(output));
|
||||
sender.send(output);
|
||||
aborted.push(request_id.clone());
|
||||
}
|
||||
aborted
|
||||
@@ -469,11 +482,13 @@ impl UtilityRegistry {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::client::state::{
|
||||
EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
|
||||
EngineLoadSnapshot, EngineRoutingState, OutputReceiver, RequestRegistry, UtilityRegistry,
|
||||
};
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::mock_engine::default_ready_response;
|
||||
use crate::protocol::output::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
@@ -508,6 +523,14 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_output(output: EngineCoreOutput) -> EngineCoreStreamOutput {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
output,
|
||||
}
|
||||
}
|
||||
|
||||
fn adapter_names(values: &[&str]) -> BTreeSet<String> {
|
||||
values.iter().map(|name| (*name).to_string()).collect()
|
||||
}
|
||||
@@ -515,8 +538,9 @@ mod tests {
|
||||
#[test]
|
||||
fn registry_rejects_duplicate_request_ids() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let error = registry.register("req-1".to_string(), None, None).unwrap_err();
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
let error =
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
|
||||
@@ -526,25 +550,99 @@ mod tests {
|
||||
#[test]
|
||||
fn registry_removes_finished_request_on_output() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
let sender = registry.sender_for_output(&EngineCoreOutput {
|
||||
let unrouted = registry.send_output(stream_output(EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
..Default::default()
|
||||
});
|
||||
}));
|
||||
|
||||
assert!(sender.is_some());
|
||||
assert!(unrouted.is_none());
|
||||
assert!(!registry.contains("req-1"));
|
||||
}
|
||||
|
||||
/// Drain the outputs that are ready on `receiver` without waiting,
|
||||
/// rendering each as its token ids and finish reason.
|
||||
fn ready_outputs(
|
||||
receiver: &mut OutputReceiver,
|
||||
) -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
|
||||
let mut outputs = Vec::new();
|
||||
while let Ok(delivery) = receiver.try_recv() {
|
||||
for output in delivery.unwrap() {
|
||||
outputs.push((output.output.new_token_ids, output.output.finish_reason));
|
||||
}
|
||||
}
|
||||
outputs
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_abort_delivers_held_back_outputs_before_terminal_output() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
let (_, mut receiver) = registry
|
||||
.register("req-1".to_string(), None, None, NonZeroU32::new(4).unwrap())
|
||||
.unwrap();
|
||||
|
||||
for token_id in 1..=3 {
|
||||
let unrouted = registry.send_output(stream_output(EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![token_id],
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(unrouted.is_none());
|
||||
}
|
||||
// Only the first output is delivered; the next two are held back
|
||||
// below the stream interval.
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
[
|
||||
1,
|
||||
],
|
||||
None,
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&ready_outputs(&mut receiver));
|
||||
|
||||
registry.abort_many(&["req-1".to_string()], 0.0);
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
[
|
||||
2,
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
[
|
||||
3,
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
[],
|
||||
Some(
|
||||
Abort,
|
||||
),
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&ready_outputs(&mut receiver));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_tracks_lora_phases_from_engine_events() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.register(
|
||||
"req-lora".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
registry.register("req-plain".to_string(), None, None).unwrap();
|
||||
registry.register("req-plain".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
// Registered but not yet scheduled: counted as waiting. The non-LoRA
|
||||
// request never shows up.
|
||||
@@ -554,7 +652,7 @@ mod tests {
|
||||
);
|
||||
|
||||
// Queued then scheduled in one output: running.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
registry.send_output(stream_output(output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled],
|
||||
None,
|
||||
@@ -565,7 +663,7 @@ mod tests {
|
||||
);
|
||||
|
||||
// Preempted: back to waiting.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
registry.send_output(stream_output(output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Preempted],
|
||||
None,
|
||||
@@ -576,7 +674,7 @@ mod tests {
|
||||
);
|
||||
|
||||
// Finished: dropped from tracking entirely.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
registry.send_output(stream_output(output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Scheduled],
|
||||
Some(EngineCoreFinishReason::Stop),
|
||||
@@ -591,18 +689,33 @@ mod tests {
|
||||
fn registry_unions_lora_adapters_across_requests() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-a1".to_string(), Some("adapter-a".to_string()), None)
|
||||
.register(
|
||||
"req-a1".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
registry
|
||||
.register("req-a2".to_string(), Some("adapter-a".to_string()), None)
|
||||
.register(
|
||||
"req-a2".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
registry
|
||||
.register("req-b".to_string(), Some("adapter-b".to_string()), None)
|
||||
.register(
|
||||
"req-b".to_string(),
|
||||
Some("adapter-b".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// One of adapter-a's requests starts running while the other waits:
|
||||
// the adapter appears in both sets.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
registry.send_output(stream_output(output_with_events(
|
||||
"req-a1",
|
||||
&[EngineCoreEventType::Scheduled],
|
||||
None,
|
||||
@@ -620,7 +733,7 @@ mod tests {
|
||||
fn registry_counts_only_active_lora_requests() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
|
||||
registry.register("req-plain".to_string(), None, None).unwrap();
|
||||
registry.register("req-plain".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(registry.active_lora_requests(), 0);
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
@@ -632,6 +745,7 @@ mod tests {
|
||||
"req-lora-a".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
registry
|
||||
@@ -639,6 +753,7 @@ mod tests {
|
||||
"req-lora-b".to_string(),
|
||||
Some("adapter-b".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(registry.active_lora_requests(), 2);
|
||||
@@ -661,7 +776,12 @@ mod tests {
|
||||
fn registry_clears_lora_count_on_close() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.register(
|
||||
"req-lora".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(registry.active_lora_requests(), 1);
|
||||
@@ -677,7 +797,12 @@ mod tests {
|
||||
fn registry_drops_lora_tracking_on_abort() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.register(
|
||||
"req-lora".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
NonZeroU32::MIN,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(registry.finish_many(&["req-lora".to_string()]));
|
||||
@@ -691,8 +816,8 @@ mod tests {
|
||||
#[test]
|
||||
fn registry_closes_all_requests_on_failure() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
registry.register("req-2".to_string(), None, None).unwrap();
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
registry.register("req-2".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
@@ -708,9 +833,12 @@ mod tests {
|
||||
connected_engine(engine_0.clone()),
|
||||
connected_engine(engine_1.clone()),
|
||||
]);
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
|
||||
let (chosen_0, _) =
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
let (chosen_1, _) =
|
||||
registry.register("req-2".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
let (chosen_0_again, _) =
|
||||
registry.register("req-3".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
@@ -737,9 +865,12 @@ mod tests {
|
||||
connected_engine(engine_1.clone()),
|
||||
]);
|
||||
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
|
||||
let (chosen_0, _) =
|
||||
registry.register("req-1".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
let (chosen_1, _) =
|
||||
registry.register("req-2".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
let (chosen_0_again, _) =
|
||||
registry.register("req-3".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
@@ -809,7 +940,8 @@ mod tests {
|
||||
}
|
||||
));
|
||||
|
||||
let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-stats".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
@@ -824,7 +956,9 @@ mod tests {
|
||||
for (rank, running) in counts.into_iter().enumerate() {
|
||||
for index in 0..running {
|
||||
let request_id = format!("old-{rank}-{index}");
|
||||
registry.register(request_id.clone(), None, Some(rank as u32)).unwrap();
|
||||
registry
|
||||
.register(request_id.clone(), None, Some(rank as u32), NonZeroU32::MIN)
|
||||
.unwrap();
|
||||
old_requests.push(request_id);
|
||||
}
|
||||
registry.apply_scheduler_counts(
|
||||
@@ -839,7 +973,8 @@ mod tests {
|
||||
|
||||
let mut distribution = [0; 2];
|
||||
for index in 0..14 {
|
||||
let (engine, _) = registry.register(format!("new-{index}"), None, None).unwrap();
|
||||
let (engine, _) =
|
||||
registry.register(format!("new-{index}"), None, None, NonZeroU32::MIN).unwrap();
|
||||
distribution[engine.engine_index().unwrap() as usize] += 1;
|
||||
}
|
||||
distributions.push(distribution);
|
||||
@@ -865,7 +1000,8 @@ mod tests {
|
||||
);
|
||||
}
|
||||
let request_id = format!("req-{index}");
|
||||
let (engine, _) = registry.register(request_id.clone(), None, None).unwrap();
|
||||
let (engine, _) =
|
||||
registry.register(request_id.clone(), None, None, NonZeroU32::MIN).unwrap();
|
||||
chosen.push(engine.engine_index().unwrap());
|
||||
drop(registry.finish_many(&[request_id]));
|
||||
}
|
||||
@@ -890,12 +1026,15 @@ mod tests {
|
||||
},
|
||||
);
|
||||
for index in 0..4 {
|
||||
registry.register(format!("pinned-{index}"), None, Some(0)).unwrap();
|
||||
registry
|
||||
.register(format!("pinned-{index}"), None, Some(0), NonZeroU32::MIN)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Even unchanged counts replace the optimistic estimate for that engine.
|
||||
registry.apply_scheduler_counts(0, snapshot);
|
||||
let (after_refresh, _) = registry.register("after-refresh".into(), None, None).unwrap();
|
||||
let (after_refresh, _) =
|
||||
registry.register("after-refresh".into(), None, None, NonZeroU32::MIN).unwrap();
|
||||
// An older, empty snapshot must not erase the five admitted requests.
|
||||
for rank in [0, 1] {
|
||||
registry.apply_scheduler_counts(
|
||||
@@ -906,7 +1045,8 @@ mod tests {
|
||||
},
|
||||
);
|
||||
}
|
||||
let (after_empty, _) = registry.register("after-empty".into(), None, None).unwrap();
|
||||
let (after_empty, _) =
|
||||
registry.register("after-empty".into(), None, None, NonZeroU32::MIN).unwrap();
|
||||
expect_test::expect!["(Some(0), Some(1))"].assert_eq(&format!(
|
||||
"{:?}",
|
||||
(after_refresh.engine_index(), after_empty.engine_index())
|
||||
@@ -927,10 +1067,11 @@ mod tests {
|
||||
},
|
||||
);
|
||||
}
|
||||
registry.register("pinned".into(), None, Some(9)).unwrap();
|
||||
registry.register("pinned".into(), None, Some(9), NonZeroU32::MIN).unwrap();
|
||||
let mut chosen = Vec::new();
|
||||
for index in 0..3 {
|
||||
let (engine, _) = registry.register(format!("auto-{index}"), None, None).unwrap();
|
||||
let (engine, _) =
|
||||
registry.register(format!("auto-{index}"), None, None, NonZeroU32::MIN).unwrap();
|
||||
chosen.push(engine.engine_index().unwrap());
|
||||
}
|
||||
expect_test::expect!["[2, 5, 2]"].assert_eq(&format!("{chosen:?}"));
|
||||
@@ -948,15 +1089,18 @@ mod tests {
|
||||
]);
|
||||
|
||||
// Explicitly target rank 2 (third engine).
|
||||
let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-1".to_string(), None, Some(2), NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_2);
|
||||
|
||||
// Explicitly target rank 0 (first engine).
|
||||
let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-2".to_string(), None, Some(0), NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
// Explicitly target rank 1.
|
||||
let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-3".to_string(), None, Some(1), NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
@@ -970,11 +1114,12 @@ mod tests {
|
||||
]);
|
||||
|
||||
// Load-balance: first two go to engine_0 and engine_1.
|
||||
registry.register("req-lb-0".to_string(), None, None).unwrap();
|
||||
registry.register("req-lb-0".to_string(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
|
||||
// But with dp_rank=0, it should still go to engine_0.
|
||||
let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-dp".to_string(), None, Some(0), NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
}
|
||||
|
||||
@@ -985,7 +1130,9 @@ mod tests {
|
||||
connected_engine(EngineId::from_engine_index(1)),
|
||||
]);
|
||||
|
||||
let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err();
|
||||
let error = registry
|
||||
.register("req-1".to_string(), None, Some(2), NonZeroU32::MIN)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
@@ -1000,10 +1147,13 @@ mod tests {
|
||||
let engine_3 = EngineId::from_engine_index(3);
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(engine_3.clone())]);
|
||||
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(3)).unwrap();
|
||||
let (chosen, _) =
|
||||
registry.register("req-ok".to_string(), None, Some(3), NonZeroU32::MIN).unwrap();
|
||||
assert_eq!(chosen, engine_3);
|
||||
|
||||
let error = registry.register("req-bad".to_string(), None, Some(0)).unwrap_err();
|
||||
let error = registry
|
||||
.register("req-bad".to_string(), None, Some(0), NonZeroU32::MIN)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
|
||||
@@ -5,8 +5,9 @@ use std::ops::Deref;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::Stream;
|
||||
use futures::stream::FusedStream;
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use itertools::Either;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, warn};
|
||||
@@ -40,6 +41,68 @@ impl Deref for EngineCoreStreamOutput {
|
||||
}
|
||||
}
|
||||
|
||||
/// Consecutive outputs of one request, delivered to its stream together and in
|
||||
/// order. Outputs are grouped by the request's `stream_interval` (see
|
||||
/// [`EngineCoreClient::call`]) but not merged; consumers merge them after
|
||||
/// observing each raw output.
|
||||
///
|
||||
/// [`EngineCoreClient::call`]: crate::EngineCoreClient::call
|
||||
#[derive(Debug)]
|
||||
#[allow(
|
||||
clippy::large_enum_variant,
|
||||
reason = "single outputs stay inline, avoiding an allocation per engine step"
|
||||
)]
|
||||
pub enum EngineCoreStreamDelivery {
|
||||
/// A single output, sent without being held back.
|
||||
One(EngineCoreStreamOutput),
|
||||
/// Outputs held back by the stream interval, in order. Never empty.
|
||||
Many(Vec<EngineCoreStreamOutput>),
|
||||
}
|
||||
|
||||
impl EngineCoreStreamDelivery {
|
||||
/// Return the first output.
|
||||
pub fn first(&self) -> &EngineCoreStreamOutput {
|
||||
match self {
|
||||
Self::One(output) => output,
|
||||
Self::Many(outputs) => outputs.first().expect("deliveries are never empty"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the last output, the only one that can be terminal.
|
||||
pub fn last(&self) -> &EngineCoreStreamOutput {
|
||||
match self {
|
||||
Self::One(output) => output,
|
||||
Self::Many(outputs) => outputs.last().expect("deliveries are never empty"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterate over the outputs in order.
|
||||
pub fn iter(
|
||||
&self,
|
||||
) -> Either<
|
||||
std::iter::Once<&EngineCoreStreamOutput>,
|
||||
std::slice::Iter<'_, EngineCoreStreamOutput>,
|
||||
> {
|
||||
match self {
|
||||
Self::One(output) => Either::Left(std::iter::once(output)),
|
||||
Self::Many(outputs) => Either::Right(outputs.iter()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoIterator for EngineCoreStreamDelivery {
|
||||
type Item = EngineCoreStreamOutput;
|
||||
type IntoIter =
|
||||
Either<std::iter::Once<EngineCoreStreamOutput>, std::vec::IntoIter<EngineCoreStreamOutput>>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
match self {
|
||||
Self::One(output) => Either::Left(std::iter::once(output)),
|
||||
Self::Many(outputs) => Either::Right(outputs.into_iter()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of raw engine-core outputs for one request.
|
||||
///
|
||||
/// The stream yields only [`EngineCoreStreamOutput`] values whose embedded
|
||||
@@ -79,10 +142,21 @@ impl EngineCoreOutputStream {
|
||||
pub fn engine_index(&self) -> u32 {
|
||||
self.engine_index
|
||||
}
|
||||
|
||||
/// Flatten the stream into one item per raw output, for consumers that do
|
||||
/// not care how outputs are grouped into deliveries.
|
||||
pub fn into_outputs(self) -> impl Stream<Item = Result<EngineCoreStreamOutput>> + Send + Unpin {
|
||||
self.flat_map(|item| {
|
||||
futures::stream::iter(match item {
|
||||
Ok(delivery) => Either::Left(delivery.into_iter().map(Ok)),
|
||||
Err(error) => Either::Right(std::iter::once(Err(error))),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for EngineCoreOutputStream {
|
||||
type Item = Result<EngineCoreStreamOutput>;
|
||||
type Item = Result<EngineCoreStreamDelivery>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
if self.is_terminated() {
|
||||
@@ -93,10 +167,11 @@ impl Stream for EngineCoreOutputStream {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(Some(item)) => {
|
||||
match &item {
|
||||
Ok(output) => {
|
||||
Ok(delivery) => {
|
||||
// If the output indicates the request is finished, mark the stream as
|
||||
// terminated with cleanly-finished state and expect no more outputs to
|
||||
// come.
|
||||
// come. Only the last output of a delivery can be terminal.
|
||||
let output = delivery.last();
|
||||
if output.finished() {
|
||||
if output.finish_reason == Some(EngineCoreFinishReason::Error) {
|
||||
error!(
|
||||
|
||||
@@ -14,7 +14,7 @@ mod transport;
|
||||
|
||||
pub use client::{
|
||||
AbortCause, CoordinatorMode, EngineCoreClient, EngineCoreClientConfig, EngineCoreOutputStream,
|
||||
EngineCoreStreamOutput, TransportMode,
|
||||
EngineCoreStreamDelivery, EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
pub use error::{Error, Result};
|
||||
pub use transport::{ENGINE_CORE_DEAD_SENTINEL, EngineId};
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
@@ -161,6 +162,11 @@ pub struct EngineCoreSamplingParams {
|
||||
/// Number of prompt tokens to skip from returned routed-expert data.
|
||||
/// A value of zero returns routing data for the entire prompt.
|
||||
pub routed_experts_prompt_start: u32,
|
||||
/// Number of newly generated tokens to batch into each streamed output.
|
||||
/// The first and final outputs are always emitted immediately. Engine-core
|
||||
/// ignores it; [`EngineCoreClient::call`](crate::EngineCoreClient::call)
|
||||
/// batches output deliveries by it.
|
||||
pub stream_interval: Option<NonZeroU32>,
|
||||
}
|
||||
|
||||
impl EngineCoreSamplingParams {
|
||||
@@ -195,6 +201,7 @@ impl EngineCoreSamplingParams {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -755,8 +755,8 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap().into_outputs();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap().into_outputs();
|
||||
|
||||
let final_1 = timeout(Duration::from_secs(1), stream_1.next())
|
||||
.await
|
||||
@@ -778,7 +778,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let mut stream_3 = client.call(sample_request_with_id("req-3")).await.unwrap();
|
||||
let mut stream_3 = client.call(sample_request_with_id("req-3")).await.unwrap().into_outputs();
|
||||
let final_3 = timeout(Duration::from_secs(1), stream_3.next())
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -933,7 +933,7 @@ async fn coordinator_accepts_stats_only_outputs() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream = client.call(sample_request_with_id("req-stats")).await.unwrap();
|
||||
let mut stream = client.call(sample_request_with_id("req-stats")).await.unwrap().into_outputs();
|
||||
let final_output =
|
||||
timeout(Duration::from_secs(1), stream.next()).await.unwrap().unwrap().unwrap();
|
||||
assert_eq!(final_output.request_id, "req-stats");
|
||||
@@ -1026,8 +1026,8 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
|
||||
assert!(client.ready_responses()[0].max_model_len > 0);
|
||||
assert_eq!(client.vllm_version(), "test-vllm-version");
|
||||
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap().into_outputs();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap().into_outputs();
|
||||
|
||||
let error_2 = timeout(Duration::from_secs(1), stream_2.next())
|
||||
.await
|
||||
@@ -1103,7 +1103,7 @@ async fn duplicate_request_ids_are_rejected_without_sending_a_second_add() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap();
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
let error = match client.call(sample_request()).await {
|
||||
Ok(_) => panic!("expected duplicate request error"),
|
||||
Err(error) => error,
|
||||
@@ -1170,7 +1170,7 @@ async fn finished_requests_without_final_output_is_treated_as_unexpected_close()
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap();
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
let error = timeout(Duration::from_secs(1), stream.next())
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -1233,7 +1233,7 @@ async fn dropping_a_live_stream_triggers_abort() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap();
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
let first = timeout(Duration::from_secs(1), stream.next()).await.unwrap().unwrap().unwrap();
|
||||
assert_eq!(first.new_token_ids, vec![99]);
|
||||
drop(stream);
|
||||
@@ -1316,7 +1316,7 @@ async fn dropping_multiple_live_streams_aborts_all_in_a_burst() {
|
||||
// emits outputs, then drain the first token from each stream.
|
||||
let mut streams = Vec::new();
|
||||
for id in request_ids {
|
||||
streams.push(client.call(sample_request_with_id(id)).await.unwrap());
|
||||
streams.push(client.call(sample_request_with_id(id)).await.unwrap().into_outputs());
|
||||
}
|
||||
for stream in streams.iter_mut() {
|
||||
let first = timeout(Duration::from_secs(1), stream.next()).await.unwrap().unwrap().unwrap();
|
||||
@@ -1363,8 +1363,8 @@ async fn dispatcher_failure_propagates_to_streams_and_future_calls() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap().into_outputs();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap().into_outputs();
|
||||
|
||||
let error_1 = timeout(Duration::from_secs(1), stream_1.next())
|
||||
.await
|
||||
@@ -1862,7 +1862,7 @@ async fn client_decodes_multipart_logprob_outputs() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let stream = client.call(sample_request()).await.unwrap();
|
||||
let stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
let outputs = stream.collect::<Vec<_>>().await;
|
||||
assert_eq!(outputs.len(), 1);
|
||||
|
||||
@@ -1950,7 +1950,7 @@ async fn client_sends_large_multimodal_tensor_as_aux_frame() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let outputs = client.call(request).await.unwrap().collect::<Vec<_>>().await;
|
||||
let outputs = client.call(request).await.unwrap().into_outputs().collect::<Vec<_>>().await;
|
||||
assert_eq!(outputs.len(), 1);
|
||||
assert!(outputs[0].is_ok());
|
||||
|
||||
@@ -2079,8 +2079,8 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
|
||||
assert_eq!(client.ready_responses().len(), 2);
|
||||
assert_eq!(client.engine_identities()[0], b"engine-0");
|
||||
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap().into_outputs();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap().into_outputs();
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(1), engine_0_seen_rx.recv()).await.unwrap().unwrap(),
|
||||
"req-1"
|
||||
@@ -2100,7 +2100,7 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
|
||||
assert_eq!(final_1.new_token_ids, vec![10]);
|
||||
assert_eq!(final_1.finish_reason, Some(EngineCoreFinishReason::Length));
|
||||
|
||||
let mut stream_3 = client.call(sample_request_with_id("req-3")).await.unwrap();
|
||||
let mut stream_3 = client.call(sample_request_with_id("req-3")).await.unwrap().into_outputs();
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(1), engine_0_seen_rx.recv()).await.unwrap().unwrap(),
|
||||
"req-3"
|
||||
@@ -2271,8 +2271,8 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
|
||||
|
||||
assert!(client.is_sleeping().await.unwrap());
|
||||
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
|
||||
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap().into_outputs();
|
||||
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap().into_outputs();
|
||||
|
||||
client
|
||||
.abort(&[
|
||||
@@ -2803,6 +2803,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -3266,7 +3267,7 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap();
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
|
||||
let wakeup = timeout(
|
||||
Duration::from_secs(1),
|
||||
@@ -3352,7 +3353,7 @@ async fn bootstrapped_external_coordinator_running_state_suppresses_wakeup() {
|
||||
send_external_coordinator_publish(&mut stats_socket, &(Value::Nil, 5_u32, true)).await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap();
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
|
||||
Reference in New Issue
Block a user