[Rust Frontend] Return sampling masks over gRPC (#56777)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
d5f0a6e829
commit
95d723700b
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
|
||||
|
||||
@@ -50,6 +51,8 @@ pub struct Finished {
|
||||
/// Connector-specific encoder cache transfer parameters for disaggregated
|
||||
/// serving.
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
/// Sampling support sets aligned with all generated token positions.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
}
|
||||
|
||||
/// Sample metadata emitted by one engine output update.
|
||||
@@ -120,6 +123,7 @@ pub async fn decoded_text_event_stream(
|
||||
let mut sampled_token_ids = Vec::new();
|
||||
let mut output_token_count: usize = 0;
|
||||
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
|
||||
let mut sampling_mask: Option<SamplingMask> = None;
|
||||
|
||||
while let Some(next) = raw_stream.next().await {
|
||||
let output = next?;
|
||||
@@ -213,6 +217,7 @@ pub async fn decoded_text_event_stream(
|
||||
|
||||
let mut new_token_ids = output.token_ids;
|
||||
let mut new_logprobs = output.logprobs;
|
||||
let mut new_sampling_mask = output.sampling_mask;
|
||||
|
||||
// Trim tokens and logprobs if we matched stop string.
|
||||
if let Some(num_tokens) = truncate_tokens_to {
|
||||
@@ -220,9 +225,15 @@ pub async fn decoded_text_event_stream(
|
||||
if let Some(logprobs) = &mut new_logprobs {
|
||||
logprobs.positions.truncate(num_tokens);
|
||||
}
|
||||
if let Some(mask) = &mut new_sampling_mask {
|
||||
mask.rows.truncate(num_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
output_token_count += new_token_ids.len();
|
||||
if let Some(mut mask) = new_sampling_mask {
|
||||
sampling_mask.get_or_insert_default().rows.append(&mut mask.rows);
|
||||
}
|
||||
|
||||
let decoded_logprobs = new_logprobs
|
||||
.as_ref()
|
||||
@@ -246,6 +257,16 @@ pub async fn decoded_text_event_stream(
|
||||
}
|
||||
|
||||
if let Some(reason) = finish_reason {
|
||||
if let Some(mask) = sampling_mask.as_ref()
|
||||
&& mask.rows.len() != output_token_count
|
||||
{
|
||||
return Err(vllm_llm::Error::SamplingMaskTokenCountMismatch {
|
||||
request_id,
|
||||
token_count: output_token_count,
|
||||
row_count: mask.rows.len(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
// Flush any remaining buffered text.
|
||||
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = full_decoded.text.len();
|
||||
@@ -293,6 +314,7 @@ pub async fn decoded_text_event_stream(
|
||||
finish_reason: reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
sampling_mask,
|
||||
})),
|
||||
})
|
||||
.await;
|
||||
@@ -354,6 +376,7 @@ mod tests {
|
||||
|
||||
use futures::{Stream, StreamExt as _, stream};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
use vllm_llm::GenerateOutput;
|
||||
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
|
||||
|
||||
@@ -384,6 +407,73 @@ mod tests {
|
||||
s.bytes().map(u32::from).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sampling_masks_survive_text_decode_and_collection() {
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let mut first = GenerateOutput::for_test(Some(prompt), vec![b'a' as u32], None);
|
||||
first.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![1, b'a' as u32]],
|
||||
});
|
||||
let mut terminal =
|
||||
GenerateOutput::for_test(None, vec![b'b' as u32], Some(FinishReason::Length));
|
||||
terminal.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![2, b'b' as u32]],
|
||||
});
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
|
||||
let collected = decoded_text_event_stream(
|
||||
"test-mask".into(),
|
||||
tokenizer,
|
||||
stream::iter([Ok(first), Ok(terminal)]),
|
||||
TextDecodeOptions::default(),
|
||||
false,
|
||||
)
|
||||
.collect_output()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
collected.sampling_mask,
|
||||
Some(SamplingMask {
|
||||
rows: vec![vec![1, b'a' as u32], vec![2, b'b' as u32]],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sampling_mask_mismatch_reuses_llm_error() {
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let mut terminal = GenerateOutput::for_test(
|
||||
Some(prompt),
|
||||
vec![b'a' as u32, b'b' as u32],
|
||||
Some(FinishReason::Length),
|
||||
);
|
||||
terminal.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![1, b'a' as u32]],
|
||||
});
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
|
||||
let error = decoded_text_event_stream(
|
||||
"test-mask-mismatch".into(),
|
||||
tokenizer,
|
||||
stream::iter([Ok(terminal)]),
|
||||
TextDecodeOptions::default(),
|
||||
false,
|
||||
)
|
||||
.collect_output()
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::Error::Llm(vllm_llm::Error::SamplingMaskTokenCountMismatch {
|
||||
token_count: 2,
|
||||
row_count: 1,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions {
|
||||
TextDecodeOptions {
|
||||
stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()),
|
||||
@@ -472,6 +562,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
@@ -503,6 +594,7 @@ mod tests {
|
||||
))),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
@@ -576,6 +668,60 @@ mod tests {
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_mid_chunk_keeps_token_logprob_mask_alignment() {
|
||||
let token_ids = ascii_tokens("abcxyz");
|
||||
let mut raw = GenerateOutput::for_test(
|
||||
Some(Arc::from([])),
|
||||
token_ids.clone(),
|
||||
Some(FinishReason::Length),
|
||||
);
|
||||
raw.logprobs = Some(Logprobs {
|
||||
positions: token_ids
|
||||
.iter()
|
||||
.map(|&token_id| PositionLogprobs {
|
||||
entries: vec![TokenLogprob {
|
||||
token_id,
|
||||
logprob: -0.25,
|
||||
rank: 1,
|
||||
}],
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
raw.sampling_mask = Some(SamplingMask {
|
||||
rows: token_ids.iter().map(|&token_id| vec![token_id, token_id + 100]).collect(),
|
||||
});
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
|
||||
let output = decoded_text_event_stream(
|
||||
"test-mid-chunk-stop".into(),
|
||||
tokenizer,
|
||||
stream::iter([Ok(raw)]),
|
||||
opts(&["cx"], 0),
|
||||
false,
|
||||
)
|
||||
.collect_output()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(output.text, "ab");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
assert_eq!(output.token_ids, ascii_tokens("abcx"));
|
||||
assert_eq!(output.logprobs.as_ref().unwrap().positions.len(), 4);
|
||||
assert_eq!(
|
||||
output.sampling_mask,
|
||||
Some(SamplingMask {
|
||||
rows: vec![
|
||||
vec![b'a' as u32, b'a' as u32 + 100],
|
||||
vec![b'b' as u32, b'b' as u32 + 100],
|
||||
vec![b'c' as u32, b'c' as u32 + 100],
|
||||
vec![b'x' as u32, b'x' as u32 + 100],
|
||||
],
|
||||
})
|
||||
);
|
||||
assert_eq!(output.usage.output_token_count, 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_at_end() {
|
||||
let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await;
|
||||
|
||||
@@ -17,6 +17,7 @@ mod logprobs;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt as _, pin_mut};
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
|
||||
use crate::{Error, FinishReason, Result, TextOutputStream};
|
||||
|
||||
@@ -35,6 +36,8 @@ pub struct CollectedTextOutput {
|
||||
/// Connector-specific encoder cache transfer parameters for disaggregated
|
||||
/// serving.
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
/// Sampling support sets aligned with generated token positions.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
}
|
||||
|
||||
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
|
||||
@@ -90,6 +93,7 @@ impl<T: TextOutputStream> T {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
})
|
||||
};
|
||||
|
||||
@@ -99,6 +103,7 @@ impl<T: TextOutputStream> T {
|
||||
collected.usage = finished.usage;
|
||||
collected.kv_transfer_params = finished.kv_transfer_params;
|
||||
collected.ec_transfer_params = finished.ec_transfer_params;
|
||||
collected.sampling_mask = finished.sampling_mask;
|
||||
return Ok(collected);
|
||||
}
|
||||
}
|
||||
@@ -173,6 +178,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
@@ -295,6 +301,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user