[Rust Frontend] Return sampling masks over gRPC (#56777)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
This commit is contained in:
Biswa Panda
2026-09-18 08:26:17 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent d5f0a6e829
commit 95d723700b
23 changed files with 570 additions and 27 deletions
+146
View File
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
@@ -50,6 +51,8 @@ pub struct Finished {
/// Connector-specific encoder cache transfer parameters for disaggregated
/// serving.
pub ec_transfer_params: Option<serde_json::Value>,
/// Sampling support sets aligned with all generated token positions.
pub sampling_mask: Option<SamplingMask>,
}
/// Sample metadata emitted by one engine output update.
@@ -120,6 +123,7 @@ pub async fn decoded_text_event_stream(
let mut sampled_token_ids = Vec::new();
let mut output_token_count: usize = 0;
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
let mut sampling_mask: Option<SamplingMask> = None;
while let Some(next) = raw_stream.next().await {
let output = next?;
@@ -213,6 +217,7 @@ pub async fn decoded_text_event_stream(
let mut new_token_ids = output.token_ids;
let mut new_logprobs = output.logprobs;
let mut new_sampling_mask = output.sampling_mask;
// Trim tokens and logprobs if we matched stop string.
if let Some(num_tokens) = truncate_tokens_to {
@@ -220,9 +225,15 @@ pub async fn decoded_text_event_stream(
if let Some(logprobs) = &mut new_logprobs {
logprobs.positions.truncate(num_tokens);
}
if let Some(mask) = &mut new_sampling_mask {
mask.rows.truncate(num_tokens);
}
}
output_token_count += new_token_ids.len();
if let Some(mut mask) = new_sampling_mask {
sampling_mask.get_or_insert_default().rows.append(&mut mask.rows);
}
let decoded_logprobs = new_logprobs
.as_ref()
@@ -246,6 +257,16 @@ pub async fn decoded_text_event_stream(
}
if let Some(reason) = finish_reason {
if let Some(mask) = sampling_mask.as_ref()
&& mask.rows.len() != output_token_count
{
return Err(vllm_llm::Error::SamplingMaskTokenCountMismatch {
request_id,
token_count: output_token_count,
row_count: mask.rows.len(),
}
.into());
}
// Flush any remaining buffered text.
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
let text_len = full_decoded.text.len();
@@ -293,6 +314,7 @@ pub async fn decoded_text_event_stream(
finish_reason: reason,
kv_transfer_params,
ec_transfer_params,
sampling_mask,
})),
})
.await;
@@ -354,6 +376,7 @@ mod tests {
use futures::{Stream, StreamExt as _, stream};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
use vllm_llm::GenerateOutput;
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
@@ -384,6 +407,73 @@ mod tests {
s.bytes().map(u32::from).collect()
}
#[tokio::test]
async fn sampling_masks_survive_text_decode_and_collection() {
let prompt: Arc<[u32]> = Arc::from([]);
let mut first = GenerateOutput::for_test(Some(prompt), vec![b'a' as u32], None);
first.sampling_mask = Some(SamplingMask {
rows: vec![vec![1, b'a' as u32]],
});
let mut terminal =
GenerateOutput::for_test(None, vec![b'b' as u32], Some(FinishReason::Length));
terminal.sampling_mask = Some(SamplingMask {
rows: vec![vec![2, b'b' as u32]],
});
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
let collected = decoded_text_event_stream(
"test-mask".into(),
tokenizer,
stream::iter([Ok(first), Ok(terminal)]),
TextDecodeOptions::default(),
false,
)
.collect_output()
.await
.unwrap();
assert_eq!(
collected.sampling_mask,
Some(SamplingMask {
rows: vec![vec![1, b'a' as u32], vec![2, b'b' as u32]],
})
);
}
#[tokio::test]
async fn sampling_mask_mismatch_reuses_llm_error() {
let prompt: Arc<[u32]> = Arc::from([]);
let mut terminal = GenerateOutput::for_test(
Some(prompt),
vec![b'a' as u32, b'b' as u32],
Some(FinishReason::Length),
);
terminal.sampling_mask = Some(SamplingMask {
rows: vec![vec![1, b'a' as u32]],
});
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
let error = decoded_text_event_stream(
"test-mask-mismatch".into(),
tokenizer,
stream::iter([Ok(terminal)]),
TextDecodeOptions::default(),
false,
)
.collect_output()
.await
.unwrap_err();
assert!(matches!(
error,
crate::Error::Llm(vllm_llm::Error::SamplingMaskTokenCountMismatch {
token_count: 2,
row_count: 1,
..
})
));
}
fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions {
TextDecodeOptions {
stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()),
@@ -472,6 +562,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
),
],
@@ -503,6 +594,7 @@ mod tests {
))),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
),
],
@@ -576,6 +668,60 @@ mod tests {
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_mid_chunk_keeps_token_logprob_mask_alignment() {
let token_ids = ascii_tokens("abcxyz");
let mut raw = GenerateOutput::for_test(
Some(Arc::from([])),
token_ids.clone(),
Some(FinishReason::Length),
);
raw.logprobs = Some(Logprobs {
positions: token_ids
.iter()
.map(|&token_id| PositionLogprobs {
entries: vec![TokenLogprob {
token_id,
logprob: -0.25,
rank: 1,
}],
})
.collect(),
});
raw.sampling_mask = Some(SamplingMask {
rows: token_ids.iter().map(|&token_id| vec![token_id, token_id + 100]).collect(),
});
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
let output = decoded_text_event_stream(
"test-mid-chunk-stop".into(),
tokenizer,
stream::iter([Ok(raw)]),
opts(&["cx"], 0),
false,
)
.collect_output()
.await
.unwrap();
assert_eq!(output.text, "ab");
assert!(output.finish_reason.is_stop());
assert_eq!(output.token_ids, ascii_tokens("abcx"));
assert_eq!(output.logprobs.as_ref().unwrap().positions.len(), 4);
assert_eq!(
output.sampling_mask,
Some(SamplingMask {
rows: vec![
vec![b'a' as u32, b'a' as u32 + 100],
vec![b'b' as u32, b'b' as u32 + 100],
vec![b'c' as u32, b'c' as u32 + 100],
vec![b'x' as u32, b'x' as u32 + 100],
],
})
);
assert_eq!(output.usage.output_token_count, 4);
}
#[tokio::test]
async fn stream_stop_string_at_end() {
let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await;
+7
View File
@@ -17,6 +17,7 @@ mod logprobs;
use std::sync::Arc;
use futures::{StreamExt as _, pin_mut};
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use crate::{Error, FinishReason, Result, TextOutputStream};
@@ -35,6 +36,8 @@ pub struct CollectedTextOutput {
/// Connector-specific encoder cache transfer parameters for disaggregated
/// serving.
pub ec_transfer_params: Option<serde_json::Value>,
/// Sampling support sets aligned with generated token positions.
pub sampling_mask: Option<SamplingMask>,
}
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
@@ -90,6 +93,7 @@ impl<T: TextOutputStream> T {
usage: vllm_llm::TokenUsage::default(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
})
};
@@ -99,6 +103,7 @@ impl<T: TextOutputStream> T {
collected.usage = finished.usage;
collected.kv_transfer_params = finished.kv_transfer_params;
collected.ec_transfer_params = finished.ec_transfer_params;
collected.sampling_mask = finished.sampling_mask;
return Ok(collected);
}
}
@@ -173,6 +178,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
})),
}),
]);
@@ -295,6 +301,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
})),
}),
]);