[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
fh3
Claude Fable 5.1
parent
c9578f10ab
commit
4a9d6f9572
@@ -231,6 +231,7 @@ async fn generate_chunk_stream(
|
||||
logprobs,
|
||||
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
|
||||
token_ids,
|
||||
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
@@ -331,6 +332,7 @@ fn collect_generate(
|
||||
logprobs,
|
||||
finish_reason: Some(finish_reason),
|
||||
token_ids: collected.token_ids,
|
||||
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
@@ -516,6 +518,7 @@ mod tests {
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_llm::GeneratePromptInfo;
|
||||
|
||||
use super::*;
|
||||
@@ -657,6 +660,18 @@ mod tests {
|
||||
.expect("collect chunks")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_returns_sampling_mask() {
|
||||
let mut output = stream_output(Some(&[11]), vec![33], Some(FinishReason::Length));
|
||||
output.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![33, 44]],
|
||||
});
|
||||
|
||||
let chunks = collect_chunks(vec![output], false, None).await;
|
||||
|
||||
assert_eq!(chunks[0].choices[0].sampling_mask, Some(vec![vec![33, 44]]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_omits_prompt_metadata_by_default() {
|
||||
let chunks = collect_chunks(
|
||||
@@ -792,7 +807,9 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
sampling_mask: Some(SamplingMask {
|
||||
rows: vec![vec![30, 40]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
};
|
||||
|
||||
@@ -813,6 +830,7 @@ mod tests {
|
||||
|
||||
assert!(response.prompt_token_ids.is_none());
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
assert_eq!(response.choices[0].sampling_mask, Some(vec![vec![30, 40]]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -64,6 +64,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
|
||||
@@ -74,6 +75,7 @@ pub(super) struct GenerateResponseStreamChoice {
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateStreamResponse` class.
|
||||
|
||||
Reference in New Issue
Block a user