[Rust Frontend] Return sampling masks over gRPC (#56777)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
This commit is contained in:
Biswa Panda
2026-09-18 08:26:17 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent d5f0a6e829
commit 95d723700b
23 changed files with 570 additions and 27 deletions
+1
View File
@@ -19,6 +19,7 @@ itertools.workspace = true
libc.workspace = true
llm-multimodal.workspace = true
openssl.workspace = true
prost.workspace = true
prost-types.workspace = true
rmp-serde.workspace = true
rmpv.workspace = true
+44
View File
@@ -316,6 +316,11 @@ pub fn to_sequence_output(
_ => (vec![], vec![], vec![]),
};
let sampling_mask = finished
.and_then(|finished| finished.sampling_mask.as_ref())
.map(|mask| mask.rows.iter().map(|row| pb::TokenIds { ids: row.clone() }).collect())
.unwrap_or_default();
Ok(pb::SequenceOutput {
index: 0, // TODO: multi-sequence (n > 1) not supported
text: if opts.output_text {
@@ -333,6 +338,7 @@ pub fn to_sequence_output(
ranks: rank_values,
candidate_tokens: candidates,
finish_info,
sampling_mask,
})
}
@@ -524,6 +530,7 @@ impl ResponseOpts {
mod tests {
use prost::Message as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_text::{
FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params,
};
@@ -720,6 +727,7 @@ mod tests {
finish_reason: reason,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}
}
@@ -804,4 +812,40 @@ mod tests {
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30)));
}
#[test]
fn sequence_output_only_carries_sampling_mask_on_terminal_output() {
let mut fin = finished(FinishReason::Length);
fin.sampling_mask = Some(SamplingMask {
rows: vec![vec![1, 10], vec![2, 20]],
});
let terminal =
to_sequence_output("", &[10, 20], None, Some(&fin), &ResponseOpts::default())
.expect("terminal output");
let intermediate = to_sequence_output("", &[9], None, None, &ResponseOpts::default())
.expect("intermediate output");
assert_eq!(
terminal.sampling_mask,
vec![
pb::TokenIds { ids: vec![1, 10] },
pb::TokenIds { ids: vec![2, 20] }
]
);
assert!(intermediate.sampling_mask.is_empty());
}
#[test]
fn sampling_mask_uses_additive_proto_tag_ten() {
let response = pb::SequenceOutput {
sampling_mask: vec![pb::TokenIds { ids: vec![7, 8] }],
..Default::default()
};
assert_eq!(
response.encode_to_vec(),
vec![0x52, 0x04, 0x0a, 0x02, 0x07, 0x08]
);
}
}
+1
View File
@@ -257,6 +257,7 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
finish_reason: collected.finish_reason,
kv_transfer_params: collected.kv_transfer_params,
ec_transfer_params: collected.ec_transfer_params,
sampling_mask: collected.sampling_mask,
};
let outputs = convert::to_sequence_output(
@@ -481,6 +481,7 @@ mod tests {
cached_token_count: 0,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
Ok(GenerateOutput {
request_id: String::new(),
@@ -494,6 +495,7 @@ mod tests {
cached_token_count: 2,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
]);
@@ -575,6 +577,7 @@ mod tests {
cached_token_count: 0,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}
}
@@ -658,6 +661,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
sampling_mask: None,
};
let response = collect_generate(
@@ -744,6 +748,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
sampling_mask: None,
};
let response = collect_generate(
@@ -776,6 +781,7 @@ mod tests {
},
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
prompt_token_ids,
};
@@ -694,6 +694,7 @@ mod tests {
))),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
@@ -806,6 +807,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
@@ -858,6 +860,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
@@ -913,6 +916,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
@@ -985,6 +989,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);
@@ -1059,6 +1064,7 @@ mod tests {
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}),
)),
]);