[Rust Frontend] Return sampling masks over gRPC (#56777)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
d5f0a6e829
commit
95d723700b
@@ -19,6 +19,7 @@ itertools.workspace = true
|
||||
libc.workspace = true
|
||||
llm-multimodal.workspace = true
|
||||
openssl.workspace = true
|
||||
prost.workspace = true
|
||||
prost-types.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
rmpv.workspace = true
|
||||
|
||||
@@ -316,6 +316,11 @@ pub fn to_sequence_output(
|
||||
_ => (vec![], vec![], vec![]),
|
||||
};
|
||||
|
||||
let sampling_mask = finished
|
||||
.and_then(|finished| finished.sampling_mask.as_ref())
|
||||
.map(|mask| mask.rows.iter().map(|row| pb::TokenIds { ids: row.clone() }).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(pb::SequenceOutput {
|
||||
index: 0, // TODO: multi-sequence (n > 1) not supported
|
||||
text: if opts.output_text {
|
||||
@@ -333,6 +338,7 @@ pub fn to_sequence_output(
|
||||
ranks: rank_values,
|
||||
candidate_tokens: candidates,
|
||||
finish_info,
|
||||
sampling_mask,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -524,6 +530,7 @@ impl ResponseOpts {
|
||||
mod tests {
|
||||
use prost::Message as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_text::{
|
||||
FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params,
|
||||
};
|
||||
@@ -720,6 +727,7 @@ mod tests {
|
||||
finish_reason: reason,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -804,4 +812,40 @@ mod tests {
|
||||
assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32);
|
||||
assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequence_output_only_carries_sampling_mask_on_terminal_output() {
|
||||
let mut fin = finished(FinishReason::Length);
|
||||
fin.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![1, 10], vec![2, 20]],
|
||||
});
|
||||
|
||||
let terminal =
|
||||
to_sequence_output("", &[10, 20], None, Some(&fin), &ResponseOpts::default())
|
||||
.expect("terminal output");
|
||||
let intermediate = to_sequence_output("", &[9], None, None, &ResponseOpts::default())
|
||||
.expect("intermediate output");
|
||||
|
||||
assert_eq!(
|
||||
terminal.sampling_mask,
|
||||
vec![
|
||||
pb::TokenIds { ids: vec![1, 10] },
|
||||
pb::TokenIds { ids: vec![2, 20] }
|
||||
]
|
||||
);
|
||||
assert!(intermediate.sampling_mask.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sampling_mask_uses_additive_proto_tag_ten() {
|
||||
let response = pb::SequenceOutput {
|
||||
sampling_mask: vec![pb::TokenIds { ids: vec![7, 8] }],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
response.encode_to_vec(),
|
||||
vec![0x52, 0x04, 0x0a, 0x02, 0x07, 0x08]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +257,7 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
finish_reason: collected.finish_reason,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
ec_transfer_params: collected.ec_transfer_params,
|
||||
sampling_mask: collected.sampling_mask,
|
||||
};
|
||||
|
||||
let outputs = convert::to_sequence_output(
|
||||
|
||||
@@ -481,6 +481,7 @@ mod tests {
|
||||
cached_token_count: 0,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
request_id: String::new(),
|
||||
@@ -494,6 +495,7 @@ mod tests {
|
||||
cached_token_count: 2,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -575,6 +577,7 @@ mod tests {
|
||||
cached_token_count: 0,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -658,6 +661,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -744,6 +748,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -776,6 +781,7 @@ mod tests {
|
||||
},
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
prompt_token_ids,
|
||||
};
|
||||
|
||||
|
||||
@@ -694,6 +694,7 @@ mod tests {
|
||||
))),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
@@ -806,6 +807,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
@@ -858,6 +860,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
@@ -913,6 +916,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
@@ -985,6 +989,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
@@ -1059,6 +1064,7 @@ mod tests {
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user