[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -9,6 +9,7 @@ anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
auto_enums.workspace = true
|
||||
axum.workspace = true
|
||||
base64.workspace = true
|
||||
educe.workspace = true
|
||||
futures.workspace = true
|
||||
http-body.workspace = true
|
||||
|
||||
@@ -16,12 +16,15 @@ use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use base64::Engine as _;
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::dtype::TensorDtype;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
|
||||
};
|
||||
@@ -330,6 +333,11 @@ fn collect_generate(
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
.prompt_token_id_logprobs
|
||||
.as_ref()
|
||||
.map(npy_base64)
|
||||
.transpose()?,
|
||||
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
|
||||
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Encode a float32 matrix as Python `numpy2base64` does.
|
||||
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
|
||||
let (TensorDtype::F32, Some(data), [rows, cols]) = (
|
||||
array.dtype.scalar,
|
||||
array.data.as_raw_view(),
|
||||
array.shape.as_slice(),
|
||||
) else {
|
||||
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
|
||||
};
|
||||
let mut header =
|
||||
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
|
||||
header.push_str(&" ".repeat(21 - rows.to_string().len()));
|
||||
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
|
||||
header.push('\n');
|
||||
|
||||
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
|
||||
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
|
||||
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
|
||||
npy.extend_from_slice(header.as_bytes());
|
||||
npy.extend_from_slice(data);
|
||||
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
|
||||
}
|
||||
|
||||
fn format_token_id(token_id: u32) -> String {
|
||||
format!("token_id:{token_id}")
|
||||
}
|
||||
@@ -509,6 +540,7 @@ mod tests {
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22_u32]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids: vec![33],
|
||||
logprobs: None,
|
||||
@@ -592,6 +624,7 @@ mod tests {
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -747,6 +780,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -781,6 +815,17 @@ mod tests {
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_token_id_logprobs_match_numpy2base64() {
|
||||
let scores =
|
||||
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
|
||||
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
|
||||
assert_eq!(
|
||||
npy_base64(&scores).unwrap(),
|
||||
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_returns_spec_decode_metrics() {
|
||||
let metrics = RequestSpecDecodeMetrics {
|
||||
@@ -792,6 +837,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-metrics".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -879,6 +925,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -914,6 +961,7 @@ mod tests {
|
||||
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
|
||||
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseChoice>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
|
||||
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
|
||||
pub prompt_token_id_logprobs: Option<String>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
|
||||
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
|
||||
}
|
||||
}
|
||||
|
||||
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"`prompt_logprob_token_ids` are not available when `stream=true`."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -169,6 +176,17 @@ mod tests {
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"token_ids": [11, 22],
|
||||
"stream": true,
|
||||
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
Reference in New Issue
Block a user