[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+1
View File
@@ -9,6 +9,7 @@ anyhow.workspace = true
asynk-strim-attr.workspace = true
auto_enums.workspace = true
axum.workspace = true
base64.workspace = true
educe.workspace = true
futures.workspace = true
http-body.workspace = true
@@ -16,12 +16,15 @@ use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use base64::Engine as _;
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
@@ -330,6 +333,11 @@ fn collect_generate(
token_ids: collected.token_ids,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
.prompt_token_id_logprobs
.as_ref()
.map(npy_base64)
.transpose()?,
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
kv_transfer_params: collected.kv_transfer_params,
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
.collect()
}
/// Encode a float32 matrix as Python `numpy2base64` does.
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
let (TensorDtype::F32, Some(data), [rows, cols]) = (
array.dtype.scalar,
array.data.as_raw_view(),
array.shape.as_slice(),
) else {
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
};
let mut header =
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
header.push_str(&" ".repeat(21 - rows.to_string().len()));
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
header.push('\n');
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
npy.extend_from_slice(header.as_bytes());
npy.extend_from_slice(data);
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
}
fn format_token_id(token_id: u32) -> String {
format!("token_id:{token_id}")
}
@@ -509,6 +540,7 @@ mod tests {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22_u32]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33],
logprobs: None,
@@ -592,6 +624,7 @@ mod tests {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -747,6 +780,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -781,6 +815,17 @@ mod tests {
assert!(response.mm_placeholders.is_none());
}
#[test]
fn prompt_token_id_logprobs_match_numpy2base64() {
let scores =
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
assert_eq!(
npy_base64(&scores).unwrap(),
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
);
}
#[test]
fn collect_generate_returns_spec_decode_metrics() {
let metrics = RequestSpecDecodeMetrics {
@@ -792,6 +837,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-metrics".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -879,6 +925,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -914,6 +961,7 @@ mod tests {
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![3],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
pub prompt_token_id_logprobs: Option<String>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
pub kv_transfer_params: Option<Value>,
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
}
}
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprob_token_ids` are not available when `stream=true`."
);
}
Ok(())
}
@@ -169,6 +176,17 @@ mod tests {
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
thinking_token_budget: request.thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,