feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -53,6 +53,7 @@ pub fn to_text_request(
|
||||
} else {
|
||||
req.request_id
|
||||
};
|
||||
let session_id = req.session_id.filter(|s| !s.is_empty());
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
@@ -100,6 +101,7 @@ pub fn to_text_request(
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
@@ -75,6 +75,7 @@ pub(super) fn prepare_generate_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id: ctx.session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::routes::openai::utils::types::{
|
||||
};
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
resolve_session_id,
|
||||
};
|
||||
|
||||
/// Lowered chat request plus the public response metadata carried by every SSE
|
||||
@@ -123,6 +124,11 @@ pub(super) fn prepare_chat_request(
|
||||
request.response_format.as_ref(),
|
||||
&request.structured_outputs,
|
||||
)?;
|
||||
let session_id = resolve_session_id(
|
||||
&ctx,
|
||||
request.session_id.as_deref(),
|
||||
request.vllm_xargs.as_ref(),
|
||||
);
|
||||
|
||||
let chat_request = ChatRequest {
|
||||
request_id: request_id.clone(),
|
||||
@@ -177,6 +183,7 @@ pub(super) fn prepare_chat_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
@@ -1241,6 +1248,49 @@ mod tests {
|
||||
assert_eq!(prepared.chat_request.data_parallel_rank, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_threads_header_session_id() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "header-session".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(
|
||||
prepared.chat_request.session_id.as_deref(),
|
||||
Some("header-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_correlation_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_empty_session_header_without_fallback() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "".parse().unwrap());
|
||||
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let prepared = prepare_chat_request(
|
||||
|
||||
@@ -222,6 +222,9 @@ pub struct ChatCompletionRequest {
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
@@ -301,6 +304,7 @@ impl Default for ChatCompletionRequest {
|
||||
structured_outputs: None,
|
||||
priority: None,
|
||||
request_id: None,
|
||||
session_id: None,
|
||||
return_tokens_as_token_ids: None,
|
||||
return_token_ids: None,
|
||||
cache_salt: None,
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::routes::openai::completions::validate;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
resolve_session_id,
|
||||
};
|
||||
|
||||
/// Lowered completion request plus the public response metadata carried by
|
||||
@@ -103,6 +104,11 @@ pub(super) fn prepare_completion_request(
|
||||
|
||||
let structured_outputs =
|
||||
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
|
||||
let session_id = resolve_session_id(
|
||||
&ctx,
|
||||
request.session_id.as_deref(),
|
||||
request.vllm_xargs.as_ref(),
|
||||
);
|
||||
|
||||
let text_request = TextRequest {
|
||||
request_id: request_id.clone(),
|
||||
@@ -147,6 +153,7 @@ pub(super) fn prepare_completion_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
@@ -586,6 +593,76 @@ mod tests {
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_threads_body_session_id() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"session_id": "body-session",
|
||||
"vllm_xargs": {"session_id": "xargs-session"},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "header-session".parse().unwrap());
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(
|
||||
prepared.text_request.session_id.as_deref(),
|
||||
Some("body-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"vllm_xargs": {"session_id": "xargs-session"},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(
|
||||
prepared.text_request.session_id.as_deref(),
|
||||
Some("xargs-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_ignores_empty_and_non_string_session_id_values() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"session_id": "",
|
||||
"vllm_xargs": {"session_id": 7},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -164,6 +164,9 @@ pub struct CompletionRequest {
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TokenizeChatRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
lora_request: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::error::ApiError;
|
||||
pub struct ResolvedRequestContext {
|
||||
pub request_id: String,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Return the current Unix timestamp in seconds for OpenAI response objects.
|
||||
@@ -66,6 +67,24 @@ pub fn merge_ec_transfer_params(
|
||||
xargs
|
||||
}
|
||||
|
||||
pub fn resolve_session_id(
|
||||
ctx: &ResolvedRequestContext,
|
||||
request_session_id: Option<&str>,
|
||||
xargs: Option<&HashMap<String, Value>>,
|
||||
) -> Option<String> {
|
||||
request_session_id
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
.or_else(|| ctx.session_id.clone())
|
||||
.or_else(|| {
|
||||
xargs
|
||||
.and_then(|map| map.get("session_id"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert OpenAI-style `logit_bias` with string token-ID keys into the
|
||||
/// internal `HashMap<u32, f32>` representation, validating that every key
|
||||
/// parses as a `u32`.
|
||||
@@ -91,8 +110,8 @@ pub fn convert_logit_bias(
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Extract common request metadata from HTTP headers: the external request ID
|
||||
/// and the optional data-parallel rank used for engine routing.
|
||||
/// Extract common request metadata from HTTP headers: the external request ID,
|
||||
/// session ID, and the optional data-parallel rank used for engine routing.
|
||||
pub fn resolve_request_context(
|
||||
headers: &HeaderMap,
|
||||
request_id: Option<&str>,
|
||||
@@ -106,10 +125,16 @@ pub fn resolve_request_context(
|
||||
// Extract request id from header.
|
||||
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
|
||||
let request_id = resolve_base_request_id(request_id_header, request_id);
|
||||
let session_id = headers
|
||||
.get("X-Session-ID")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned);
|
||||
|
||||
ResolvedRequestContext {
|
||||
request_id,
|
||||
data_parallel_rank,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user