feat(frontend): session id plumbing into requests (#48048)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-08-03 22:18:58 +00:00
committed by GitHub
parent e578de311c
commit f57123aa2d
39 changed files with 438 additions and 4 deletions
+1
View File
@@ -168,6 +168,7 @@ impl ChatRequestProcessor {
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id,
reasoning_parser_kwargs,
lora_request: request.lora_request,
arrival_time: Some(arrival_time),
+4
View File
@@ -491,6 +491,9 @@ pub struct ChatRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
@@ -514,6 +517,7 @@ impl ChatRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
session_id: None,
lora_request: None,
}
}
@@ -124,6 +124,9 @@ pub struct EngineCoreRequest {
/// standard `request_finished` hook.
#[serde(default)]
pub abort_immediately: bool,
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
}
impl EngineCoreRequest {
@@ -175,6 +178,7 @@ mod tests {
}),
arrival_time: 1234.5,
client_index: 7,
session_id: Some("session-1".to_string()),
..EngineCoreRequest::default()
};
@@ -185,12 +189,13 @@ mod tests {
other => panic!("expected array, got {other:?}"),
};
assert_eq!(array.len(), 20);
assert_eq!(array.len(), 21);
assert_eq!(array[0], Value::from("req-1"));
assert_eq!(array[2], Value::Nil);
assert_eq!(array[4], Value::Nil);
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
assert_eq!(array[20], Value::from("session-1"));
}
#[test]
@@ -162,6 +162,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 42.5,
session_id: Some("session-1".to_string()),
..EngineCoreRequest::default()
}
}
@@ -75,6 +75,7 @@ class EngineCoreRequest(
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, object] | None = None
abort_immediately: bool = False
session_id: str | None = None
class EngineCoreOutput(
@@ -137,6 +138,7 @@ request = EngineCoreRequest(
pooling_params=None,
arrival_time=42.5,
client_index=0,
session_id="session-1",
)
# All defaults -> empty map. Regression guard for the sparse-map decode.
@@ -59,6 +59,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
trace_headers: None,
priority: 0,
data_parallel_rank: None,
session_id: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
+6
View File
@@ -46,6 +46,8 @@ pub struct GenerateRequest {
pub priority: i32,
/// Optional data-parallel rank override for routing this request.
pub data_parallel_rank: Option<u32>,
/// Stable session identity shared by related requests.
pub session_id: Option<String>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
@@ -76,6 +78,7 @@ impl GenerateRequest {
trace_headers,
priority,
data_parallel_rank,
session_id,
reasoning_parser_kwargs,
lora_request,
} = self;
@@ -105,6 +108,7 @@ impl GenerateRequest {
priority,
trace_headers,
resumable: false,
session_id,
external_req_id: Some(external_request_id),
// Rust parser doesn't expose this information, leave it unset and let the
// reasoning logic in engine-sided structured output manager handle it.
@@ -150,6 +154,7 @@ mod tests {
)])),
priority: 3,
data_parallel_rank: Some(2),
session_id: Some("session-1".to_string()),
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
chat_template_kwargs: [(
"chat_template_kwargs".to_string(),
@@ -177,6 +182,7 @@ mod tests {
assert_eq!(request.arrival_time, 42.5);
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
assert_eq!(request.data_parallel_rank, Some(2));
assert_eq!(request.session_id.as_deref(), Some("session-1"));
assert_eq!(
request.trace_headers,
Some(BTreeMap::from([(
+1
View File
@@ -188,6 +188,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
trace_headers: None,
priority: 0,
data_parallel_rank: None,
session_id: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
+2
View File
@@ -53,6 +53,7 @@ pub fn to_text_request(
} else {
req.request_id
};
let session_id = req.session_id.filter(|s| !s.is_empty());
let sampling = req.sampling.as_ref();
let decoding = req.decoding.as_ref();
@@ -100,6 +101,7 @@ pub fn to_text_request(
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
add_special_tokens: true,
data_parallel_rank: None,
session_id,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,
@@ -75,6 +75,7 @@ pub(super) fn prepare_generate_request(
cache_salt: request.cache_salt,
add_special_tokens: false,
data_parallel_rank: ctx.data_parallel_rank,
session_id: ctx.session_id,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
arrival_time: None,
@@ -18,6 +18,7 @@ use crate::routes::openai::utils::types::{
};
use crate::utils::{
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
resolve_session_id,
};
/// Lowered chat request plus the public response metadata carried by every SSE
@@ -123,6 +124,11 @@ pub(super) fn prepare_chat_request(
request.response_format.as_ref(),
&request.structured_outputs,
)?;
let session_id = resolve_session_id(
&ctx,
request.session_id.as_deref(),
request.vllm_xargs.as_ref(),
);
let chat_request = ChatRequest {
request_id: request_id.clone(),
@@ -177,6 +183,7 @@ pub(super) fn prepare_chat_request(
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
session_id,
lora_request: lora_resolution.lora_request.clone(),
};
@@ -1241,6 +1248,49 @@ mod tests {
assert_eq!(prepared.chat_request.data_parallel_rank, Some(7));
}
#[test]
fn prepare_chat_request_threads_header_session_id() {
let mut headers = HeaderMap::new();
headers.insert("X-Session-ID", "header-session".parse().unwrap());
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.session_id.as_deref(),
Some("header-session")
);
}
#[test]
fn prepare_chat_request_ignores_correlation_header() {
let mut headers = HeaderMap::new();
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.session_id, None);
}
#[test]
fn prepare_chat_request_ignores_empty_session_header_without_fallback() {
let mut headers = HeaderMap::new();
headers.insert("X-Session-ID", "".parse().unwrap());
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.session_id, None);
}
#[test]
fn prepare_chat_request_leaves_data_parallel_rank_none_when_absent() {
let prepared = prepare_chat_request(
@@ -222,6 +222,9 @@ pub struct ChatCompletionRequest {
/// External request ID used for response correlation.
pub request_id: Option<String>,
/// Stable session identity shared by related requests.
pub session_id: Option<String>,
/// Tokens represented as strings of the form 'token_id:{token_id}' in
/// logprobs
pub return_tokens_as_token_ids: Option<bool>,
@@ -301,6 +304,7 @@ impl Default for ChatCompletionRequest {
structured_outputs: None,
priority: None,
request_id: None,
session_id: None,
return_tokens_as_token_ids: None,
return_token_ids: None,
cache_salt: None,
@@ -12,6 +12,7 @@ use crate::routes::openai::completions::validate;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
use crate::utils::{
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
resolve_session_id,
};
/// Lowered completion request plus the public response metadata carried by
@@ -103,6 +104,11 @@ pub(super) fn prepare_completion_request(
let structured_outputs =
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
let session_id = resolve_session_id(
&ctx,
request.session_id.as_deref(),
request.vllm_xargs.as_ref(),
);
let text_request = TextRequest {
request_id: request_id.clone(),
@@ -147,6 +153,7 @@ pub(super) fn prepare_completion_request(
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
session_id,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
arrival_time: None,
@@ -586,6 +593,76 @@ mod tests {
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
}
#[test]
fn prepare_completion_request_threads_body_session_id() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
"session_id": "body-session",
"vllm_xargs": {"session_id": "xargs-session"},
}))
.expect("parse request");
let mut headers = HeaderMap::new();
headers.insert("X-Session-ID", "header-session".parse().unwrap());
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(
prepared.text_request.session_id.as_deref(),
Some("body-session")
);
}
#[test]
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
"vllm_xargs": {"session_id": "xargs-session"},
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(
prepared.text_request.session_id.as_deref(),
Some("xargs-session")
);
}
#[test]
fn prepare_completion_request_ignores_empty_and_non_string_session_id_values() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
"session_id": "",
"vllm_xargs": {"session_id": 7},
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.session_id, None);
}
#[test]
fn prepare_completion_request_leaves_data_parallel_rank_none_when_absent() {
let request: CompletionRequest = serde_json::from_value(json!({
@@ -164,6 +164,9 @@ pub struct CompletionRequest {
/// External request ID used for response correlation.
pub request_id: Option<String>,
/// Stable session identity shared by related requests.
pub session_id: Option<String>,
/// Tokens represented as strings of the form 'token_id:{token_id}' in
/// logprobs
pub return_tokens_as_token_ids: Option<bool>,
@@ -95,6 +95,7 @@ impl TokenizeChatRequest {
cache_salt: None,
add_special_tokens: self.add_special_tokens,
data_parallel_rank: None,
session_id: None,
lora_request: None,
})
}
+27 -2
View File
@@ -15,6 +15,7 @@ use crate::error::ApiError;
pub struct ResolvedRequestContext {
pub request_id: String,
pub data_parallel_rank: Option<u32>,
pub session_id: Option<String>,
}
/// Return the current Unix timestamp in seconds for OpenAI response objects.
@@ -66,6 +67,24 @@ pub fn merge_ec_transfer_params(
xargs
}
pub fn resolve_session_id(
ctx: &ResolvedRequestContext,
request_session_id: Option<&str>,
xargs: Option<&HashMap<String, Value>>,
) -> Option<String> {
request_session_id
.filter(|value| !value.is_empty())
.map(str::to_owned)
.or_else(|| ctx.session_id.clone())
.or_else(|| {
xargs
.and_then(|map| map.get("session_id"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
})
}
/// Convert OpenAI-style `logit_bias` with string token-ID keys into the
/// internal `HashMap<u32, f32>` representation, validating that every key
/// parses as a `u32`.
@@ -91,8 +110,8 @@ pub fn convert_logit_bias(
.transpose()
}
/// Extract common request metadata from HTTP headers: the external request ID
/// and the optional data-parallel rank used for engine routing.
/// Extract common request metadata from HTTP headers: the external request ID,
/// session ID, and the optional data-parallel rank used for engine routing.
pub fn resolve_request_context(
headers: &HeaderMap,
request_id: Option<&str>,
@@ -106,10 +125,16 @@ pub fn resolve_request_context(
// Extract request id from header.
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
let request_id = resolve_base_request_id(request_id_header, request_id);
let session_id = headers
.get("X-Session-ID")
.and_then(|value| value.to_str().ok())
.filter(|value| !value.is_empty())
.map(str::to_owned);
ResolvedRequestContext {
request_id,
data_parallel_rank,
session_id,
}
}
+1
View File
@@ -59,6 +59,7 @@ pub fn lower_text_request(
cache_salt: request.cache_salt.clone(),
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id.clone(),
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
lora_request: request.lora_request.clone(),
arrival_time: request.arrival_time,
+4
View File
@@ -183,6 +183,9 @@ pub struct TextRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
#[serde(default)]
@@ -212,6 +215,7 @@ impl TextRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
session_id: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,