feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -168,6 +168,7 @@ impl ChatRequestProcessor {
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
arrival_time: Some(arrival_time),
|
||||
|
||||
@@ -491,6 +491,9 @@ pub struct ChatRequest {
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
/// LoRA adapter selected for this request.
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
@@ -514,6 +517,7 @@ impl ChatRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,6 +124,9 @@ pub struct EngineCoreRequest {
|
||||
/// standard `request_finished` hook.
|
||||
#[serde(default)]
|
||||
pub abort_immediately: bool,
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
@@ -175,6 +178,7 @@ mod tests {
|
||||
}),
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
session_id: Some("session-1".to_string()),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
@@ -185,12 +189,13 @@ mod tests {
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 20);
|
||||
assert_eq!(array.len(), 21);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
assert_eq!(array[20], Value::from("session-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -162,6 +162,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 42.5,
|
||||
session_id: Some("session-1".to_string()),
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ class EngineCoreRequest(
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, object] | None = None
|
||||
abort_immediately: bool = False
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
@@ -137,6 +138,7 @@ request = EngineCoreRequest(
|
||||
pooling_params=None,
|
||||
arrival_time=42.5,
|
||||
client_index=0,
|
||||
session_id="session-1",
|
||||
)
|
||||
|
||||
# All defaults -> empty map. Regression guard for the sparse-map decode.
|
||||
|
||||
@@ -59,6 +59,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
|
||||
@@ -46,6 +46,8 @@ pub struct GenerateRequest {
|
||||
pub priority: i32,
|
||||
/// Optional data-parallel rank override for routing this request.
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
@@ -76,6 +78,7 @@ impl GenerateRequest {
|
||||
trace_headers,
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
session_id,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request,
|
||||
} = self;
|
||||
@@ -105,6 +108,7 @@ impl GenerateRequest {
|
||||
priority,
|
||||
trace_headers,
|
||||
resumable: false,
|
||||
session_id,
|
||||
external_req_id: Some(external_request_id),
|
||||
// Rust parser doesn't expose this information, leave it unset and let the
|
||||
// reasoning logic in engine-sided structured output manager handle it.
|
||||
@@ -150,6 +154,7 @@ mod tests {
|
||||
)])),
|
||||
priority: 3,
|
||||
data_parallel_rank: Some(2),
|
||||
session_id: Some("session-1".to_string()),
|
||||
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
|
||||
chat_template_kwargs: [(
|
||||
"chat_template_kwargs".to_string(),
|
||||
@@ -177,6 +182,7 @@ mod tests {
|
||||
assert_eq!(request.arrival_time, 42.5);
|
||||
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
|
||||
assert_eq!(request.data_parallel_rank, Some(2));
|
||||
assert_eq!(request.session_id.as_deref(), Some("session-1"));
|
||||
assert_eq!(
|
||||
request.trace_headers,
|
||||
Some(BTreeMap::from([(
|
||||
|
||||
@@ -188,6 +188,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ pub fn to_text_request(
|
||||
} else {
|
||||
req.request_id
|
||||
};
|
||||
let session_id = req.session_id.filter(|s| !s.is_empty());
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
@@ -100,6 +101,7 @@ pub fn to_text_request(
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
@@ -75,6 +75,7 @@ pub(super) fn prepare_generate_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id: ctx.session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::routes::openai::utils::types::{
|
||||
};
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
resolve_session_id,
|
||||
};
|
||||
|
||||
/// Lowered chat request plus the public response metadata carried by every SSE
|
||||
@@ -123,6 +124,11 @@ pub(super) fn prepare_chat_request(
|
||||
request.response_format.as_ref(),
|
||||
&request.structured_outputs,
|
||||
)?;
|
||||
let session_id = resolve_session_id(
|
||||
&ctx,
|
||||
request.session_id.as_deref(),
|
||||
request.vllm_xargs.as_ref(),
|
||||
);
|
||||
|
||||
let chat_request = ChatRequest {
|
||||
request_id: request_id.clone(),
|
||||
@@ -177,6 +183,7 @@ pub(super) fn prepare_chat_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
@@ -1241,6 +1248,49 @@ mod tests {
|
||||
assert_eq!(prepared.chat_request.data_parallel_rank, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_threads_header_session_id() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "header-session".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(
|
||||
prepared.chat_request.session_id.as_deref(),
|
||||
Some("header-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_correlation_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_empty_session_header_without_fallback() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "".parse().unwrap());
|
||||
headers.insert("X-Correlation-ID", "correlation-header".parse().unwrap());
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let prepared = prepare_chat_request(
|
||||
|
||||
@@ -222,6 +222,9 @@ pub struct ChatCompletionRequest {
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
@@ -301,6 +304,7 @@ impl Default for ChatCompletionRequest {
|
||||
structured_outputs: None,
|
||||
priority: None,
|
||||
request_id: None,
|
||||
session_id: None,
|
||||
return_tokens_as_token_ids: None,
|
||||
return_token_ids: None,
|
||||
cache_salt: None,
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::routes::openai::completions::validate;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
resolve_session_id,
|
||||
};
|
||||
|
||||
/// Lowered completion request plus the public response metadata carried by
|
||||
@@ -103,6 +104,11 @@ pub(super) fn prepare_completion_request(
|
||||
|
||||
let structured_outputs =
|
||||
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
|
||||
let session_id = resolve_session_id(
|
||||
&ctx,
|
||||
request.session_id.as_deref(),
|
||||
request.vllm_xargs.as_ref(),
|
||||
);
|
||||
|
||||
let text_request = TextRequest {
|
||||
request_id: request_id.clone(),
|
||||
@@ -147,6 +153,7 @@ pub(super) fn prepare_completion_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
@@ -586,6 +593,76 @@ mod tests {
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_threads_body_session_id() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"session_id": "body-session",
|
||||
"vllm_xargs": {"session_id": "xargs-session"},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Session-ID", "header-session".parse().unwrap());
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(
|
||||
prepared.text_request.session_id.as_deref(),
|
||||
Some("body-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"vllm_xargs": {"session_id": "xargs-session"},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(
|
||||
prepared.text_request.session_id.as_deref(),
|
||||
Some("xargs-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_ignores_empty_and_non_string_session_id_values() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"session_id": "",
|
||||
"vllm_xargs": {"session_id": 7},
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.session_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_leaves_data_parallel_rank_none_when_absent() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -164,6 +164,9 @@ pub struct CompletionRequest {
|
||||
/// External request ID used for response correlation.
|
||||
pub request_id: Option<String>,
|
||||
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
|
||||
/// Tokens represented as strings of the form 'token_id:{token_id}' in
|
||||
/// logprobs
|
||||
pub return_tokens_as_token_ids: Option<bool>,
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TokenizeChatRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
lora_request: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::error::ApiError;
|
||||
pub struct ResolvedRequestContext {
|
||||
pub request_id: String,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Return the current Unix timestamp in seconds for OpenAI response objects.
|
||||
@@ -66,6 +67,24 @@ pub fn merge_ec_transfer_params(
|
||||
xargs
|
||||
}
|
||||
|
||||
pub fn resolve_session_id(
|
||||
ctx: &ResolvedRequestContext,
|
||||
request_session_id: Option<&str>,
|
||||
xargs: Option<&HashMap<String, Value>>,
|
||||
) -> Option<String> {
|
||||
request_session_id
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
.or_else(|| ctx.session_id.clone())
|
||||
.or_else(|| {
|
||||
xargs
|
||||
.and_then(|map| map.get("session_id"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert OpenAI-style `logit_bias` with string token-ID keys into the
|
||||
/// internal `HashMap<u32, f32>` representation, validating that every key
|
||||
/// parses as a `u32`.
|
||||
@@ -91,8 +110,8 @@ pub fn convert_logit_bias(
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Extract common request metadata from HTTP headers: the external request ID
|
||||
/// and the optional data-parallel rank used for engine routing.
|
||||
/// Extract common request metadata from HTTP headers: the external request ID,
|
||||
/// session ID, and the optional data-parallel rank used for engine routing.
|
||||
pub fn resolve_request_context(
|
||||
headers: &HeaderMap,
|
||||
request_id: Option<&str>,
|
||||
@@ -106,10 +125,16 @@ pub fn resolve_request_context(
|
||||
// Extract request id from header.
|
||||
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
|
||||
let request_id = resolve_base_request_id(request_id_header, request_id);
|
||||
let session_id = headers
|
||||
.get("X-Session-ID")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned);
|
||||
|
||||
ResolvedRequestContext {
|
||||
request_id,
|
||||
data_parallel_rank,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ pub fn lower_text_request(
|
||||
cache_salt: request.cache_salt.clone(),
|
||||
priority: request.priority,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id.clone(),
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
lora_request: request.lora_request.clone(),
|
||||
arrival_time: request.arrival_time,
|
||||
|
||||
@@ -183,6 +183,9 @@ pub struct TextRequest {
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
#[serde(default)]
|
||||
@@ -212,6 +215,7 @@ impl TextRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
Reference in New Issue
Block a user