[Rust Frontend] Deduplicate request preprocessing for /tokenize (#50448)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Sage
2026-08-05 08:27:35 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent fd859d2e06
commit 5bf32605a9
5 changed files with 170 additions and 50 deletions
+25 -33
View File
@@ -155,17 +155,9 @@ impl ChatRequestProcessor {
Ok(Some(features))
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
mut request: ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
// Stamp before rendering so render and tokenize count toward TTFT/e2e.
let arrival_time = vllm_llm::current_unix_timestamp_secs();
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
let rendered = self.backend.chat_renderer().render(&request)?;
let reasoning_parser_kwargs =
request
@@ -176,7 +168,7 @@ impl ChatRequestProcessor {
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
});
let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
let text_request = TextRequest {
Ok(TextRequest {
request_id: request.request_id,
prompt,
mm_features,
@@ -191,7 +183,24 @@ impl ChatRequestProcessor {
reasoning_parser_kwargs,
lora_request: request.lora_request,
arrival_time: Some(arrival_time),
};
})
}
/// Prepare one chat request for tokenization without constructing an output processor.
pub async fn prepare_for_tokenization(&self, request: ChatRequest) -> Result<TextRequest> {
request.validate()?;
self.prepare_text_request(request).await
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
mut request: ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
let text_request = self.prepare_text_request(request).await?;
Ok((text_request, output_processor))
}
}
@@ -265,6 +274,11 @@ impl ChatLlm {
&self.text
}
/// Return the chat request processor.
pub fn request_processor(&self) -> &ChatRequestProcessor {
&self.processor
}
/// Return the model ID reported by the underlying text backend.
pub fn model_id(&self) -> &str {
self.text.model_id()
@@ -332,28 +346,6 @@ impl ChatLlm {
Ok(ChatEventStream::new(request_id, structured_stream))
}
/// Render through the chat template and tokenize, without submitting to the engine.
///
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
/// generation would see. Used by `POST /tokenize` (chat form).
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
request.validate()?;
let rendered = self.processor.backend.chat_renderer().render(&request)?;
let (prompt, _mm_features) =
self.processor.finalize_rendered_prompt(&request, rendered).await?;
let tokenizer = self.text.tokenizer();
let token_ids = match prompt {
// Rendered string from the template (usual chat path).
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
// Already tokenized (e.g. multimodal path); pass through unchanged.
Prompt::TokenIds(ids) => ids,
};
Ok(token_ids)
}
/// Abort in-flight requests by their external (user-supplied) request ids.
pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
self.text.abort(external_ids).await?;
+69
View File
@@ -6069,6 +6069,75 @@ async fn tokenize_chat_includes_generation_prompt_in_token_count() {
assert!(with_len > without_len);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_chat_with_tools_does_not_require_output_parser() {
let mut app = test_app().await;
let (status, json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}],
}),
)
.await;
assert_eq!(status, StatusCode::OK);
assert!(!json["tokens"].as_array().expect("tokens").is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_allows_prompts_at_or_above_max_model_len() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
max_model_len: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (completion_status, completion_json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"add_special_tokens": false,
}),
)
.await;
assert_eq!(completion_status, StatusCode::OK);
assert_eq!(completion_json["count"], 5);
assert_eq!(completion_json["max_model_len"], 4);
let (chat_status, chat_json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hello"}],
"add_generation_prompt": false,
"add_special_tokens": false,
}),
)
.await;
assert_eq!(chat_status, StatusCode::OK);
assert!(chat_json["count"].as_u64().expect("count") > 4);
assert_eq!(chat_json["max_model_len"], 4);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_chat_conflicting_generation_flags_returns_400() {
+20 -11
View File
@@ -60,9 +60,7 @@ pub async fn tokenize(
let max_model_len = state.chat.engine_core_client().max_model_len();
let result = match body {
// Completion form: encode the raw `prompt` string (no chat template).
TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req),
// Chat form: render `messages` through the template, then encode (see `tokenize_chat`).
TokenizeRequest::Completion(req) => tokenize_completion(&state, &request_id, req),
TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,
};
@@ -83,20 +81,24 @@ pub async fn tokenize(
fn tokenize_completion(
state: &AppState,
tokenizer: &vllm_text::tokenizer::DynTokenizer,
request_id: &str,
req: TokenizeCompletionRequest,
) -> Result<(Vec<u32>, bool), ApiError> {
check_model(state, req.model.as_deref())?;
let tokens = tokenizer
.encode(&req.prompt, req.add_special_tokens)
let return_token_strs = req.return_token_strs;
let tokens = state
.chat
.text()
.request_processor()
.tokenize(req.into_text_request(request_id.to_string()))
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, req.return_token_strs))
Ok((tokens, return_token_strs))
}
/// HTTP adapter for the chat-shaped `/tokenize` body.
///
/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps
/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.
/// Checks the model name, prepares without engine submission, and maps errors
/// to [`ApiError`].
async fn tokenize_chat(
state: &AppState,
request_id: &str,
@@ -106,10 +108,17 @@ async fn tokenize_chat(
let return_token_strs = req.return_token_strs;
// `continue_final_message` / `add_generation_prompt` mutual exclusion is
// enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.
let text_request = state
.chat
.request_processor()
.prepare_for_tokenization(req.into_chat_request(request_id.to_string())?)
.await
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
let tokens = state
.chat
.tokenize_chat(req.into_chat_request(request_id.to_string())?)
.await
.text()
.request_processor()
.tokenize(text_request)
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, return_token_strs))
}
@@ -9,6 +9,7 @@ use serde_json::Value;
use validator::{Validate, ValidationErrors};
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
use vllm_text::output::TextDecodeOptions;
use vllm_text::{Prompt, TextRequest};
use crate::error::ApiError;
use crate::routes::openai::chat_completions::convert::{
@@ -37,6 +38,28 @@ pub struct TokenizeCompletionRequest {
pub return_token_strs: bool,
}
impl TokenizeCompletionRequest {
/// Lower this tokenize body into a [`TextRequest`] for prompt tokenization.
pub fn into_text_request(self, request_id: String) -> TextRequest {
TextRequest {
request_id,
prompt: Prompt::Text(self.prompt),
mm_features: None,
sampling_params: SamplingParams::default(),
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: 0,
cache_salt: None,
add_special_tokens: self.add_special_tokens,
data_parallel_rank: None,
session_id: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,
}
}
}
#[derive(Debug, Clone, Deserialize, Validate)]
pub struct TokenizeChatRequest {
pub model: Option<String>,
+33 -6
View File
@@ -78,6 +78,29 @@ impl TextRequestProcessor {
self.max_model_len
}
fn tokenize_prompt(
tokenizer: &DynTokenizer,
prompt: Prompt,
add_special_tokens: bool,
) -> Result<Vec<u32>> {
match prompt {
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens).map_err(Into::into),
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
Prompt::TokenIds(token_ids) => Ok(token_ids),
}
}
/// Tokenize one request without generation-specific lowering.
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
request.validate()?;
Self::tokenize_prompt(
&self.backend.tokenizer(),
request.prompt,
request.add_special_tokens,
)
}
/// Tokenize and lower one request without submitting it to an engine.
pub fn prepare(&self, mut request: TextRequest) -> Result<PreparedTextRequest> {
request.validate()?;
@@ -87,12 +110,11 @@ impl TextRequestProcessor {
}
let tokenizer = self.backend.tokenizer();
let prompt_token_ids = match take(&mut request.prompt) {
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
Prompt::TokenIds(token_ids) => token_ids,
};
let prompt_token_ids = Self::tokenize_prompt(
&tokenizer,
take(&mut request.prompt),
request.add_special_tokens,
)?;
let sampling_hints = self.backend.sampling_hints()?;
let sampling_limits = SamplingLimits {
max_model_len: self.max_model_len,
@@ -153,6 +175,11 @@ impl TextLlm {
self.llm.engine_core_client()
}
/// Return the text request processor.
pub fn request_processor(&self) -> &TextRequestProcessor {
&self.processor
}
/// Return the tokenizer used by this text backend.
pub fn tokenizer(&self) -> DynTokenizer {
self.processor.tokenizer()