[Rust Frontend] Deduplicate request preprocessing for /tokenize (#50448)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Sage
2026-08-05 08:27:35 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent fd859d2e06
commit 5bf32605a9
5 changed files with 170 additions and 50 deletions
+69
View File
@@ -6069,6 +6069,75 @@ async fn tokenize_chat_includes_generation_prompt_in_token_count() {
assert!(with_len > without_len);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_chat_with_tools_does_not_require_output_parser() {
let mut app = test_app().await;
let (status, json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}],
}),
)
.await;
assert_eq!(status, StatusCode::OK);
assert!(!json["tokens"].as_array().expect("tokens").is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_allows_prompts_at_or_above_max_model_len() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
max_model_len: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (completion_status, completion_json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"add_special_tokens": false,
}),
)
.await;
assert_eq!(completion_status, StatusCode::OK);
assert_eq!(completion_json["count"], 5);
assert_eq!(completion_json["max_model_len"], 4);
let (chat_status, chat_json) = post_json(
&mut app,
"/tokenize",
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hello"}],
"add_generation_prompt": false,
"add_special_tokens": false,
}),
)
.await;
assert_eq!(chat_status, StatusCode::OK);
assert!(chat_json["count"].as_u64().expect("count") > 4);
assert_eq!(chat_json["max_model_len"], 4);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn tokenize_chat_conflicting_generation_flags_returns_400() {
+20 -11
View File
@@ -60,9 +60,7 @@ pub async fn tokenize(
let max_model_len = state.chat.engine_core_client().max_model_len();
let result = match body {
// Completion form: encode the raw `prompt` string (no chat template).
TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req),
// Chat form: render `messages` through the template, then encode (see `tokenize_chat`).
TokenizeRequest::Completion(req) => tokenize_completion(&state, &request_id, req),
TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,
};
@@ -83,20 +81,24 @@ pub async fn tokenize(
fn tokenize_completion(
state: &AppState,
tokenizer: &vllm_text::tokenizer::DynTokenizer,
request_id: &str,
req: TokenizeCompletionRequest,
) -> Result<(Vec<u32>, bool), ApiError> {
check_model(state, req.model.as_deref())?;
let tokens = tokenizer
.encode(&req.prompt, req.add_special_tokens)
let return_token_strs = req.return_token_strs;
let tokens = state
.chat
.text()
.request_processor()
.tokenize(req.into_text_request(request_id.to_string()))
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, req.return_token_strs))
Ok((tokens, return_token_strs))
}
/// HTTP adapter for the chat-shaped `/tokenize` body.
///
/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps
/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.
/// Checks the model name, prepares without engine submission, and maps errors
/// to [`ApiError`].
async fn tokenize_chat(
state: &AppState,
request_id: &str,
@@ -106,10 +108,17 @@ async fn tokenize_chat(
let return_token_strs = req.return_token_strs;
// `continue_final_message` / `add_generation_prompt` mutual exclusion is
// enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.
let text_request = state
.chat
.request_processor()
.prepare_for_tokenization(req.into_chat_request(request_id.to_string())?)
.await
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
let tokens = state
.chat
.tokenize_chat(req.into_chat_request(request_id.to_string())?)
.await
.text()
.request_processor()
.tokenize(text_request)
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, return_token_strs))
}
@@ -9,6 +9,7 @@ use serde_json::Value;
use validator::{Validate, ValidationErrors};
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
use vllm_text::output::TextDecodeOptions;
use vllm_text::{Prompt, TextRequest};
use crate::error::ApiError;
use crate::routes::openai::chat_completions::convert::{
@@ -37,6 +38,28 @@ pub struct TokenizeCompletionRequest {
pub return_token_strs: bool,
}
impl TokenizeCompletionRequest {
/// Lower this tokenize body into a [`TextRequest`] for prompt tokenization.
pub fn into_text_request(self, request_id: String) -> TextRequest {
TextRequest {
request_id,
prompt: Prompt::Text(self.prompt),
mm_features: None,
sampling_params: SamplingParams::default(),
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: 0,
cache_salt: None,
add_special_tokens: self.add_special_tokens,
data_parallel_rank: None,
session_id: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,
}
}
}
#[derive(Debug, Clone, Deserialize, Validate)]
pub struct TokenizeChatRequest {
pub model: Option<String>,