[Rust Frontend] Deduplicate request preprocessing for /tokenize (#50448)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Sage Ahrac <sagiahrak@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -6069,6 +6069,75 @@ async fn tokenize_chat_includes_generation_prompt_in_token_count() {
|
||||
assert!(with_len > without_len);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_chat_with_tools_does_not_require_output_parser() {
|
||||
let mut app = test_app().await;
|
||||
|
||||
let (status, json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(!json["tokens"].as_array().expect("tokens").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_allows_prompts_at_or_above_max_model_len() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
max_model_len: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
|
||||
let (completion_status, completion_json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"add_special_tokens": false,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(completion_status, StatusCode::OK);
|
||||
assert_eq!(completion_json["count"], 5);
|
||||
assert_eq!(completion_json["max_model_len"], 4);
|
||||
|
||||
let (chat_status, chat_json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"add_generation_prompt": false,
|
||||
"add_special_tokens": false,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(chat_status, StatusCode::OK);
|
||||
assert!(chat_json["count"].as_u64().expect("count") > 4);
|
||||
assert_eq!(chat_json["max_model_len"], 4);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_chat_conflicting_generation_flags_returns_400() {
|
||||
|
||||
@@ -60,9 +60,7 @@ pub async fn tokenize(
|
||||
let max_model_len = state.chat.engine_core_client().max_model_len();
|
||||
|
||||
let result = match body {
|
||||
// Completion form: encode the raw `prompt` string (no chat template).
|
||||
TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req),
|
||||
// Chat form: render `messages` through the template, then encode (see `tokenize_chat`).
|
||||
TokenizeRequest::Completion(req) => tokenize_completion(&state, &request_id, req),
|
||||
TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,
|
||||
};
|
||||
|
||||
@@ -83,20 +81,24 @@ pub async fn tokenize(
|
||||
|
||||
fn tokenize_completion(
|
||||
state: &AppState,
|
||||
tokenizer: &vllm_text::tokenizer::DynTokenizer,
|
||||
request_id: &str,
|
||||
req: TokenizeCompletionRequest,
|
||||
) -> Result<(Vec<u32>, bool), ApiError> {
|
||||
check_model(state, req.model.as_deref())?;
|
||||
let tokens = tokenizer
|
||||
.encode(&req.prompt, req.add_special_tokens)
|
||||
let return_token_strs = req.return_token_strs;
|
||||
let tokens = state
|
||||
.chat
|
||||
.text()
|
||||
.request_processor()
|
||||
.tokenize(req.into_text_request(request_id.to_string()))
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
Ok((tokens, req.return_token_strs))
|
||||
Ok((tokens, return_token_strs))
|
||||
}
|
||||
|
||||
/// HTTP adapter for the chat-shaped `/tokenize` body.
|
||||
///
|
||||
/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps
|
||||
/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.
|
||||
/// Checks the model name, prepares without engine submission, and maps errors
|
||||
/// to [`ApiError`].
|
||||
async fn tokenize_chat(
|
||||
state: &AppState,
|
||||
request_id: &str,
|
||||
@@ -106,10 +108,17 @@ async fn tokenize_chat(
|
||||
let return_token_strs = req.return_token_strs;
|
||||
// `continue_final_message` / `add_generation_prompt` mutual exclusion is
|
||||
// enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.
|
||||
let text_request = state
|
||||
.chat
|
||||
.request_processor()
|
||||
.prepare_for_tokenization(req.into_chat_request(request_id.to_string())?)
|
||||
.await
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
let tokens = state
|
||||
.chat
|
||||
.tokenize_chat(req.into_chat_request(request_id.to_string())?)
|
||||
.await
|
||||
.text()
|
||||
.request_processor()
|
||||
.tokenize(text_request)
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
Ok((tokens, return_token_strs))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ use serde_json::Value;
|
||||
use validator::{Validate, ValidationErrors};
|
||||
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
use vllm_text::{Prompt, TextRequest};
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::routes::openai::chat_completions::convert::{
|
||||
@@ -37,6 +38,28 @@ pub struct TokenizeCompletionRequest {
|
||||
pub return_token_strs: bool,
|
||||
}
|
||||
|
||||
impl TokenizeCompletionRequest {
|
||||
/// Lower this tokenize body into a [`TextRequest`] for prompt tokenization.
|
||||
pub fn into_text_request(self, request_id: String) -> TextRequest {
|
||||
TextRequest {
|
||||
request_id,
|
||||
prompt: Prompt::Text(self.prompt),
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams::default(),
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
priority: 0,
|
||||
cache_salt: None,
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Validate)]
|
||||
pub struct TokenizeChatRequest {
|
||||
pub model: Option<String>,
|
||||
|
||||
Reference in New Issue
Block a user