[Rust Frontend] Deduplicate request preprocessing for /tokenize (#50448)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Sage Ahrac <sagiahrak@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+25
-33
@@ -155,17 +155,9 @@ impl ChatRequestProcessor {
|
||||
Ok(Some(features))
|
||||
}
|
||||
|
||||
/// Prepare one chat request without submitting it to an engine.
|
||||
pub async fn prepare(
|
||||
&self,
|
||||
mut request: ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> Result<(TextRequest, DynChatOutputProcessor)> {
|
||||
request.validate()?;
|
||||
|
||||
async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
|
||||
// Stamp before rendering so render and tokenize count toward TTFT/e2e.
|
||||
let arrival_time = vllm_llm::current_unix_timestamp_secs();
|
||||
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
let reasoning_parser_kwargs =
|
||||
request
|
||||
@@ -176,7 +168,7 @@ impl ChatRequestProcessor {
|
||||
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
|
||||
});
|
||||
let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
|
||||
let text_request = TextRequest {
|
||||
Ok(TextRequest {
|
||||
request_id: request.request_id,
|
||||
prompt,
|
||||
mm_features,
|
||||
@@ -191,7 +183,24 @@ impl ChatRequestProcessor {
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
arrival_time: Some(arrival_time),
|
||||
};
|
||||
})
|
||||
}
|
||||
|
||||
/// Prepare one chat request for tokenization without constructing an output processor.
|
||||
pub async fn prepare_for_tokenization(&self, request: ChatRequest) -> Result<TextRequest> {
|
||||
request.validate()?;
|
||||
self.prepare_text_request(request).await
|
||||
}
|
||||
|
||||
/// Prepare one chat request without submitting it to an engine.
|
||||
pub async fn prepare(
|
||||
&self,
|
||||
mut request: ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> Result<(TextRequest, DynChatOutputProcessor)> {
|
||||
request.validate()?;
|
||||
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
|
||||
let text_request = self.prepare_text_request(request).await?;
|
||||
Ok((text_request, output_processor))
|
||||
}
|
||||
}
|
||||
@@ -265,6 +274,11 @@ impl ChatLlm {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Return the chat request processor.
|
||||
pub fn request_processor(&self) -> &ChatRequestProcessor {
|
||||
&self.processor
|
||||
}
|
||||
|
||||
/// Return the model ID reported by the underlying text backend.
|
||||
pub fn model_id(&self) -> &str {
|
||||
self.text.model_id()
|
||||
@@ -332,28 +346,6 @@ impl ChatLlm {
|
||||
Ok(ChatEventStream::new(request_id, structured_stream))
|
||||
}
|
||||
|
||||
/// Render through the chat template and tokenize, without submitting to the engine.
|
||||
///
|
||||
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
|
||||
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
|
||||
/// generation would see. Used by `POST /tokenize` (chat form).
|
||||
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
|
||||
request.validate()?;
|
||||
|
||||
let rendered = self.processor.backend.chat_renderer().render(&request)?;
|
||||
let (prompt, _mm_features) =
|
||||
self.processor.finalize_rendered_prompt(&request, rendered).await?;
|
||||
|
||||
let tokenizer = self.text.tokenizer();
|
||||
let token_ids = match prompt {
|
||||
// Rendered string from the template (usual chat path).
|
||||
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
|
||||
// Already tokenized (e.g. multimodal path); pass through unchanged.
|
||||
Prompt::TokenIds(ids) => ids,
|
||||
};
|
||||
Ok(token_ids)
|
||||
}
|
||||
|
||||
/// Abort in-flight requests by their external (user-supplied) request ids.
|
||||
pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
|
||||
self.text.abort(external_ids).await?;
|
||||
|
||||
@@ -6069,6 +6069,75 @@ async fn tokenize_chat_includes_generation_prompt_in_token_count() {
|
||||
assert!(with_len > without_len);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_chat_with_tools_does_not_require_output_parser() {
|
||||
let mut app = test_app().await;
|
||||
|
||||
let (status, json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(!json["tokens"].as_array().expect("tokens").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_allows_prompts_at_or_above_max_model_len() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
max_model_len: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
|
||||
let (completion_status, completion_json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"add_special_tokens": false,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(completion_status, StatusCode::OK);
|
||||
assert_eq!(completion_json["count"], 5);
|
||||
assert_eq!(completion_json["max_model_len"], 4);
|
||||
|
||||
let (chat_status, chat_json) = post_json(
|
||||
&mut app,
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"add_generation_prompt": false,
|
||||
"add_special_tokens": false,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(chat_status, StatusCode::OK);
|
||||
assert!(chat_json["count"].as_u64().expect("count") > 4);
|
||||
assert_eq!(chat_json["max_model_len"], 4);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn tokenize_chat_conflicting_generation_flags_returns_400() {
|
||||
|
||||
@@ -60,9 +60,7 @@ pub async fn tokenize(
|
||||
let max_model_len = state.chat.engine_core_client().max_model_len();
|
||||
|
||||
let result = match body {
|
||||
// Completion form: encode the raw `prompt` string (no chat template).
|
||||
TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req),
|
||||
// Chat form: render `messages` through the template, then encode (see `tokenize_chat`).
|
||||
TokenizeRequest::Completion(req) => tokenize_completion(&state, &request_id, req),
|
||||
TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,
|
||||
};
|
||||
|
||||
@@ -83,20 +81,24 @@ pub async fn tokenize(
|
||||
|
||||
fn tokenize_completion(
|
||||
state: &AppState,
|
||||
tokenizer: &vllm_text::tokenizer::DynTokenizer,
|
||||
request_id: &str,
|
||||
req: TokenizeCompletionRequest,
|
||||
) -> Result<(Vec<u32>, bool), ApiError> {
|
||||
check_model(state, req.model.as_deref())?;
|
||||
let tokens = tokenizer
|
||||
.encode(&req.prompt, req.add_special_tokens)
|
||||
let return_token_strs = req.return_token_strs;
|
||||
let tokens = state
|
||||
.chat
|
||||
.text()
|
||||
.request_processor()
|
||||
.tokenize(req.into_text_request(request_id.to_string()))
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
Ok((tokens, req.return_token_strs))
|
||||
Ok((tokens, return_token_strs))
|
||||
}
|
||||
|
||||
/// HTTP adapter for the chat-shaped `/tokenize` body.
|
||||
///
|
||||
/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps
|
||||
/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.
|
||||
/// Checks the model name, prepares without engine submission, and maps errors
|
||||
/// to [`ApiError`].
|
||||
async fn tokenize_chat(
|
||||
state: &AppState,
|
||||
request_id: &str,
|
||||
@@ -106,10 +108,17 @@ async fn tokenize_chat(
|
||||
let return_token_strs = req.return_token_strs;
|
||||
// `continue_final_message` / `add_generation_prompt` mutual exclusion is
|
||||
// enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.
|
||||
let text_request = state
|
||||
.chat
|
||||
.request_processor()
|
||||
.prepare_for_tokenization(req.into_chat_request(request_id.to_string())?)
|
||||
.await
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
let tokens = state
|
||||
.chat
|
||||
.tokenize_chat(req.into_chat_request(request_id.to_string())?)
|
||||
.await
|
||||
.text()
|
||||
.request_processor()
|
||||
.tokenize(text_request)
|
||||
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
|
||||
Ok((tokens, return_token_strs))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ use serde_json::Value;
|
||||
use validator::{Validate, ValidationErrors};
|
||||
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
use vllm_text::{Prompt, TextRequest};
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::routes::openai::chat_completions::convert::{
|
||||
@@ -37,6 +38,28 @@ pub struct TokenizeCompletionRequest {
|
||||
pub return_token_strs: bool,
|
||||
}
|
||||
|
||||
impl TokenizeCompletionRequest {
|
||||
/// Lower this tokenize body into a [`TextRequest`] for prompt tokenization.
|
||||
pub fn into_text_request(self, request_id: String) -> TextRequest {
|
||||
TextRequest {
|
||||
request_id,
|
||||
prompt: Prompt::Text(self.prompt),
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams::default(),
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
priority: 0,
|
||||
cache_salt: None,
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Validate)]
|
||||
pub struct TokenizeChatRequest {
|
||||
pub model: Option<String>,
|
||||
|
||||
@@ -78,6 +78,29 @@ impl TextRequestProcessor {
|
||||
self.max_model_len
|
||||
}
|
||||
|
||||
fn tokenize_prompt(
|
||||
tokenizer: &DynTokenizer,
|
||||
prompt: Prompt,
|
||||
add_special_tokens: bool,
|
||||
) -> Result<Vec<u32>> {
|
||||
match prompt {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens).map_err(Into::into),
|
||||
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
|
||||
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
|
||||
Prompt::TokenIds(token_ids) => Ok(token_ids),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokenize one request without generation-specific lowering.
|
||||
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
|
||||
request.validate()?;
|
||||
Self::tokenize_prompt(
|
||||
&self.backend.tokenizer(),
|
||||
request.prompt,
|
||||
request.add_special_tokens,
|
||||
)
|
||||
}
|
||||
|
||||
/// Tokenize and lower one request without submitting it to an engine.
|
||||
pub fn prepare(&self, mut request: TextRequest) -> Result<PreparedTextRequest> {
|
||||
request.validate()?;
|
||||
@@ -87,12 +110,11 @@ impl TextRequestProcessor {
|
||||
}
|
||||
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let prompt_token_ids = match take(&mut request.prompt) {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
|
||||
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
|
||||
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
let prompt_token_ids = Self::tokenize_prompt(
|
||||
&tokenizer,
|
||||
take(&mut request.prompt),
|
||||
request.add_special_tokens,
|
||||
)?;
|
||||
let sampling_hints = self.backend.sampling_hints()?;
|
||||
let sampling_limits = SamplingLimits {
|
||||
max_model_len: self.max_model_len,
|
||||
@@ -153,6 +175,11 @@ impl TextLlm {
|
||||
self.llm.engine_core_client()
|
||||
}
|
||||
|
||||
/// Return the text request processor.
|
||||
pub fn request_processor(&self) -> &TextRequestProcessor {
|
||||
&self.processor
|
||||
}
|
||||
|
||||
/// Return the tokenizer used by this text backend.
|
||||
pub fn tokenizer(&self) -> DynTokenizer {
|
||||
self.processor.tokenizer()
|
||||
|
||||
Reference in New Issue
Block a user