diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index a0c81b29d6..e07fb1b657 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -155,17 +155,9 @@ impl ChatRequestProcessor { Ok(Some(features)) } - /// Prepare one chat request without submitting it to an engine. - pub async fn prepare( - &self, - mut request: ChatRequest, - options: NewChatOutputProcessorOptions<'_>, - ) -> Result<(TextRequest, DynChatOutputProcessor)> { - request.validate()?; - + async fn prepare_text_request(&self, request: ChatRequest) -> Result { // Stamp before rendering so render and tokenize count toward TTFT/e2e. let arrival_time = vllm_llm::current_unix_timestamp_secs(); - let output_processor = self.backend.new_chat_output_processor(&mut request, options)?; let rendered = self.backend.chat_renderer().render(&request)?; let reasoning_parser_kwargs = request @@ -176,7 +168,7 @@ impl ChatRequestProcessor { chat_template_kwargs: rendered.effective_template_kwargs.clone(), }); let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?; - let text_request = TextRequest { + Ok(TextRequest { request_id: request.request_id, prompt, mm_features, @@ -191,7 +183,24 @@ impl ChatRequestProcessor { reasoning_parser_kwargs, lora_request: request.lora_request, arrival_time: Some(arrival_time), - }; + }) + } + + /// Prepare one chat request for tokenization without constructing an output processor. + pub async fn prepare_for_tokenization(&self, request: ChatRequest) -> Result { + request.validate()?; + self.prepare_text_request(request).await + } + + /// Prepare one chat request without submitting it to an engine. + pub async fn prepare( + &self, + mut request: ChatRequest, + options: NewChatOutputProcessorOptions<'_>, + ) -> Result<(TextRequest, DynChatOutputProcessor)> { + request.validate()?; + let output_processor = self.backend.new_chat_output_processor(&mut request, options)?; + let text_request = self.prepare_text_request(request).await?; Ok((text_request, output_processor)) } } @@ -265,6 +274,11 @@ impl ChatLlm { &self.text } + /// Return the chat request processor. + pub fn request_processor(&self) -> &ChatRequestProcessor { + &self.processor + } + /// Return the model ID reported by the underlying text backend. pub fn model_id(&self) -> &str { self.text.model_id() @@ -332,28 +346,6 @@ impl ChatLlm { Ok(ChatEventStream::new(request_id, structured_stream)) } - /// Render through the chat template and tokenize, without submitting to the engine. - /// - /// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as - /// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what - /// generation would see. Used by `POST /tokenize` (chat form). - pub async fn tokenize_chat(&self, request: ChatRequest) -> Result> { - request.validate()?; - - let rendered = self.processor.backend.chat_renderer().render(&request)?; - let (prompt, _mm_features) = - self.processor.finalize_rendered_prompt(&request, rendered).await?; - - let tokenizer = self.text.tokenizer(); - let token_ids = match prompt { - // Rendered string from the template (usual chat path). - Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?, - // Already tokenized (e.g. multimodal path); pass through unchanged. - Prompt::TokenIds(ids) => ids, - }; - Ok(token_ids) - } - /// Abort in-flight requests by their external (user-supplied) request ids. pub async fn abort(&self, external_ids: &[String]) -> Result<()> { self.text.abort(external_ids).await?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index e4e7fed996..7480656b02 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -6069,6 +6069,75 @@ async fn tokenize_chat_includes_generation_prompt_in_token_count() { assert!(with_len > without_len); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn tokenize_chat_with_tools_does_not_require_output_parser() { + let mut app = test_app().await; + + let (status, json) = post_json( + &mut app, + "/tokenize", + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + }], + }), + ) + .await; + + assert_eq!(status, StatusCode::OK); + assert!(!json["tokens"].as_array().expect("tokens").is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn tokenize_allows_prompts_at_or_above_max_model_len() { + let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { + max_model_len: 4, + ..default_ready_response() + }; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + + let (completion_status, completion_json) = post_json( + &mut app, + "/tokenize", + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "add_special_tokens": false, + }), + ) + .await; + assert_eq!(completion_status, StatusCode::OK); + assert_eq!(completion_json["count"], 5); + assert_eq!(completion_json["max_model_len"], 4); + + let (chat_status, chat_json) = post_json( + &mut app, + "/tokenize", + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "add_generation_prompt": false, + "add_special_tokens": false, + }), + ) + .await; + assert_eq!(chat_status, StatusCode::OK); + assert!(chat_json["count"].as_u64().expect("count") > 4); + assert_eq!(chat_json["max_model_len"], 4); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn tokenize_chat_conflicting_generation_flags_returns_400() { diff --git a/rust/src/server/src/routes/tokenize.rs b/rust/src/server/src/routes/tokenize.rs index 0825e295d7..233a24c33e 100644 --- a/rust/src/server/src/routes/tokenize.rs +++ b/rust/src/server/src/routes/tokenize.rs @@ -60,9 +60,7 @@ pub async fn tokenize( let max_model_len = state.chat.engine_core_client().max_model_len(); let result = match body { - // Completion form: encode the raw `prompt` string (no chat template). - TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req), - // Chat form: render `messages` through the template, then encode (see `tokenize_chat`). + TokenizeRequest::Completion(req) => tokenize_completion(&state, &request_id, req), TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await, }; @@ -83,20 +81,24 @@ pub async fn tokenize( fn tokenize_completion( state: &AppState, - tokenizer: &vllm_text::tokenizer::DynTokenizer, + request_id: &str, req: TokenizeCompletionRequest, ) -> Result<(Vec, bool), ApiError> { check_model(state, req.model.as_deref())?; - let tokens = tokenizer - .encode(&req.prompt, req.add_special_tokens) + let return_token_strs = req.return_token_strs; + let tokens = state + .chat + .text() + .request_processor() + .tokenize(req.into_text_request(request_id.to_string())) .map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?; - Ok((tokens, req.return_token_strs)) + Ok((tokens, return_token_strs)) } /// HTTP adapter for the chat-shaped `/tokenize` body. /// -/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps -/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode. +/// Checks the model name, prepares without engine submission, and maps errors +/// to [`ApiError`]. async fn tokenize_chat( state: &AppState, request_id: &str, @@ -106,10 +108,17 @@ async fn tokenize_chat( let return_token_strs = req.return_token_strs; // `continue_final_message` / `add_generation_prompt` mutual exclusion is // enforced in `normalize_generation_prompt_mode` inside `into_chat_request`. + let text_request = state + .chat + .request_processor() + .prepare_for_tokenization(req.into_chat_request(request_id.to_string())?) + .await + .map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?; let tokens = state .chat - .tokenize_chat(req.into_chat_request(request_id.to_string())?) - .await + .text() + .request_processor() + .tokenize(text_request) .map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?; Ok((tokens, return_token_strs)) } diff --git a/rust/src/server/src/routes/tokenize/types.rs b/rust/src/server/src/routes/tokenize/types.rs index 39dbeb49a2..65b3e7a2b5 100644 --- a/rust/src/server/src/routes/tokenize/types.rs +++ b/rust/src/server/src/routes/tokenize/types.rs @@ -9,6 +9,7 @@ use serde_json::Value; use validator::{Validate, ValidationErrors}; use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams}; use vllm_text::output::TextDecodeOptions; +use vllm_text::{Prompt, TextRequest}; use crate::error::ApiError; use crate::routes::openai::chat_completions::convert::{ @@ -37,6 +38,28 @@ pub struct TokenizeCompletionRequest { pub return_token_strs: bool, } +impl TokenizeCompletionRequest { + /// Lower this tokenize body into a [`TextRequest`] for prompt tokenization. + pub fn into_text_request(self, request_id: String) -> TextRequest { + TextRequest { + request_id, + prompt: Prompt::Text(self.prompt), + mm_features: None, + sampling_params: SamplingParams::default(), + decode_options: TextDecodeOptions::default(), + intermediate: false, + priority: 0, + cache_salt: None, + add_special_tokens: self.add_special_tokens, + data_parallel_rank: None, + session_id: None, + reasoning_parser_kwargs: None, + lora_request: None, + arrival_time: None, + } + } +} + #[derive(Debug, Clone, Deserialize, Validate)] pub struct TokenizeChatRequest { pub model: Option, diff --git a/rust/src/text/src/lib.rs b/rust/src/text/src/lib.rs index 646e4bac9f..53edf7e309 100644 --- a/rust/src/text/src/lib.rs +++ b/rust/src/text/src/lib.rs @@ -78,6 +78,29 @@ impl TextRequestProcessor { self.max_model_len } + fn tokenize_prompt( + tokenizer: &DynTokenizer, + prompt: Prompt, + add_special_tokens: bool, + ) -> Result> { + match prompt { + Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens).map_err(Into::into), + // Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark + // and infra workloads bypass chat rendering and tokenizer overhead entirely. + Prompt::TokenIds(token_ids) => Ok(token_ids), + } + } + + /// Tokenize one request without generation-specific lowering. + pub fn tokenize(&self, request: TextRequest) -> Result> { + request.validate()?; + Self::tokenize_prompt( + &self.backend.tokenizer(), + request.prompt, + request.add_special_tokens, + ) + } + /// Tokenize and lower one request without submitting it to an engine. pub fn prepare(&self, mut request: TextRequest) -> Result { request.validate()?; @@ -87,12 +110,11 @@ impl TextRequestProcessor { } let tokenizer = self.backend.tokenizer(); - let prompt_token_ids = match take(&mut request.prompt) { - Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?, - // Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark - // and infra workloads bypass chat rendering and tokenizer overhead entirely. - Prompt::TokenIds(token_ids) => token_ids, - }; + let prompt_token_ids = Self::tokenize_prompt( + &tokenizer, + take(&mut request.prompt), + request.add_special_tokens, + )?; let sampling_hints = self.backend.sampling_hints()?; let sampling_limits = SamplingLimits { max_model_len: self.max_model_len, @@ -153,6 +175,11 @@ impl TextLlm { self.llm.engine_core_client() } + /// Return the text request processor. + pub fn request_processor(&self) -> &TextRequestProcessor { + &self.processor + } + /// Return the tokenizer used by this text backend. pub fn tokenizer(&self) -> DynTokenizer { self.processor.tokenizer()