[Rust Frontend] Deduplicate request preprocessing for /tokenize (#50448)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Sage
2026-08-05 08:27:35 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent fd859d2e06
commit 5bf32605a9
5 changed files with 170 additions and 50 deletions
+25 -33
View File
@@ -155,17 +155,9 @@ impl ChatRequestProcessor {
Ok(Some(features))
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
mut request: ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
// Stamp before rendering so render and tokenize count toward TTFT/e2e.
let arrival_time = vllm_llm::current_unix_timestamp_secs();
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
let rendered = self.backend.chat_renderer().render(&request)?;
let reasoning_parser_kwargs =
request
@@ -176,7 +168,7 @@ impl ChatRequestProcessor {
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
});
let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
let text_request = TextRequest {
Ok(TextRequest {
request_id: request.request_id,
prompt,
mm_features,
@@ -191,7 +183,24 @@ impl ChatRequestProcessor {
reasoning_parser_kwargs,
lora_request: request.lora_request,
arrival_time: Some(arrival_time),
};
})
}
/// Prepare one chat request for tokenization without constructing an output processor.
pub async fn prepare_for_tokenization(&self, request: ChatRequest) -> Result<TextRequest> {
request.validate()?;
self.prepare_text_request(request).await
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
mut request: ChatRequest,
options: NewChatOutputProcessorOptions<'_>,
) -> Result<(TextRequest, DynChatOutputProcessor)> {
request.validate()?;
let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
let text_request = self.prepare_text_request(request).await?;
Ok((text_request, output_processor))
}
}
@@ -265,6 +274,11 @@ impl ChatLlm {
&self.text
}
/// Return the chat request processor.
pub fn request_processor(&self) -> &ChatRequestProcessor {
&self.processor
}
/// Return the model ID reported by the underlying text backend.
pub fn model_id(&self) -> &str {
self.text.model_id()
@@ -332,28 +346,6 @@ impl ChatLlm {
Ok(ChatEventStream::new(request_id, structured_stream))
}
/// Render through the chat template and tokenize, without submitting to the engine.
///
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
/// generation would see. Used by `POST /tokenize` (chat form).
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
request.validate()?;
let rendered = self.processor.backend.chat_renderer().render(&request)?;
let (prompt, _mm_features) =
self.processor.finalize_rendered_prompt(&request, rendered).await?;
let tokenizer = self.text.tokenizer();
let token_ids = match prompt {
// Rendered string from the template (usual chat path).
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
// Already tokenized (e.g. multimodal path); pass through unchanged.
Prompt::TokenIds(ids) => ids,
};
Ok(token_ids)
}
/// Abort in-flight requests by their external (user-supplied) request ids.
pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
self.text.abort(external_ids).await?;