[Rust Frontend] Add /tokenize and /detokenize endpoints (#44222)

Signed-off-by: Tan Ngoc Do <darkknightkhtn2008@gmail.com>
Signed-off-by: TanNgocDo <darkknightkhtn2008@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
TanNgocDo
2026-06-09 05:11:37 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Bugen Zhao
parent 80e2c4462d
commit 69fdaffbcd
9 changed files with 836 additions and 39 deletions
+2
View File
@@ -66,6 +66,8 @@ pub enum Error {
ToolCallStreamInvariant { message: String },
#[error(transparent)]
Text(#[from] vllm_text::Error),
#[error(transparent)]
Tokenizer(#[from] vllm_tokenizer::TokenizerError),
}
pub type Result<T> = std::result::Result<T, Error>;
+28 -1
View File
@@ -52,7 +52,7 @@ mod stream;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::ModelDtype;
use vllm_llm::Llm;
use vllm_text::{TextLlm, TextRequest};
use vllm_text::{Prompt, TextLlm, TextRequest};
/// Validate explicit parser override names without starting request processing.
pub fn validate_parser_overrides(
@@ -198,6 +198,33 @@ impl ChatLlm {
Ok(ChatEventStream::new(request.request_id, structured_stream))
}
/// Render through the chat template and tokenize, without submitting to the engine.
///
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
/// generation would see. Used by `POST /tokenize` (chat form).
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
request.validate()?;
let rendered = self.backend.chat_renderer().render(&request)?;
let (prompt, _mm_features) = multimodal::finalize_rendered_prompt(
&request,
rendered,
self.backend.multimodal_model_info(),
self.model_dtype,
)
.await?;
let tokenizer = self.text.tokenizer();
let token_ids = match prompt {
// Rendered string from the template (usual chat path).
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
// Already tokenized (e.g. multimodal path); pass through unchanged.
Prompt::TokenIds(ids) => ids,
};
Ok(token_ids)
}
/// Shut down the underlying LLM client and its background tasks.
pub async fn shutdown(self) -> Result<()> {
self.text.shutdown().await?;