[Rust Frontend] Add /tokenize and /detokenize endpoints (#44222)
Signed-off-by: Tan Ngoc Do <darkknightkhtn2008@gmail.com> Signed-off-by: TanNgocDo <darkknightkhtn2008@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Bugen Zhao
parent
80e2c4462d
commit
69fdaffbcd
@@ -66,6 +66,8 @@ pub enum Error {
|
||||
ToolCallStreamInvariant { message: String },
|
||||
#[error(transparent)]
|
||||
Text(#[from] vllm_text::Error),
|
||||
#[error(transparent)]
|
||||
Tokenizer(#[from] vllm_tokenizer::TokenizerError),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
@@ -52,7 +52,7 @@ mod stream;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{TextLlm, TextRequest};
|
||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||
|
||||
/// Validate explicit parser override names without starting request processing.
|
||||
pub fn validate_parser_overrides(
|
||||
@@ -198,6 +198,33 @@ impl ChatLlm {
|
||||
Ok(ChatEventStream::new(request.request_id, structured_stream))
|
||||
}
|
||||
|
||||
/// Render through the chat template and tokenize, without submitting to the engine.
|
||||
///
|
||||
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
|
||||
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
|
||||
/// generation would see. Used by `POST /tokenize` (chat form).
|
||||
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
|
||||
request.validate()?;
|
||||
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
let (prompt, _mm_features) = multimodal::finalize_rendered_prompt(
|
||||
&request,
|
||||
rendered,
|
||||
self.backend.multimodal_model_info(),
|
||||
self.model_dtype,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let tokenizer = self.text.tokenizer();
|
||||
let token_ids = match prompt {
|
||||
// Rendered string from the template (usual chat path).
|
||||
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
|
||||
// Already tokenized (e.g. multimodal path); pass through unchanged.
|
||||
Prompt::TokenIds(ids) => ids,
|
||||
};
|
||||
Ok(token_ids)
|
||||
}
|
||||
|
||||
/// Shut down the underlying LLM client and its background tasks.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
self.text.shutdown().await?;
|
||||
|
||||
Reference in New Issue
Block a user