vLLM/rust/src/chat/src/lib.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

//! Minimal chat facade above [`vllm_text`].
//!
//! This crate keeps the northbound boundary intentionally small:
//! `messages -> rendered prompt -> tokenized prompt -> engine request ->
//! streamed structured assistant events`. The request side remains text-first,
//! while the response side can emit structured reasoning and final-answer
//! blocks. It is closer to vLLM's internal chat-rendering flow than to a full
//! OpenAI-compatible surface.

pub use backend::hf::HfChatBackend;
pub use backend::{
    ChatBackend, ChatTextBackend, DynChatBackend, DynChatTextBackend, LoadModelBackendsOptions,
    LoadedModelBackends, NewChatOutputProcessorOptions, load_model_backends,
};
pub use error::{Error, Result};
pub use event::{
    AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantMessageExt,
    AssistantToolCall, ChatEvent, ChatTokenUsage,
};
use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart;
pub use output::{
    ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
    HarmonyChatOutputProcessor,
};
pub use parser::reasoning::{
    ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
};
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use parser::{ParserSelection, validate_parser_overrides};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
    ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
    DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, MediaPartSource,
    RenderedPrompt, RendererSelection,
};
pub use request::{
    ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
    ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams,
};
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
pub use vllm_text::GenerationConfigMode;

mod backend;
mod error;
mod event;
pub mod multimodal;
mod output;
mod parser;
mod renderer;
mod request;
mod stream;

use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};

/// Chat request preparation shared by inference and render-only frontends.
pub struct ChatRequestProcessor {
    backend: DynChatBackend,
    /// Effective model dtype reported by the engine.
    /// Absent for text-only frontends without an engine handshake.
    model_dtype: Option<ModelDtype>,
    /// Tool-call parser selection used when preparing generation requests.
    tool_call_parser: ParserSelection,
    /// Reasoning parser selection used when preparing generation requests.
    reasoning_parser: ParserSelection,
}

impl ChatRequestProcessor {
    /// Create a processor with multimodal support using the effective model
    /// dtype reported by the engine.
    fn new(backend: DynChatBackend, model_dtype: ModelDtype) -> Self {
        Self {
            backend,
            model_dtype: Some(model_dtype),
            tool_call_parser: ParserSelection::Auto,
            reasoning_parser: ParserSelection::Auto,
        }
    }

    /// Create a render-only processor that rejects multimodal requests.
    pub fn render_only(backend: DynChatBackend) -> Self {
        Self {
            backend,
            model_dtype: None,
            tool_call_parser: ParserSelection::Auto,
            reasoning_parser: ParserSelection::Auto,
        }
    }

    /// Configure the parser selections used to prepare generation requests.
    pub fn with_parser_selections(
        mut self,
        tool_call_parser: ParserSelection,
        reasoning_parser: ParserSelection,
    ) -> Self {
        self.tool_call_parser = tool_call_parser;
        self.reasoning_parser = reasoning_parser;
        self
    }

    async fn finalize_rendered_prompt(
        &self,
        request: &ChatRequest,
        rendered: RenderedPrompt,
    ) -> Result<(Prompt, Option<MmFeatures>)> {
        let media_is_empty = match &rendered.media_order {
            Some(media_order) => media_order.is_empty(),
            None => !request.has_multimodal(),
        };
        match self.model_dtype {
            Some(model_dtype) => {
                multimodal::finalize_rendered_prompt(
                    request,
                    rendered,
                    self.backend.multimodal_model_info(),
                    model_dtype,
                )
                .await
            }
            None if media_is_empty => Ok((rendered.prompt, None)),
            None => Err(Error::UnsupportedMultimodalRenderer),
        }
    }

    /// Prepare media for an already-tokenized request.
    async fn prepare_media(
        &self,
        media: multimodal::MultimodalInput,
        token_ids: &mut Vec<u32>,
    ) -> Result<Option<MmFeatures>> {
        if media.is_empty() {
            return Ok(None);
        }
        let info = self
            .backend
            .multimodal_model_info()
            .ok_or(Error::UnsupportedMultimodalRenderer)?;
        let features = match media {
            multimodal::MultimodalInput::Raw(parts) => {
                let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
                info.prepare_multimodal(parts, token_ids, model_dtype).await?
            }
            multimodal::MultimodalInput::Preprocessed(features) => {
                info.prepare_preprocessed(features, token_ids.len())?
            }
        };
        Ok(Some(features))
    }

    async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
        // Stamp before rendering so render and tokenize count toward TTFT/e2e.
        let arrival_time = vllm_llm::current_unix_timestamp_secs();
        let rendered = self.backend.chat_renderer().render(&request)?;
        let reasoning_parser_kwargs =
            request
                .sampling_params
                .structured_outputs
                .is_some()
                .then(|| ReasoningParserKwargs {
                    chat_template_kwargs: rendered.effective_template_kwargs.clone(),
                });
        let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
        Ok(TextRequest {
            request_id: request.request_id,
            prompt,
            mm_features,
            sampling_params: request.sampling_params,
            decode_options: request.decode_options,
            intermediate: request.intermediate,
            prompt_truncation: request.prompt_truncation,
            priority: request.priority,
            cache_salt: request.cache_salt,
            add_special_tokens: request.add_special_tokens,
            data_parallel_rank: request.data_parallel_rank,
            session_id: request.session_id,
            reasoning_parser_kwargs,
            lora_request: request.lora_request,
            arrival_time: Some(arrival_time),
        })
    }

    /// Prepare one chat request for tokenization without constructing an output processor.
    pub async fn prepare_for_tokenization(&self, request: ChatRequest) -> Result<TextRequest> {
        request.validate()?;
        self.prepare_text_request(request).await
    }

    /// Prepare one chat request without submitting it to an engine.
    pub async fn prepare(
        &self,
        mut request: ChatRequest,
    ) -> Result<(TextRequest, DynChatOutputProcessor)> {
        request.validate()?;
        let output_processor = self.backend.new_chat_output_processor(
            &mut request,
            NewChatOutputProcessorOptions {
                tool_call_parser: &self.tool_call_parser,
                reasoning_parser: &self.reasoning_parser,
            },
        )?;
        let text_request = self.prepare_text_request(request).await?;
        Ok((text_request, output_processor))
    }
}

/// Structured chat facade above [`TextLlm`].
///
/// This layer stays above raw text semantics: it takes care of chat-template
/// rendering, exposes structured assistant events, and adds chat-specific
/// request semantics such as tool calls.
pub struct ChatLlm {
    text: TextLlm,
    processor: ChatRequestProcessor,
}

impl ChatLlm {
    /// Create a new chat facade from a text-generation facade plus a chat
    /// backend.
    pub fn new(text: TextLlm, backend: DynChatBackend) -> Self {
        let model_dtype = text.engine_core_client().model_dtype();

        Self {
            text,
            processor: ChatRequestProcessor::new(backend, model_dtype),
        }
    }

    /// Convenience constructor for one shared backend object that implements
    /// both text and chat responsibilities.
    pub fn from_shared_backend(llm: Llm, backend: DynChatTextBackend) -> Self {
        let text = TextLlm::new(llm, backend.clone());
        Self::new(text, backend)
    }

    /// Set tool-call parser selection.
    pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self {
        self.processor.tool_call_parser = selection;
        self
    }

    /// Set reasoning parser selection.
    pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self {
        self.processor.reasoning_parser = selection;
        self
    }

    /// Override the effective model dtype used for multimodal tensor encoding.
    pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
        self.processor.model_dtype = Some(model_dtype);
        self
    }

    /// Tokenizer vocabulary size.
    pub fn tokenizer_vocab_size(&self) -> usize {
        self.text.tokenizer_vocab_size()
    }

    /// Model vocabulary size from the model config.
    pub fn model_vocab_size(&self) -> usize {
        self.text.model_vocab_size()
    }

    /// Expose the underlying text facade for raw text-generation routes such as
    /// `/v1/completions`.
    pub fn text(&self) -> &TextLlm {
        &self.text
    }

    /// Return the chat request processor.
    pub fn request_processor(&self) -> &ChatRequestProcessor {
        &self.processor
    }

    /// Return the model ID reported by the underlying text backend.
    pub fn model_id(&self) -> &str {
        self.text.model_id()
    }

    /// Expose the underlying engine-core client for low-level utility/admin
    /// calls.
    pub fn engine_core_client(&self) -> &EngineCoreClient {
        self.text.engine_core_client()
    }

    /// Whether the loaded backend has a registered multimodal processor.
    pub fn supports_multimodal(&self) -> bool {
        self.processor.backend.multimodal_model_info().is_some()
    }

    /// Prepare media for an already-tokenized request.
    /// Raw content is preprocessed; inline features are checked against model
    /// capabilities, modality limits, and the final prompt positions.
    pub async fn prepare_media(
        &self,
        media: impl Into<multimodal::MultimodalInput>,
        token_ids: &mut Vec<u32>,
    ) -> Result<Option<MmFeatures>> {
        self.processor.prepare_media(media.into(), token_ids).await
    }

    /// Effective tool-call parser name for this model, if parsing is enabled.
    pub fn tool_call_parser_name(&self) -> Option<&str> {
        self.processor.tool_call_parser.resolve_tool_name(self.model_id())
    }

    /// Effective reasoning parser name for this model, if parsing is enabled.
    pub fn reasoning_parser_name(&self) -> Option<&str> {
        self.processor.reasoning_parser.resolve_reasoning_name(self.model_id())
    }

    /// Render, tokenize, and submit one chat request.
    pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
        let (text_request, output_processor) = self.processor.prepare(request).await?;
        let request_id = text_request.request_id.clone();
        let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();

        let structured_stream = output_processor.process(decoded_stream)?;

        Ok(ChatEventStream::new(request_id, structured_stream))
    }

    /// Abort in-flight requests by their external (user-supplied) request ids.
    pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
        self.text.abort(external_ids).await?;
        Ok(())
    }

    /// Shut down the underlying LLM client and its background tasks.
    pub async fn shutdown(self) -> Result<()> {
        self.text.shutdown().await?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use thiserror_ext::AsReport;

    use super::{ParserSelection, validate_parser_overrides};
    use crate::parser::reasoning::names;

    #[test]
    fn validate_parser_overrides_accepts_registered_names() {
        validate_parser_overrides(
            &ParserSelection::Explicit("llama3_json".to_string()),
            &ParserSelection::Explicit(names::QWEN3.to_string()),
        )
        .unwrap();
    }

    #[test]
    fn validate_parser_overrides_accepts_explicit_kimi_k3() {
        let selection = ParserSelection::Explicit("kimi_k3".to_string());
        validate_parser_overrides(&selection, &selection).unwrap();
    }

    #[test]
    fn validate_parser_overrides_accepts_auto_and_none() {
        validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
    }

    #[test]
    fn validate_parser_overrides_rejects_unknown_tool_parser() {
        let error = validate_parser_overrides(
            &ParserSelection::Explicit("definitely_missing_tool_parser".to_string()),
            &ParserSelection::Auto,
        )
        .unwrap_err();

        expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
    }

    #[test]
    fn validate_parser_overrides_rejects_unknown_reasoning_parser() {
        let error = validate_parser_overrides(
            &ParserSelection::Auto,
            &ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
        )
        .unwrap_err();

        expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
    }
}