[Rust Frontend] Support --generation-config vllm (#53044)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 17:31:25 -07:00
committed by GitHub
parent 0a5a55136f
commit f32b17b6d6
14 changed files with 244 additions and 41 deletions
+6 -2
View File
@@ -129,8 +129,11 @@ pub(super) async fn load_model_backends(
options: LoadModelBackendsOptions,
) -> Result<LoadedModelBackends> {
let files = ResolvedModelFiles::new(model_id).await?;
let text_backend =
HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?;
let text_backend = HfTextBackend::from_resolved_model_files(
files.clone(),
model_id.to_string(),
options.generation_config,
)?;
let tokenizer = text_backend.tokenizer();
let text_backend: DynTextBackend = Arc::new(text_backend);
@@ -227,6 +230,7 @@ mod tests {
resolved_files(config_json, tokenizer_config_json),
"test-model".to_string(),
LoadModelBackendsOptions {
generation_config: Default::default(),
renderer,
language_model_only: false,
chat_template_content_format: Default::default(),
+3 -1
View File
@@ -5,7 +5,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use serde_json::Value;
use vllm_text::{DynTextBackend, TextBackend};
use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};
use crate::error::Result;
use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
@@ -61,6 +61,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
/// Frontend-side chat backend loading options.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct LoadModelBackendsOptions {
/// Which generation-config sampling defaults to inherit.
pub generation_config: GenerationConfigMode,
/// Which chat renderer implementation to use.
pub renderer: RendererSelection,
/// Disable frontend-side multimodal preprocessing and render the model as
+1
View File
@@ -44,6 +44,7 @@ pub use request::{
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
pub use vllm_text::GenerationConfigMode;
mod backend;
mod error;