[Rust Frontend] Support --generation-config vllm (#53044)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 17:31:25 -07:00
committed by GitHub
parent 0a5a55136f
commit f32b17b6d6
14 changed files with 244 additions and 41 deletions
@@ -60,6 +60,7 @@ async fn main() -> Result<()> {
},
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
generation_config: Default::default(),
served_model_name: vec![],
listener_mode: HttpListenerMode::BindTcp {
host: "127.0.0.1".to_string(),
+5 -1
View File
@@ -11,7 +11,9 @@ use educe::Educe;
use serde::Serialize;
use serde_json::Value;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
/// Default keep-alive idle timeout (seconds); also the head-read bound
@@ -165,6 +167,8 @@ pub struct Config {
pub coordinator_mode: CoordinatorMode,
/// Backend model identifier used for engine-core loading.
pub model: String,
/// Which generation-config sampling defaults to inherit.
pub generation_config: GenerationConfigMode,
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
/// the first entry is used as the primary ID in responses and all entries
/// are accepted in requests. When empty, falls back to `model`.
+4 -1
View File
@@ -45,7 +45,9 @@ use tonic_health::server::health_reporter;
use tower::ServiceExt as _;
use tracing::{info, trace, warn};
use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
pub use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_llm::Llm;
use vllm_text::TextLlm;
@@ -96,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
generation_config: config.generation_config,
renderer: config.renderer,
language_model_only: config.language_model_only,
chat_template: config.chat_template.clone(),
+1
View File
@@ -56,6 +56,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
generation_config: Default::default(),
renderer: config.renderer,
language_model_only: true,
chat_template: config.chat_template.clone(),