[Rust Frontend] Support --generation-config vllm (#53044)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -60,6 +60,7 @@ async fn main() -> Result<()> {
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
generation_config: Default::default(),
|
||||
served_model_name: vec![],
|
||||
listener_mode: HttpListenerMode::BindTcp {
|
||||
host: "127.0.0.1".to_string(),
|
||||
|
||||
@@ -11,7 +11,9 @@ use educe::Educe;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
|
||||
/// Default keep-alive idle timeout (seconds); also the head-read bound
|
||||
@@ -165,6 +167,8 @@ pub struct Config {
|
||||
pub coordinator_mode: CoordinatorMode,
|
||||
/// Backend model identifier used for engine-core loading.
|
||||
pub model: String,
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
|
||||
/// the first entry is used as the primary ID in responses and all entries
|
||||
/// are accepted in requests. When empty, falls back to `model`.
|
||||
|
||||
@@ -45,7 +45,9 @@ use tonic_health::server::health_reporter;
|
||||
use tower::ServiceExt as _;
|
||||
use tracing::{info, trace, warn};
|
||||
use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
|
||||
pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
pub use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::TextLlm;
|
||||
@@ -96,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
generation_config: config.generation_config,
|
||||
renderer: config.renderer,
|
||||
language_model_only: config.language_model_only,
|
||||
chat_template: config.chat_template.clone(),
|
||||
|
||||
@@ -56,6 +56,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
generation_config: Default::default(),
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
chat_template: config.chat_template.clone(),
|
||||
|
||||
Reference in New Issue
Block a user