[Rust Frontend] Support --generation-config vllm (#53044)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -129,8 +129,11 @@ pub(super) async fn load_model_backends(
|
||||
options: LoadModelBackendsOptions,
|
||||
) -> Result<LoadedModelBackends> {
|
||||
let files = ResolvedModelFiles::new(model_id).await?;
|
||||
let text_backend =
|
||||
HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?;
|
||||
let text_backend = HfTextBackend::from_resolved_model_files(
|
||||
files.clone(),
|
||||
model_id.to_string(),
|
||||
options.generation_config,
|
||||
)?;
|
||||
let tokenizer = text_backend.tokenizer();
|
||||
let text_backend: DynTextBackend = Arc::new(text_backend);
|
||||
|
||||
@@ -227,6 +230,7 @@ mod tests {
|
||||
resolved_files(config_json, tokenizer_config_json),
|
||||
"test-model".to_string(),
|
||||
LoadModelBackendsOptions {
|
||||
generation_config: Default::default(),
|
||||
renderer,
|
||||
language_model_only: false,
|
||||
chat_template_content_format: Default::default(),
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use vllm_text::{DynTextBackend, TextBackend};
|
||||
use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
|
||||
@@ -61,6 +61,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
|
||||
/// Frontend-side chat backend loading options.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct LoadModelBackendsOptions {
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Which chat renderer implementation to use.
|
||||
pub renderer: RendererSelection,
|
||||
/// Disable frontend-side multimodal preprocessing and render the model as
|
||||
|
||||
@@ -44,6 +44,7 @@ pub use request::{
|
||||
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
|
||||
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
pub use vllm_llm::FinishReason;
|
||||
pub use vllm_text::GenerationConfigMode;
|
||||
|
||||
mod backend;
|
||||
mod error;
|
||||
|
||||
Reference in New Issue
Block a user