From f32b17b6d616605eb5ed498777f150242bd1bc30 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 21 Aug 2026 10:31:25 +1000 Subject: [PATCH] [Rust Frontend] Support `--generation-config vllm` (#53044) Signed-off-by: Bugen Zhao --- rust/src/chat/src/backend/hf.rs | 8 +- rust/src/chat/src/backend/mod.rs | 4 +- rust/src/chat/src/lib.rs | 1 + rust/src/cmd/src/cli.rs | 10 +- rust/src/cmd/src/cli/tests.rs | 32 +++- rust/src/cmd/src/cli/unsupported.rs | 13 +- .../examples/external_engine_openai_qwen.rs | 1 + rust/src/server/src/config.rs | 6 +- rust/src/server/src/lib.rs | 5 +- rust/src/server/src/render.rs | 1 + rust/src/text/src/backend/hf/mod.rs | 141 +++++++++++++++--- rust/src/text/src/backend/mod.rs | 44 ++++++ rust/src/text/src/lib.rs | 4 +- rust/src/text/src/lower.rs | 15 +- 14 files changed, 244 insertions(+), 41 deletions(-) diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 20d2b81cc1..0be33f91a8 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -129,8 +129,11 @@ pub(super) async fn load_model_backends( options: LoadModelBackendsOptions, ) -> Result { let files = ResolvedModelFiles::new(model_id).await?; - let text_backend = - HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?; + let text_backend = HfTextBackend::from_resolved_model_files( + files.clone(), + model_id.to_string(), + options.generation_config, + )?; let tokenizer = text_backend.tokenizer(); let text_backend: DynTextBackend = Arc::new(text_backend); @@ -227,6 +230,7 @@ mod tests { resolved_files(config_json, tokenizer_config_json), "test-model".to_string(), LoadModelBackendsOptions { + generation_config: Default::default(), renderer, language_model_only: false, chat_template_content_format: Default::default(), diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index 0a0f757d65..77b9114a58 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::sync::Arc; use serde_json::Value; -use vllm_text::{DynTextBackend, TextBackend}; +use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend}; use crate::error::Result; use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo}; @@ -61,6 +61,8 @@ pub type DynChatTextBackend = Arc; /// Frontend-side chat backend loading options. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct LoadModelBackendsOptions { + /// Which generation-config sampling defaults to inherit. + pub generation_config: GenerationConfigMode, /// Which chat renderer implementation to use. pub renderer: RendererSelection, /// Disable frontend-side multimodal preprocessing and render the model as diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index cf22722dd7..c6b91679e0 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -44,6 +44,7 @@ pub use request::{ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage}; pub use vllm_engine_core_client::protocol::multimodal::MmFeatures; pub use vllm_llm::FinishReason; +pub use vllm_text::GenerationConfigMode; mod backend; mod error; diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 64b6dc9800..a061c8eff0 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -22,8 +22,8 @@ use serde_json::Value; use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; -use vllm_chat::ReasoningParserFactory; use vllm_chat::multimodal::MmLimitPerPrompt; +use vllm_chat::{GenerationConfigMode, ReasoningParserFactory}; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args}; @@ -186,6 +186,12 @@ pub struct SharedRuntimeArgs { /// public model ID. pub model: String, + /// The source of generation-config sampling defaults. `"auto"` loads the + /// model's defaults, while `"vllm"` uses vLLM's neutral defaults. + #[arg(long, default_value_t)] + #[serde(default)] + pub generation_config: GenerationConfigMode, + /// Maximum time to wait for the expected engines to register on the /// frontend transport. #[arg( @@ -482,6 +488,7 @@ impl SharedRuntimeArgs { None => CoordinatorMode::None, }, model: self.model, + generation_config: self.generation_config, served_model_name: self.served_model_name, listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd }, tool_call_parser: self.tool_call_parser, @@ -535,6 +542,7 @@ impl SharedRuntimeArgs { }, coordinator_mode: CoordinatorMode::MaybeInProc, model: self.model, + generation_config: self.generation_config, served_model_name: self.served_model_name, listener_mode, tool_call_parser: self.tool_call_parser, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 80436ec423..89979ec6d0 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -3,7 +3,9 @@ use expect_test::expect; use vllm_engine_core_client::TransportMode; -use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection}; +use vllm_server::{ + Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection, +}; use super::{BenchCommand, Cli, Command}; @@ -88,6 +90,7 @@ fn serve_args_forward_python_flags_with_separator() { uds: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", + generation_config: Auto, engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, @@ -732,6 +735,24 @@ fn serve_args_reject_unsupported_flag_arg() { "#]].assert_eq(&error.to_string()); } +#[test] +fn serve_args_reject_custom_generation_config_source() { + let error = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--generation-config", + "/tmp/custom-config", + ]) + .unwrap_err(); + + expect![[r#" + error: invalid value '/tmp/custom-config' for '--generation-config ': generation config source `/tmp/custom-config` is not implemented yet (expected one of: auto, vllm) + + For more information, try '--help'. + "#]].assert_eq(&error.to_string()); +} + #[test] fn serve_args_reject_unsupported_no_flag_alias() { let error = Cli::try_parse_from([ @@ -788,6 +809,7 @@ fn frontend_args_accept_json() { data_parallel_size: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", + generation_config: Auto, engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, @@ -858,6 +880,7 @@ fn frontend_args_json_applies_defaults() { panic!("expected frontend args"); }; assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B"); + assert_eq!(args.runtime.generation_config, GenerationConfigMode::Auto); assert_eq!(args.runtime.engine_ready_timeout_secs, 600); assert_eq!(args.runtime.tool_call_parser, ParserSelection::None); assert_eq!(args.runtime.reasoning_parser, ParserSelection::None); @@ -901,7 +924,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() { "--output-address", "ipc:///tmp/output.sock", "--args-json", - r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#, + r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#, ]) .unwrap(); @@ -909,6 +932,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() { panic!("expected frontend args"); }; assert_eq!(args.runtime.engine_ready_timeout_secs, 42); + assert_eq!(args.runtime.generation_config, GenerationConfigMode::Vllm); assert_eq!( args.runtime.tool_call_parser, ParserSelection::Explicit("hermes".to_string()) @@ -1379,6 +1403,7 @@ fn serve_args_accept_handshake_aliases() { uds: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", + generation_config: Auto, engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, @@ -1523,6 +1548,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { }, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", + generation_config: Auto, served_model_name: [], listener_mode: BindTcp { host: "127.0.0.1", @@ -1608,6 +1634,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { }, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", + generation_config: Auto, served_model_name: [], listener_mode: BindTcp { host: "127.0.0.1", @@ -1715,6 +1742,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present address: "tcp://127.0.0.1:7000", }, model: "Qwen/Qwen3-0.6B", + generation_config: Auto, served_model_name: [], listener_mode: InheritedFd { fd: 3, diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index a48e61dbf7..d2372ed566 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -240,15 +240,12 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub hf_overrides: Option, - /// The folder path to the generation config. Defaults to `"auto"`, the - /// generation config will be loaded from model path. If set to `"vllm"`, no - /// generation config is loaded, vLLM defaults will be used. If set to a - /// folder path, the generation config will be loaded from the specified - /// folder path. If `max_new_tokens` is specified in generation config, - /// then it sets a server-wide limit on the number of output tokens for - /// all requests. + /// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If + /// used with `--generation-config auto`, the override parameters will be + /// merged with the default config from the model. If used with + /// `--generation-config vllm`, only the override parameters are used. #[arg(long)] - pub generation_config: Option, + pub override_generation_config: Option, /// IOProcessor plugin name to load at model startup #[arg(long)] diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 3877762630..5c99fc92eb 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -60,6 +60,7 @@ async fn main() -> Result<()> { }, coordinator_mode: CoordinatorMode::MaybeInProc, model: args.model, + generation_config: Default::default(), served_model_name: vec![], listener_mode: HttpListenerMode::BindTcp { host: "127.0.0.1".to_string(), diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index a33d7e25fe..eb690d4ea0 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -11,7 +11,9 @@ use educe::Educe; use serde::Serialize; use serde_json::Value; use vllm_chat::multimodal::MmLimitPerPrompt; -use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; +use vllm_chat::{ + ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection, +}; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; /// Default keep-alive idle timeout (seconds); also the head-read bound @@ -165,6 +167,8 @@ pub struct Config { pub coordinator_mode: CoordinatorMode, /// Backend model identifier used for engine-core loading. pub model: String, + /// Which generation-config sampling defaults to inherit. + pub generation_config: GenerationConfigMode, /// Model name(s) exposed to clients via the OpenAI API. When non-empty, /// the first entry is used as the primary ID in responses and all entries /// are accepted in requests. When empty, falls back to `model`. diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index f9f5c42350..e8138b2494 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -45,7 +45,9 @@ use tonic_health::server::health_reporter; use tower::ServiceExt as _; use tracing::{info, trace, warn}; use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends}; -pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; +pub use vllm_chat::{ + ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection, +}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig}; use vllm_llm::Llm; use vllm_text::TextLlm; @@ -96,6 +98,7 @@ async fn build_state(config: &Config) -> Result> { let loaded = load_model_backends( &config.model, LoadModelBackendsOptions { + generation_config: config.generation_config, renderer: config.renderer, language_model_only: config.language_model_only, chat_template: config.chat_template.clone(), diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index ee8098f8e9..35c821bc6e 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -56,6 +56,7 @@ async fn build_state(config: &RenderConfig) -> Result> { let loaded = load_model_backends( &config.model, LoadModelBackendsOptions { + generation_config: Default::default(), renderer: config.renderer, language_model_only: true, chat_template: config.chat_template.clone(), diff --git a/rust/src/text/src/backend/hf/mod.rs b/rust/src/text/src/backend/hf/mod.rs index 4fc7a18753..ae768767d9 100644 --- a/rust/src/text/src/backend/hf/mod.rs +++ b/rust/src/text/src/backend/hf/mod.rs @@ -18,7 +18,7 @@ pub use self::config::{ load_tokenizer_config, }; pub use self::model_files::{ResolvedModelFiles, TokenizerSource}; -use crate::backend::{SamplingHints, TextBackend}; +use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend}; use crate::error::Result; fn load_tokenizer(tokenizer: &TokenizerSource) -> Result { @@ -41,6 +41,7 @@ pub struct HfTextBackend { /// Generation-config for sampling defaults that may be inherited when the /// user does not explicitly override them. generation_config: GenerationConfig, + generation_config_mode: GenerationConfigMode, /// Model vocabulary size from the selected text config. model_vocab_size: usize, /// Model config (`config.json`). @@ -48,14 +49,12 @@ pub struct HfTextBackend { } impl HfTextBackend { - /// Load the text backend with the given model id. - pub async fn from_model(model_id: &str) -> Result { - let files = ResolvedModelFiles::new(model_id).await?; - Self::from_resolved_model_files(files, model_id.to_string()) - } - /// Load the text backend from resolved Hugging Face model files. - pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result { + pub fn from_resolved_model_files( + files: ResolvedModelFiles, + model_id: String, + generation_config_mode: GenerationConfigMode, + ) -> Result { let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?; let tokenizer = load_tokenizer(&files.tokenizer)?; let model_config = load_model_config(files.config_path.as_deref())?; @@ -80,6 +79,7 @@ impl HfTextBackend { primary_eos_token_id, extra_eos_token_ids, generation_config, + generation_config_mode, model_vocab_size, model_config, }) @@ -146,16 +146,38 @@ impl TextBackend for HfTextBackend { } fn sampling_hints(&self) -> Result { - Ok(SamplingHints { - primary_eos_token_id: self.primary_eos_token_id, - extra_eos_token_ids: self.extra_eos_token_ids.clone(), - default_temperature: self.generation_config.temperature, - default_top_p: self.generation_config.top_p, - default_top_k: self.generation_config.top_k, - default_min_p: self.generation_config.min_p, - default_repetition_penalty: self.generation_config.repetition_penalty, - default_max_tokens: self.generation_config.max_new_tokens, - }) + Ok(build_sampling_hints( + self.generation_config_mode, + &self.generation_config, + self.primary_eos_token_id, + self.extra_eos_token_ids.clone(), + )) + } +} + +fn build_sampling_hints( + mode: GenerationConfigMode, + generation_config: &GenerationConfig, + primary_eos_token_id: Option, + extra_eos_token_ids: BTreeSet, +) -> SamplingHints { + let sampling_config = match mode { + // Auto inherits sampling defaults from the model's generation config. + GenerationConfigMode::Auto => Some(generation_config), + // Vllm skips model-provided sampling defaults. `lower_sampling_params` + // applies vLLM fallback values; separately resolved EOS tokens remain. + GenerationConfigMode::Vllm => None, + }; + + SamplingHints { + primary_eos_token_id, + extra_eos_token_ids, + default_temperature: sampling_config.and_then(|config| config.temperature), + default_top_p: sampling_config.and_then(|config| config.top_p), + default_top_k: sampling_config.and_then(|config| config.top_k), + default_min_p: sampling_config.and_then(|config| config.min_p), + default_repetition_penalty: sampling_config.and_then(|config| config.repetition_penalty), + default_max_tokens: sampling_config.and_then(|config| config.max_new_tokens), } } @@ -163,7 +185,10 @@ impl TextBackend for HfTextBackend { mod tests { use std::collections::BTreeSet; - use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids}; + use super::{ + GenerationConfig, GenerationConfigMode, HfTokenizerConfig, ModelConfig, + build_sampling_hints, resolve_eos_token_ids, + }; use vllm_tokenizer::Tokenizer; struct FakeTokenizer; @@ -235,4 +260,82 @@ mod tests { assert_eq!(primary, Some(2)); assert_eq!(extra, BTreeSet::from([200006, 200010])); } + + #[test] + fn vllm_generation_config_keeps_eos_and_uses_neutral_sampling_defaults() { + let generation_config: GenerationConfig = serde_json::from_str( + r#"{ + "eos_token_id": [2, 3], + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.1, + "repetition_penalty": 1.1, + "max_new_tokens": 4096 + }"#, + ) + .unwrap(); + let extra_eos_token_ids = BTreeSet::from([3]); + + let hints = ( + build_sampling_hints( + GenerationConfigMode::Auto, + &generation_config, + Some(2), + extra_eos_token_ids.clone(), + ), + build_sampling_hints( + GenerationConfigMode::Vllm, + &generation_config, + Some(2), + extra_eos_token_ids, + ), + ); + + expect_test::expect![[r#" + ( + SamplingHints { + primary_eos_token_id: Some( + 2, + ), + extra_eos_token_ids: { + 3, + }, + default_temperature: Some( + 0.6, + ), + default_top_p: Some( + 0.95, + ), + default_top_k: Some( + 20, + ), + default_min_p: Some( + 0.1, + ), + default_repetition_penalty: Some( + 1.1, + ), + default_max_tokens: Some( + 4096, + ), + }, + SamplingHints { + primary_eos_token_id: Some( + 2, + ), + extra_eos_token_ids: { + 3, + }, + default_temperature: None, + default_top_p: None, + default_top_k: None, + default_min_p: None, + default_repetition_penalty: None, + default_max_tokens: None, + }, + ) + "#]] + .assert_debug_eq(&hints); + } } diff --git a/rust/src/text/src/backend/mod.rs b/rust/src/text/src/backend/mod.rs index 98b709f118..3bda9212a8 100644 --- a/rust/src/text/src/backend/mod.rs +++ b/rust/src/text/src/backend/mod.rs @@ -3,12 +3,56 @@ pub mod hf; +use std::fmt; +use std::str::FromStr; use std::sync::Arc; +use serde_with::{DeserializeFromStr, SerializeDisplay}; use vllm_tokenizer::DynTokenizer; use crate::error::Result; +/// Select which sampling defaults to inherit from generation config. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] +pub enum GenerationConfigMode { + /// Inherit sampling defaults from the model's generation config. + #[default] + Auto, + /// Use vLLM's neutral sampling defaults. + Vllm, +} + +impl GenerationConfigMode { + pub const AUTO_LITERAL: &str = "auto"; + pub const VLLM_LITERAL: &str = "vllm"; +} + +impl FromStr for GenerationConfigMode { + type Err = String; + + fn from_str(value: &str) -> std::result::Result { + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) { + Ok(Self::Auto) + } else if value.eq_ignore_ascii_case(Self::VLLM_LITERAL) { + Ok(Self::Vllm) + } else { + Err(format!( + "generation config source `{value}` is not implemented yet \ + (expected one of: auto, vllm)" + )) + } + } +} + +impl fmt::Display for GenerationConfigMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Auto => f.write_str(Self::AUTO_LITERAL), + Self::Vllm => f.write_str(Self::VLLM_LITERAL), + } + } +} + /// Tokenizer/model-derived defaults used to enrich text-generation requests /// before they are lowered into engine-core. #[derive(Debug, Clone, Default, PartialEq)] diff --git a/rust/src/text/src/lib.rs b/rust/src/text/src/lib.rs index 53edf7e309..f49038b27f 100644 --- a/rust/src/text/src/lib.rs +++ b/rust/src/text/src/lib.rs @@ -9,7 +9,9 @@ use std::mem::take; -pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend}; +pub use backend::{ + DynTextBackend, GenerationConfigMode, SamplingHints, SamplingLimits, TextBackend, +}; pub use error::{Error, LogprobsError, Result, SamplingParamsError, TokenIdsError}; use futures::Stream; pub use lower::{ diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 55be99cc4e..0e1fbd26bc 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -322,8 +322,8 @@ mod tests { use vllm_tokenizer::test_utils::TestTokenizer; use super::*; - use crate::backend::hf::HfTextBackend; - use crate::backend::{SamplingHints, TextBackend as _}; + use crate::backend::hf::{HfTextBackend, ResolvedModelFiles}; + use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend as _}; use crate::error::{LogprobsError, SamplingParamsError, TokenIdsError}; use crate::request::{Prompt, TextRequest}; @@ -785,9 +785,14 @@ mod tests { #[tokio::test] #[file_serial(hf_qwen3)] async fn lower_text_request_uses_real_qwen_generation_defaults() { - let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B") - .await - .expect("load qwen tokenizer and generation config"); + let model_id = "Qwen/Qwen3-0.6B"; + let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files"); + let backend = HfTextBackend::from_resolved_model_files( + files, + model_id.to_string(), + GenerationConfigMode::Auto, + ) + .expect("load qwen tokenizer and generation config"); let hints = backend.sampling_hints().expect("collect sampling hints"); expect_test::expect![[r#"