[Rust Frontend] Pass effective reasoning_parser_kwargs for structured output (#46360)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4ed8eaafb0
commit
ce9f64020b
@@ -50,7 +50,7 @@ mod request;
|
||||
mod stream;
|
||||
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||
|
||||
@@ -179,6 +179,14 @@ impl ChatLlm {
|
||||
},
|
||||
)?;
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
let reasoning_parser_kwargs =
|
||||
request
|
||||
.sampling_params
|
||||
.structured_outputs
|
||||
.is_some()
|
||||
.then(|| ReasoningParserKwargs {
|
||||
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
|
||||
});
|
||||
|
||||
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
|
||||
&request,
|
||||
@@ -199,6 +207,7 @@ impl ChatLlm {
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
};
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
Reference in New Issue
Block a user