[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,52 @@
|
||||
[package]
|
||||
name = "vllm-chat"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
easy-ext.workspace = true
|
||||
futures.workspace = true
|
||||
half.workspace = true
|
||||
itertools.workspace = true
|
||||
llm-multimodal.workspace = true
|
||||
minijinja.workspace = true
|
||||
minijinja-contrib.workspace = true
|
||||
openai-harmony.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde-json-fmt.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
subenum.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
trait-set.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-llm.workspace = true
|
||||
vllm-reasoning-parser.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
vllm-tool-parser.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
bytes.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serial_test = { workspace = true, features = ["file_locks"] }
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
zeromq.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,39 @@
|
||||
# Chat Smoke Test
|
||||
|
||||
Start a fresh headless `vllm` engine:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust chat smoke test through the `vllm-chat` interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-chat --example external_engine_chat_qwen -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1 \
|
||||
--prompt 'What is the capital of France? Answer with one word.'
|
||||
```
|
||||
|
||||
The example now defaults to `Qwen/Qwen3-0.6B`. The current `vllm-chat`
|
||||
request model stays text-first and supports either plain string content or
|
||||
OpenAI-style text blocks, while the output side now emits structured assistant
|
||||
events and automatically separates reasoning blocks for supported models. Tool
|
||||
use and multimodal inputs are still out of scope. It uses the Rust
|
||||
`tokenizers` library for the tokenizer itself, plus standard Hugging Face
|
||||
config files to load the chat template and EOS metadata.
|
||||
|
||||
IMPORTANT: Restart `vllm` each time you run the smoke test. The current headless
|
||||
engine cannot safely handle frontend reconnects after the client shuts down.
|
||||
@@ -0,0 +1,178 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatLlm, ChatMessage, ChatRequest,
|
||||
ChatRole, SamplingParams, load_model_backends,
|
||||
};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::TextLlm;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test the Rust chat facade against an external Qwen vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long)]
|
||||
prompt: String,
|
||||
}
|
||||
|
||||
const CLIENT_INDEX: u32 = 0;
|
||||
const OUTPUT_TIMEOUT_SECS: u64 = 120;
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
format!("rust-chat-smoke-{}", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let loaded = load_model_backends(&args.model, Default::default())
|
||||
.await
|
||||
.with_context(|| format!("failed to load backends for {}", args.model))?;
|
||||
let text_backend = loaded.text_backend;
|
||||
let chat_backend = loaded.chat_backend;
|
||||
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(OUTPUT_TIMEOUT_SECS);
|
||||
let request_id = unique_request_id();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: CLIENT_INDEX,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("tokenizer_source=tokenizers + hf-hub");
|
||||
println!("chat_template_source=tokenizer_config.json or adjacent chat template file");
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let llm = Llm::new(client);
|
||||
let chat = ChatLlm::new(TextLlm::new(llm, text_backend), chat_backend);
|
||||
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::text(ChatRole::User, args.prompt.clone())],
|
||||
sampling_params: SamplingParams {
|
||||
temperature: Some(0.0),
|
||||
..Default::default()
|
||||
},
|
||||
request_id: request_id.clone(),
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt={}", args.prompt);
|
||||
|
||||
let mut stream = chat.chat(request).await.context("failed to submit chat request")?;
|
||||
let output = tokio::time::timeout(output_timeout, async {
|
||||
let mut final_reasoning = String::new();
|
||||
let mut final_text = String::new();
|
||||
let mut final_output_token_count = 0usize;
|
||||
let mut finish_reason = None;
|
||||
let mut saw_start = false;
|
||||
let mut saw_stream_output = false;
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
ChatEvent::Start { .. } => {
|
||||
saw_start = true;
|
||||
}
|
||||
ChatEvent::BlockStart { kind, .. } => {
|
||||
if saw_stream_output {
|
||||
println!();
|
||||
}
|
||||
match kind {
|
||||
AssistantBlockKind::Reasoning => print!("[reasoning] "),
|
||||
AssistantBlockKind::Text => print!("[answer] "),
|
||||
AssistantBlockKind::ToolCall => {}
|
||||
}
|
||||
saw_stream_output = true;
|
||||
}
|
||||
ChatEvent::ToolCallStart { name, .. } => {
|
||||
if saw_stream_output {
|
||||
println!();
|
||||
}
|
||||
print!("[tool:{name}] ");
|
||||
saw_stream_output = true;
|
||||
}
|
||||
ChatEvent::LogprobsDelta { .. } => {}
|
||||
ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
finish_reason: reason,
|
||||
..
|
||||
} => {
|
||||
final_reasoning = message.reasoning().unwrap_or_default();
|
||||
final_text = message.text();
|
||||
final_output_token_count = output_token_count;
|
||||
finish_reason = Some(reason);
|
||||
break;
|
||||
}
|
||||
ChatEvent::BlockDelta { kind, delta, .. } => match kind {
|
||||
AssistantBlockKind::Reasoning | AssistantBlockKind::Text => {
|
||||
print!("{delta}");
|
||||
}
|
||||
AssistantBlockKind::ToolCall => {}
|
||||
},
|
||||
ChatEvent::ToolCallArgumentsDelta { delta, .. } => print!("{delta}"),
|
||||
ChatEvent::BlockEnd { .. } | ChatEvent::ToolCallEnd { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
if !saw_start {
|
||||
bail!("chat stream ended without a start event");
|
||||
}
|
||||
Ok::<_, anyhow::Error>((
|
||||
final_reasoning,
|
||||
final_text,
|
||||
final_output_token_count,
|
||||
finish_reason,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.context("timed out waiting for chat output")??;
|
||||
|
||||
chat.shutdown().await.context("failed to shut down chat client")?;
|
||||
|
||||
println!("final_reasoning={:?}", output.0);
|
||||
println!("final_text={:?}", output.1);
|
||||
println!("final_output_token_count={:?}", output.2);
|
||||
println!("finish_reason={:?}", output.3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::info;
|
||||
use vllm_text::backend::hf::{HfTextBackend, ResolvedModelFiles, load_model_config};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{DynTextBackend, TextBackend as _};
|
||||
|
||||
use crate::backend::{
|
||||
ChatBackend, DynChatBackend, LoadModelBackendsOptions, LoadedModelBackends,
|
||||
NewChatOutputProcessorOptions,
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::multimodal::MultimodalModelInfo;
|
||||
use crate::output::{
|
||||
DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
|
||||
};
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
|
||||
/// [`ChatBackend`] implementation built on Hugging Face model files.
|
||||
pub struct HfChatBackend {
|
||||
model_id: String,
|
||||
model_type: String,
|
||||
tokenizer: DynTokenizer,
|
||||
chat_renderer: DynChatRenderer,
|
||||
multimodal_model_info: Option<MultimodalModelInfo>,
|
||||
}
|
||||
|
||||
impl HfChatBackend {
|
||||
/// Load the chat backend from resolved Hugging Face model files.
|
||||
pub fn from_resolved_model_files(
|
||||
files: ResolvedModelFiles,
|
||||
model_id: String,
|
||||
options: LoadModelBackendsOptions,
|
||||
tokenizer: DynTokenizer,
|
||||
) -> Result<Self> {
|
||||
let model_config = load_model_config(files.config_path.as_deref())?;
|
||||
let model_type = model_config.model_type().unwrap_or_default();
|
||||
let multimodal_model_info = MultimodalModelInfo::from_paths(
|
||||
model_id.clone(),
|
||||
(!model_type.is_empty()).then_some(model_type.to_string()),
|
||||
files.config_path.as_deref(),
|
||||
files.preprocessor_config_path.as_deref(),
|
||||
tokenizer.clone(),
|
||||
)?;
|
||||
let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
|
||||
|
||||
let renderer = options.renderer.resolve(model_type);
|
||||
let chat_renderer: DynChatRenderer = match renderer {
|
||||
RendererSelection::Auto => unreachable!("renderer auto should be resolved above"),
|
||||
RendererSelection::Hf => Arc::new(HfChatRenderer::load(
|
||||
&files,
|
||||
options,
|
||||
multimodal_render_info,
|
||||
)?),
|
||||
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
};
|
||||
|
||||
info!(
|
||||
model_id,
|
||||
model_type,
|
||||
%renderer,
|
||||
"loaded chat backend with Hugging Face model files"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
model_id,
|
||||
model_type: model_type.to_string(),
|
||||
tokenizer,
|
||||
chat_renderer,
|
||||
multimodal_model_info,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for HfChatBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
self.chat_renderer.clone()
|
||||
}
|
||||
|
||||
fn multimodal_model_info(&self) -> Option<&MultimodalModelInfo> {
|
||||
self.multimodal_model_info.as_ref()
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
request: &mut ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> Result<DynChatOutputProcessor> {
|
||||
if self.model_type == "gpt_oss" {
|
||||
validate_harmony_parser_overrides(options.tool_call_parser, options.reasoning_parser)?;
|
||||
return Ok(Box::new(HarmonyChatOutputProcessor::new(request)?));
|
||||
}
|
||||
|
||||
Ok(Box::new(DefaultChatOutputProcessor::new(
|
||||
request,
|
||||
&self.model_id,
|
||||
self.tokenizer.clone(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the Hugging Face text and chat backends for the given model id.
|
||||
pub(super) async fn load_model_backends(
|
||||
model_id: &str,
|
||||
options: LoadModelBackendsOptions,
|
||||
) -> Result<LoadedModelBackends> {
|
||||
let files = ResolvedModelFiles::new(model_id).await?;
|
||||
let text_backend =
|
||||
HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?;
|
||||
let tokenizer = text_backend.tokenizer();
|
||||
let text_backend: DynTextBackend = Arc::new(text_backend);
|
||||
|
||||
let chat_backend: DynChatBackend = Arc::new(HfChatBackend::from_resolved_model_files(
|
||||
files,
|
||||
model_id.to_string(),
|
||||
options,
|
||||
tokenizer,
|
||||
)?);
|
||||
|
||||
Ok(LoadedModelBackends {
|
||||
text_backend,
|
||||
chat_backend,
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_multimodal_render_info(
|
||||
info: Option<&MultimodalModelInfo>,
|
||||
) -> Option<MultimodalRenderInfo> {
|
||||
info.map(|info| MultimodalRenderInfo {
|
||||
placeholder_token: info.placeholder_token().to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use vllm_text::backend::hf::TokenizerSource;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use super::HfChatBackend;
|
||||
use crate::RendererSelection;
|
||||
use crate::backend::{ChatBackend, LoadModelBackendsOptions};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest};
|
||||
|
||||
fn request_with_user_text(text: &str) -> ChatRequest {
|
||||
ChatRequest {
|
||||
request_id: "renderer-selection-test".to_string(),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: ChatContent::Text(text.to_string()),
|
||||
}],
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
fn write_json(path: &std::path::Path, content: &str) {
|
||||
std::fs::write(path, content).unwrap();
|
||||
}
|
||||
|
||||
fn resolved_files(
|
||||
config_json: &str,
|
||||
tokenizer_config_json: &str,
|
||||
) -> vllm_text::backend::hf::ResolvedModelFiles {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.keep();
|
||||
let config_path = root.join("config.json");
|
||||
let tokenizer_config_path = root.join("tokenizer_config.json");
|
||||
write_json(&config_path, config_json);
|
||||
write_json(&tokenizer_config_path, tokenizer_config_json);
|
||||
|
||||
vllm_text::backend::hf::ResolvedModelFiles {
|
||||
tokenizer: TokenizerSource::HuggingFace(PathBuf::from("/tmp/unused-tokenizer.json")),
|
||||
tokenizer_config_path: Some(tokenizer_config_path),
|
||||
generation_config_path: None,
|
||||
preprocessor_config_path: None,
|
||||
chat_template_path: None,
|
||||
config_path: Some(config_path),
|
||||
}
|
||||
}
|
||||
|
||||
struct TestTokenizer;
|
||||
|
||||
impl Tokenizer for TestTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn test_tokenizer() -> DynTokenizer {
|
||||
Arc::new(TestTokenizer)
|
||||
}
|
||||
|
||||
fn render_prompt(
|
||||
renderer: RendererSelection,
|
||||
config_json: &str,
|
||||
tokenizer_config_json: &str,
|
||||
) -> String {
|
||||
let backend = HfChatBackend::from_resolved_model_files(
|
||||
resolved_files(config_json, tokenizer_config_json),
|
||||
"test-model".to_string(),
|
||||
LoadModelBackendsOptions {
|
||||
renderer,
|
||||
chat_template_content_format: Default::default(),
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: HashMap::new(),
|
||||
},
|
||||
test_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
backend
|
||||
.chat_renderer()
|
||||
.render(&request_with_user_text("hello"))
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_text()
|
||||
.expect("renderer should return text prompt")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_uses_deepseek_renderer_for_deepseek_v32_model_type() {
|
||||
let prompt = render_prompt(
|
||||
RendererSelection::Auto,
|
||||
r#"{"model_type":"deepseek_v32"}"#,
|
||||
r#"{}"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
prompt,
|
||||
"<|begin▁of▁sentence|><|User|>hello<|Assistant|></think>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_uses_hf_renderer_for_other_model_types() {
|
||||
let prompt = render_prompt(
|
||||
RendererSelection::Auto,
|
||||
r#"{"model_type":"qwen2"}"#,
|
||||
r#"{"chat_template":"{{ messages[0].content }}"}"#,
|
||||
);
|
||||
|
||||
assert_eq!(prompt, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_deepseek_renderer_overrides_generic_model_type() {
|
||||
let prompt = render_prompt(
|
||||
RendererSelection::DeepSeekV32,
|
||||
r#"{"model_type":"qwen2"}"#,
|
||||
r#"{"chat_template":"{{ messages[0].content }}"}"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
prompt,
|
||||
"<|begin▁of▁sentence|><|User|>hello<|Assistant|></think>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_hf_renderer_overrides_deepseek_v32_model_type() {
|
||||
let prompt = render_prompt(
|
||||
RendererSelection::Hf,
|
||||
r#"{"model_type":"deepseek_v32"}"#,
|
||||
r#"{"chat_template":"{{ messages[0].content }}"}"#,
|
||||
);
|
||||
|
||||
assert_eq!(prompt, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_uses_nested_text_config_model_type() {
|
||||
let prompt = render_prompt(
|
||||
RendererSelection::Auto,
|
||||
r#"{"text_config":{"model_type":"deepseek_v32","num_attention_heads":32}}"#,
|
||||
r#"{}"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
prompt,
|
||||
"<|begin▁of▁sentence|><|User|>hello<|Assistant|></think>"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use vllm_text::{DynTextBackend, TextBackend};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::multimodal::MultimodalModelInfo;
|
||||
use crate::output::DynChatOutputProcessor;
|
||||
use crate::renderer::DynChatRenderer;
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
|
||||
pub mod hf;
|
||||
|
||||
/// Options for creating a new chat output processor.
|
||||
pub struct NewChatOutputProcessorOptions<'a> {
|
||||
pub tool_call_parser: &'a ParserSelection,
|
||||
pub reasoning_parser: &'a ParserSelection,
|
||||
}
|
||||
|
||||
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
||||
pub trait ChatBackend: Send + Sync {
|
||||
/// Return the renderer used for chat-prompt construction.
|
||||
fn chat_renderer(&self) -> DynChatRenderer;
|
||||
|
||||
/// Return model files/config needed for request-scoped multimodal
|
||||
/// preprocessing, if supported.
|
||||
fn multimodal_model_info(&self) -> Option<&MultimodalModelInfo> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Create a request-scoped output processor after request-level adjustments
|
||||
/// are applied.
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
request: &mut ChatRequest,
|
||||
options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> Result<DynChatOutputProcessor>;
|
||||
}
|
||||
|
||||
/// Shared trait-object form of [`ChatBackend`].
|
||||
pub type DynChatBackend = Arc<dyn ChatBackend>;
|
||||
|
||||
/// Convenience trait for backends that can serve both raw text generation and
|
||||
/// chat templating.
|
||||
///
|
||||
/// This is mainly useful in tests and small examples, where one mock/backend
|
||||
/// often implements both sides and callers want `ChatLlm` to wire the shared
|
||||
/// object into `TextLlm` automatically.
|
||||
pub trait ChatTextBackend: ChatBackend + TextBackend {}
|
||||
|
||||
impl<T> ChatTextBackend for T where T: ChatBackend + TextBackend + ?Sized {}
|
||||
|
||||
/// Shared trait-object form of [`ChatTextBackend`].
|
||||
pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
|
||||
|
||||
/// Frontend-side chat backend loading options.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct LoadModelBackendsOptions {
|
||||
/// Which chat renderer implementation to use.
|
||||
pub renderer: RendererSelection,
|
||||
/// How to serialize `message.content` when rendering the chat template.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Optional server-default chat template override, provided either as an
|
||||
/// inline template or as a path to a template file.
|
||||
pub chat_template: Option<String>,
|
||||
/// Optional server-default keyword arguments merged into every
|
||||
/// chat-template render before request-level `chat_template_kwargs`.
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Shared backends loaded from a model id.
|
||||
pub struct LoadedModelBackends {
|
||||
pub text_backend: DynTextBackend,
|
||||
pub chat_backend: DynChatBackend,
|
||||
}
|
||||
|
||||
/// Load text and chat backends for the given model id.
|
||||
pub async fn load_model_backends(
|
||||
model_id: &str,
|
||||
options: LoadModelBackendsOptions,
|
||||
) -> Result<LoadedModelBackends> {
|
||||
// Currently, we only have HuggingFace backends.
|
||||
hf::load_model_backends(model_id, options).await
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
|
||||
type BoxedError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
#[derive(Debug, Error, Macro)]
|
||||
#[thiserror_ext(macro(path = "crate::error"))]
|
||||
pub enum Error {
|
||||
#[error("chat request must contain at least one message")]
|
||||
EmptyMessages,
|
||||
#[error("cannot continue the final message when the last message is not from the assistant")]
|
||||
ContinueFinalAssistantWithoutFinalAssistant,
|
||||
#[error("chat template is required but none was configured")]
|
||||
MissingChatTemplate,
|
||||
#[error("chat template error: {0}")]
|
||||
ChatTemplate(String),
|
||||
#[error("multimodal input is not supported by this chat renderer")]
|
||||
UnsupportedMultimodalRenderer,
|
||||
#[error("unsupported multimodal content: {0}")]
|
||||
UnsupportedMultimodalContent(&'static str),
|
||||
#[error("multimodal preprocessing error: {0}")]
|
||||
Multimodal(#[message] String),
|
||||
#[error("{kind} parsing is not available for model `{model_id}`")]
|
||||
ParserUnavailableForModel {
|
||||
kind: &'static str,
|
||||
model_id: String,
|
||||
},
|
||||
#[error("{kind} parsing is disabled by frontend configuration")]
|
||||
ParserDisabled { kind: &'static str },
|
||||
#[error(
|
||||
"{kind} parser `{name}` is not registered{}",
|
||||
available_parser_hint(.available_names)
|
||||
)]
|
||||
ParserUnavailableByName {
|
||||
kind: &'static str,
|
||||
name: String,
|
||||
available_names: Vec<String>,
|
||||
},
|
||||
#[error("failed to initialize {kind} parser `{name}`")]
|
||||
ParserInitialization {
|
||||
kind: &'static str,
|
||||
name: String,
|
||||
#[source]
|
||||
error: BoxedError,
|
||||
},
|
||||
#[error(
|
||||
"gpt_oss uses native Harmony output parsing; generic {kind} parser override `{selection}` is not supported"
|
||||
)]
|
||||
HarmonyParserOverrideUnsupported {
|
||||
kind: &'static str,
|
||||
selection: String,
|
||||
},
|
||||
#[error("harmony output parsing failed")]
|
||||
HarmonyOutputParsing {
|
||||
#[source]
|
||||
error: BoxedError,
|
||||
},
|
||||
#[error(
|
||||
"this model's maximum context length is {max_model_len} tokens, \
|
||||
but the prompt contains {prompt_len} input tokens"
|
||||
)]
|
||||
PromptTooLong { max_model_len: u32, prompt_len: u32 },
|
||||
#[error("chat request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error("tool call stream state is inconsistent: {message}")]
|
||||
ToolCallStreamInvariant { message: String },
|
||||
#[error(transparent)]
|
||||
Text(#[from] vllm_text::Error),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// Format the available-parser suffix used in user-facing error messages.
|
||||
fn available_parser_hint(available_names: &[String]) -> String {
|
||||
if available_names.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" (choose from: {})", available_names.join(", "))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_text::{DecodedLogprobs, DecodedPromptLogprobs};
|
||||
|
||||
use crate::FinishReason;
|
||||
|
||||
/// One finalized assistant tool call.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AssistantToolCall {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub arguments: String,
|
||||
}
|
||||
|
||||
/// Semantic kind of one assistant output block.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum AssistantBlockKind {
|
||||
/// Visible final-answer text.
|
||||
Text,
|
||||
/// Extracted reasoning content.
|
||||
Reasoning,
|
||||
/// One finalized tool call.
|
||||
ToolCall,
|
||||
}
|
||||
|
||||
/// One structured assistant output block.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum AssistantContentBlock {
|
||||
/// Visible final-answer text.
|
||||
Text { text: String },
|
||||
/// Extracted reasoning content.
|
||||
Reasoning { text: String },
|
||||
/// One finalized tool call.
|
||||
ToolCall(AssistantToolCall),
|
||||
}
|
||||
|
||||
impl AssistantContentBlock {
|
||||
/// Return the semantic kind of this block.
|
||||
pub fn kind(&self) -> AssistantBlockKind {
|
||||
match self {
|
||||
Self::Text { .. } => AssistantBlockKind::Text,
|
||||
Self::Reasoning { .. } => AssistantBlockKind::Reasoning,
|
||||
Self::ToolCall(..) => AssistantBlockKind::ToolCall,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return this block as one finalized tool call, if applicable.
|
||||
pub fn as_tool_call(&self) -> Option<&AssistantToolCall> {
|
||||
match self {
|
||||
Self::ToolCall(call) => Some(call),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[easy_ext::ext(AssistantMessageExt)]
|
||||
impl [AssistantContentBlock] {
|
||||
/// Concatenate all visible final-answer text blocks.
|
||||
pub fn text(&self) -> String {
|
||||
self.iter()
|
||||
.filter_map(|block| match block {
|
||||
AssistantContentBlock::Text { text } => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Concatenate all extracted reasoning blocks, if any.
|
||||
pub fn reasoning(&self) -> Option<String> {
|
||||
Some(
|
||||
self.iter()
|
||||
.filter_map(|block| match block {
|
||||
AssistantContentBlock::Reasoning { text } => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.filter(|s: &String| !s.is_empty())
|
||||
}
|
||||
|
||||
/// Return whether this assistant message contains any non-empty reasoning
|
||||
/// text blocks.
|
||||
pub fn has_reasoning(&self) -> bool {
|
||||
self.iter().any(|block| match block {
|
||||
AssistantContentBlock::Reasoning { text } => !text.is_empty(),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return finalized assistant tool calls in encounter order.
|
||||
pub fn tool_calls(&self) -> impl Iterator<Item = &AssistantToolCall> {
|
||||
self.iter().filter_map(AssistantContentBlock::as_tool_call)
|
||||
}
|
||||
|
||||
/// Return whether this assistant message contains any tool-call blocks.
|
||||
pub fn has_tool_calls(&self) -> bool {
|
||||
self.iter().any(|block| matches!(block, AssistantContentBlock::ToolCall(_)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Final structured assistant message assembled from the event stream.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AssistantMessage {
|
||||
pub content: Vec<AssistantContentBlock>,
|
||||
}
|
||||
|
||||
impl Deref for AssistantMessage {
|
||||
type Target = [AssistantContentBlock];
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.content
|
||||
}
|
||||
}
|
||||
|
||||
impl AssistantMessage {
|
||||
/// Push one new block to the end of the message content.
|
||||
pub(crate) fn push_block(&mut self, block: AssistantContentBlock) {
|
||||
self.content.push(block);
|
||||
}
|
||||
}
|
||||
|
||||
/// Streamed chat event emitted by [`crate::ChatEventStream`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ChatEvent {
|
||||
/// The request was accepted, streaming has started, and prompt metadata is
|
||||
/// ready.
|
||||
Start {
|
||||
/// The actual prompt token IDs for this request.
|
||||
prompt_token_ids: Arc<[u32]>,
|
||||
/// Once-only prompt logprobs metadata, when requested.
|
||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
},
|
||||
/// A new assistant output block has started.
|
||||
BlockStart {
|
||||
index: usize,
|
||||
kind: AssistantBlockKind,
|
||||
},
|
||||
/// A newly observed delta for one open assistant output block.
|
||||
BlockDelta {
|
||||
index: usize,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
},
|
||||
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
||||
LogprobsDelta {
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
token_ids: Vec<u32>,
|
||||
},
|
||||
/// One assistant output block has ended.
|
||||
BlockEnd {
|
||||
index: usize,
|
||||
block: AssistantContentBlock,
|
||||
},
|
||||
/// One tool call has started.
|
||||
ToolCallStart {
|
||||
index: usize,
|
||||
id: String,
|
||||
name: String,
|
||||
},
|
||||
/// One incremental tool-call arguments delta for the currently open tool
|
||||
/// call.
|
||||
ToolCallArgumentsDelta { index: usize, delta: String },
|
||||
/// One tool call has ended.
|
||||
ToolCallEnd {
|
||||
index: usize,
|
||||
call: AssistantToolCall,
|
||||
},
|
||||
/// Terminal event carrying the final assembled assistant message and finish
|
||||
/// metadata.
|
||||
Done {
|
||||
message: AssistantMessage,
|
||||
/// Number of prompt tokens actually sent to the engine after chat
|
||||
/// template rendering and tokenization.
|
||||
prompt_token_count: usize,
|
||||
/// Number of output tokens generated.
|
||||
output_token_count: usize,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
//! Minimal chat facade above [`vllm_text`].
|
||||
//!
|
||||
//! This crate keeps the northbound boundary intentionally small:
|
||||
//! `messages -> rendered prompt -> tokenized prompt -> engine request ->
|
||||
//! streamed structured assistant events`. The request side remains text-first,
|
||||
//! while the response side can emit structured reasoning and final-answer
|
||||
//! blocks. It is closer to vLLM's internal chat-rendering flow than to a full
|
||||
//! OpenAI-compatible surface.
|
||||
|
||||
pub use backend::hf::HfChatBackend;
|
||||
pub use backend::{
|
||||
ChatBackend, ChatTextBackend, DynChatBackend, DynChatTextBackend, LoadModelBackendsOptions,
|
||||
LoadedModelBackends, NewChatOutputProcessorOptions, load_model_backends,
|
||||
};
|
||||
pub use error::{Error, Result};
|
||||
pub use event::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantMessageExt,
|
||||
AssistantToolCall, ChatEvent,
|
||||
};
|
||||
use futures::{StreamExt, TryStreamExt as _};
|
||||
pub use output::{
|
||||
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
HarmonyChatOutputProcessor,
|
||||
};
|
||||
pub use parser::ParserSelection;
|
||||
pub use parser::reasoning::{
|
||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
|
||||
};
|
||||
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, RenderedPrompt,
|
||||
RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
|
||||
ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
|
||||
};
|
||||
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
|
||||
pub use vllm_llm::FinishReason;
|
||||
|
||||
mod backend;
|
||||
mod error;
|
||||
mod event;
|
||||
pub mod multimodal;
|
||||
mod output;
|
||||
mod parser;
|
||||
mod renderer;
|
||||
mod request;
|
||||
mod stream;
|
||||
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{TextLlm, TextRequest};
|
||||
|
||||
/// Validate explicit parser override names without starting request processing.
|
||||
pub fn validate_parser_overrides(
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
) -> Result<()> {
|
||||
let tool_parser_factory = ToolParserFactory::global();
|
||||
if let ParserSelection::Explicit(name) = tool_call_parser
|
||||
&& !tool_parser_factory.contains(name)
|
||||
{
|
||||
return Err(Error::ParserUnavailableByName {
|
||||
kind: "tool",
|
||||
name: name.clone(),
|
||||
available_names: tool_parser_factory.list(),
|
||||
});
|
||||
}
|
||||
|
||||
let reasoning_parser_factory = ReasoningParserFactory::global();
|
||||
if let ParserSelection::Explicit(name) = reasoning_parser
|
||||
&& !reasoning_parser_factory.contains(name)
|
||||
{
|
||||
return Err(Error::ParserUnavailableByName {
|
||||
kind: "reasoning",
|
||||
name: name.clone(),
|
||||
available_names: reasoning_parser_factory.list(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Structured chat facade above [`TextLlm`].
|
||||
///
|
||||
/// This layer stays above raw text semantics: it takes care of chat-template
|
||||
/// rendering, exposes structured assistant events, and adds chat-specific
|
||||
/// request semantics such as tool calls.
|
||||
pub struct ChatLlm {
|
||||
text: TextLlm,
|
||||
backend: DynChatBackend,
|
||||
/// Effective model dtype reported by the engine.
|
||||
model_dtype: Option<ModelDtype>,
|
||||
/// Tool-call parser selection.
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
reasoning_parser: ParserSelection,
|
||||
}
|
||||
|
||||
impl ChatLlm {
|
||||
/// Create a new chat facade from a text-generation facade plus a chat
|
||||
/// backend.
|
||||
pub fn new(text: TextLlm, backend: DynChatBackend) -> Self {
|
||||
let model_dtype = text.engine_core_client().model_dtype();
|
||||
|
||||
Self {
|
||||
text,
|
||||
backend,
|
||||
model_dtype,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience constructor for one shared backend object that implements
|
||||
/// both text and chat responsibilities.
|
||||
pub fn from_shared_backend(llm: Llm, backend: DynChatTextBackend) -> Self {
|
||||
let text = TextLlm::new(llm, backend.clone());
|
||||
Self::new(text, backend)
|
||||
}
|
||||
|
||||
/// Set tool-call parser selection.
|
||||
pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self {
|
||||
self.tool_call_parser = selection;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning parser selection.
|
||||
pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self {
|
||||
self.reasoning_parser = selection;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the effective model dtype used for multimodal tensor encoding.
|
||||
pub fn with_model_dtype(mut self, model_dtype: Option<ModelDtype>) -> Self {
|
||||
self.model_dtype = model_dtype;
|
||||
self
|
||||
}
|
||||
|
||||
/// Expose the underlying text facade for raw text-generation routes such as
|
||||
/// `/v1/completions`.
|
||||
pub fn text(&self) -> &TextLlm {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Return the model ID reported by the underlying text backend.
|
||||
pub fn model_id(&self) -> &str {
|
||||
self.text.model_id()
|
||||
}
|
||||
|
||||
/// Expose the underlying engine-core client for low-level utility/admin
|
||||
/// calls.
|
||||
pub fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
self.text.engine_core_client()
|
||||
}
|
||||
|
||||
/// Render, tokenize, and submit one chat request.
|
||||
pub async fn chat(&self, mut request: ChatRequest) -> Result<ChatEventStream> {
|
||||
request.validate()?;
|
||||
|
||||
let output_processor = self.backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &self.tool_call_parser,
|
||||
reasoning_parser: &self.reasoning_parser,
|
||||
},
|
||||
)?;
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
|
||||
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
|
||||
&request,
|
||||
rendered,
|
||||
self.backend.multimodal_model_info(),
|
||||
self.model_dtype,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let text_request = TextRequest {
|
||||
request_id: request.request_id.clone(),
|
||||
prompt,
|
||||
mm_features,
|
||||
sampling_params: request.sampling_params,
|
||||
decode_options: request.decode_options,
|
||||
intermediate: request.intermediate,
|
||||
priority: request.priority,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
};
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
let structured_stream = output_processor.process(decoded_stream)?;
|
||||
|
||||
Ok(ChatEventStream::new(request.request_id, structured_stream))
|
||||
}
|
||||
|
||||
/// Shut down the underlying LLM client and its background tasks.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
self.text.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::{ParserSelection, validate_parser_overrides};
|
||||
use crate::parser::reasoning::names;
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_accepts_registered_names() {
|
||||
validate_parser_overrides(
|
||||
&ParserSelection::Explicit("llama3_json".to_string()),
|
||||
&ParserSelection::Explicit(names::QWEN3.to_string()),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_accepts_auto_and_none() {
|
||||
validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_rejects_unknown_tool_parser() {
|
||||
let error = validate_parser_overrides(
|
||||
&ParserSelection::Explicit("definitely_missing_tool_parser".to_string()),
|
||||
&ParserSelection::Auto,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, hermes, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_rejects_unknown_reasoning_parser() {
|
||||
let error = validate_parser_overrides(
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, nemotron_v3, qwen3, step3)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
//! Chat-layer multimodal image preparation.
|
||||
//!
|
||||
//! This module owns the narrow image-only multimodal path for chat requests:
|
||||
//! it extracts image parts from structured chat messages, fetches and
|
||||
//! preprocesses them through `llm-multimodal`, expands rendered prompt
|
||||
//! placeholders after tokenization, and builds the engine-facing
|
||||
//! `MmFeatures` payload.
|
||||
//!
|
||||
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
|
||||
//! opaque tensor payloads before the request is handed to text generation.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, LazyLock, Once};
|
||||
|
||||
use itertools::izip;
|
||||
use llm_multimodal::{
|
||||
AsyncMultiModalTracker, FieldLayout, ImagePreProcessor, ImageProcessorRegistry, MediaConnector,
|
||||
MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec,
|
||||
ModelRegistry, PreProcessorConfig, PreprocessedImages, PromptReplacement, TokenResolver,
|
||||
TrackedMedia,
|
||||
};
|
||||
use tracing::warn;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmFeatureSpec, MmFeatures, MmField, MmFieldElem, MmFlatField, MmKwargsItem,
|
||||
MmSharedField, MmSlice, PlaceholderRange, SliceSpec,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::tensor::WireTensor;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
use crate::renderer::RenderedPrompt;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
|
||||
mod tensor;
|
||||
|
||||
/// Resolved multimodal support for one loaded model.
|
||||
#[derive(Clone)]
|
||||
pub struct MultimodalModelInfo {
|
||||
context: MultimodalModelContext,
|
||||
spec: ResolvedMultimodalSpec,
|
||||
image_processor: ResolvedImageProcessor,
|
||||
media_connector: Arc<MediaConnector>,
|
||||
}
|
||||
|
||||
/// Model metadata and tokenizer access shared by all multimodal specs.
|
||||
#[derive(Clone)]
|
||||
struct MultimodalModelContext {
|
||||
model_id: String,
|
||||
model_type: Option<String>,
|
||||
config: serde_json::Value,
|
||||
tokenizer: TokenizerResolver,
|
||||
}
|
||||
|
||||
impl MultimodalModelContext {
|
||||
fn metadata(&self) -> ModelMetadata<'_> {
|
||||
ModelMetadata {
|
||||
model_id: &self.model_id,
|
||||
tokenizer: &self.tokenizer,
|
||||
config: &self.config,
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer(&self) -> &dyn Tokenizer {
|
||||
self.tokenizer.0.as_ref()
|
||||
}
|
||||
|
||||
/// Resolve a static model processor spec for one loaded model.
|
||||
fn resolve_model_spec(&self) -> Option<&'static dyn ModelProcessorSpec> {
|
||||
static REGISTRY: LazyLock<ModelRegistry> = LazyLock::new(ModelRegistry::new);
|
||||
REGISTRY.lookup(&self.metadata())
|
||||
}
|
||||
|
||||
/// Resolve a static image preprocessor for one loaded model.
|
||||
fn resolve_image_processor(&self) -> Option<&'static dyn ImagePreProcessor> {
|
||||
static REGISTRY: LazyLock<ImageProcessorRegistry> =
|
||||
LazyLock::new(ImageProcessorRegistry::with_defaults);
|
||||
REGISTRY.find(&self.model_id, self.model_type.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Static model-specific prompt and tensor-layout behavior.
|
||||
#[derive(Clone)]
|
||||
struct ResolvedMultimodalSpec {
|
||||
raw: &'static dyn ModelProcessorSpec,
|
||||
placeholder_token: String,
|
||||
placeholder_marker_token_id: u32,
|
||||
placeholder_embed_token_id: u32,
|
||||
field_layouts: HashMap<String, FieldLayout>,
|
||||
keep_on_cpu_keys: HashSet<String>,
|
||||
}
|
||||
|
||||
impl ResolvedMultimodalSpec {
|
||||
fn new(raw: &'static dyn ModelProcessorSpec, context: &MultimodalModelContext) -> Result<Self> {
|
||||
let metadata = context.metadata();
|
||||
let placeholder_token =
|
||||
raw.placeholder_token(&metadata).map_err(|error| multimodal!("{error}"))?;
|
||||
// This is the rendered prompt marker, so resolve it from the token
|
||||
// string itself. Do not use `ModelProcessorSpec::placeholder_token_id()`:
|
||||
// for some specs that ID is the replacement vision/patch token,
|
||||
// not necessarily the token ID of `placeholder_token`.
|
||||
let placeholder_marker_token_id =
|
||||
context.tokenizer().token_to_id(&placeholder_token).ok_or_else(|| {
|
||||
multimodal!(
|
||||
"placeholder token `{placeholder_token}` is not in the tokenizer vocabulary"
|
||||
)
|
||||
})?;
|
||||
let placeholder_embed_token_id =
|
||||
raw.placeholder_token_id(&metadata).map_err(|error| multimodal!("{error}"))? as u32;
|
||||
|
||||
Ok(Self {
|
||||
raw,
|
||||
placeholder_token,
|
||||
placeholder_marker_token_id,
|
||||
placeholder_embed_token_id,
|
||||
field_layouts: raw.field_layouts(),
|
||||
keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn prompt_replacements(
|
||||
&self,
|
||||
context: &MultimodalModelContext,
|
||||
preprocessed: &PreprocessedImages,
|
||||
) -> Result<Vec<PromptReplacement>> {
|
||||
self.raw
|
||||
.prompt_replacements(&context.metadata(), preprocessed)
|
||||
.map_err(|error| multimodal!("{error}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Static image preprocessor plus its loaded config.
|
||||
#[derive(Clone)]
|
||||
struct ResolvedImageProcessor {
|
||||
raw: &'static dyn ImagePreProcessor,
|
||||
config: PreProcessorConfig,
|
||||
}
|
||||
|
||||
/// Request-scoped fetched media, kept together with tracker UUID metadata.
|
||||
struct FetchedImageMedia {
|
||||
frames: Vec<Arc<llm_multimodal::ImageFrame>>,
|
||||
uuids: Vec<Option<String>>,
|
||||
}
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Load and resolve multimodal support from model files.
|
||||
///
|
||||
/// Returns `Ok(Some(_))` only when both the model spec and image processor
|
||||
/// are registered. File read/parse failures are real errors; unsupported
|
||||
/// model families are logged and returned as `Ok(None)`.
|
||||
pub fn from_paths(
|
||||
model_id: String,
|
||||
model_type: Option<String>,
|
||||
config_path: Option<&Path>,
|
||||
preprocessor_config_path: Option<&Path>,
|
||||
tokenizer: DynTokenizer,
|
||||
) -> Result<Option<Self>> {
|
||||
let config = match config_path {
|
||||
Some(path) => {
|
||||
let text = fs::read_to_string(path)
|
||||
.map_err(|error| multimodal!("failed to read config.json: {error}"))?;
|
||||
serde_json::from_str(&text)
|
||||
.map_err(|error| multimodal!("failed to parse config.json: {error}"))?
|
||||
}
|
||||
None => serde_json::Value::Object(Default::default()),
|
||||
};
|
||||
let preprocessor_config = match preprocessor_config_path {
|
||||
Some(path) => {
|
||||
let text = fs::read_to_string(path).map_err(|error| {
|
||||
multimodal!("failed to read preprocessor_config.json: {error}")
|
||||
})?;
|
||||
PreProcessorConfig::from_json(&text).map_err(|error| {
|
||||
multimodal!("failed to parse preprocessor_config.json: {error}")
|
||||
})?
|
||||
}
|
||||
None => PreProcessorConfig::default(),
|
||||
};
|
||||
|
||||
let context = MultimodalModelContext {
|
||||
model_id,
|
||||
model_type,
|
||||
config,
|
||||
tokenizer: TokenizerResolver(tokenizer),
|
||||
};
|
||||
|
||||
let Some(spec) = context.resolve_model_spec() else {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"multimodal model spec is not registered; disabling multimodal support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
let spec = ResolvedMultimodalSpec::new(spec, &context)?;
|
||||
|
||||
let Some(image_processor) = context.resolve_image_processor() else {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"image processor is not registered; disabling multimodal support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let media_connector = Arc::new(
|
||||
MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default())
|
||||
.map_err(|error| multimodal!("{error}"))?,
|
||||
);
|
||||
|
||||
Ok(Some(Self {
|
||||
context,
|
||||
spec,
|
||||
image_processor: ResolvedImageProcessor {
|
||||
raw: image_processor,
|
||||
config: preprocessor_config,
|
||||
},
|
||||
media_connector,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Return the template-visible placeholder token for this model.
|
||||
///
|
||||
/// The HF renderer uses this token while flattening image content in string
|
||||
/// content format.
|
||||
pub(crate) fn placeholder_token(&self) -> &str {
|
||||
&self.spec.placeholder_token
|
||||
}
|
||||
}
|
||||
|
||||
/// Finalize a rendered chat prompt into text-generation input.
|
||||
///
|
||||
/// Text-only requests pass through unchanged as `Prompt::Text`. Multimodal
|
||||
/// requests are tokenized in chat, their image placeholders are expanded, and
|
||||
/// preprocessed image features are attached for engine-core transport.
|
||||
pub(crate) async fn finalize_rendered_prompt(
|
||||
request: &ChatRequest,
|
||||
rendered: RenderedPrompt,
|
||||
info: Option<&MultimodalModelInfo>,
|
||||
model_dtype: Option<ModelDtype>,
|
||||
) -> Result<(Prompt, Option<MmFeatures>)> {
|
||||
if !request.has_multimodal() {
|
||||
return Ok((rendered.prompt, None));
|
||||
}
|
||||
let info = info.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
let Prompt::Text(prompt) = rendered.prompt else {
|
||||
bail_multimodal!("multimodal chat renderer must return a text prompt before expansion");
|
||||
};
|
||||
let media_parts = extract_media_parts(request)?;
|
||||
let model_dtype = model_dtype.unwrap_or_else(|| {
|
||||
static WARN_ONCE: Once = Once::new();
|
||||
WARN_ONCE.call_once(|| {
|
||||
warn!(
|
||||
"engine handshake did not report model dtype; \
|
||||
falling back to float32 for multimodal tensor encoding"
|
||||
);
|
||||
});
|
||||
ModelDtype::Float32
|
||||
});
|
||||
|
||||
let mut prompt_token_ids = info
|
||||
.context
|
||||
.tokenizer()
|
||||
.encode(&prompt, request.add_special_tokens)
|
||||
.map_err(|error| multimodal!("{error}"))?;
|
||||
let prepared = info.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype).await?;
|
||||
|
||||
Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared)))
|
||||
}
|
||||
|
||||
/// Extract image media parts from chat messages in message/content order.
|
||||
///
|
||||
/// Assistant history is skipped because generated assistant blocks are already
|
||||
/// represented as text for prompt rendering in this crate.
|
||||
fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
|
||||
let mut all_parts = Vec::new();
|
||||
for message in &request.messages {
|
||||
let content = match message {
|
||||
ChatMessage::System { content }
|
||||
| ChatMessage::Developer { content, .. }
|
||||
| ChatMessage::User { content }
|
||||
| ChatMessage::ToolResponse { content, .. } => content,
|
||||
ChatMessage::Assistant { .. } => continue,
|
||||
};
|
||||
let ChatContent::Parts(parts) = content else {
|
||||
continue;
|
||||
};
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { .. } => {}
|
||||
ChatContentPart::ImageUrl {
|
||||
image_url,
|
||||
detail,
|
||||
uuid,
|
||||
} => all_parts.push(MediaContentPart::ImageUrl {
|
||||
url: image_url.clone(),
|
||||
detail: *detail,
|
||||
uuid: uuid.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(all_parts)
|
||||
}
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Run media fetch, image preprocessing, prompt expansion, and feature
|
||||
/// build.
|
||||
///
|
||||
/// `prompt_token_ids` is mutated in place because placeholder expansion
|
||||
/// changes both the final prompt and the offsets recorded in
|
||||
/// `PlaceholderRange`.
|
||||
async fn prepare_multimodal(
|
||||
&self,
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<MmFeatures> {
|
||||
if media_parts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let media_parts_len = media_parts.len();
|
||||
|
||||
let fetched = self.fetch_images(media_parts).await?;
|
||||
let preprocessed = self.preprocess_images(&fetched.frames).await?;
|
||||
let replacements = self.spec.prompt_replacements(&self.context, &preprocessed)?;
|
||||
let ranges = self.expand_prompt_tokens(prompt_token_ids, replacements)?;
|
||||
|
||||
let features = self.build_features(preprocessed, fetched, ranges, model_dtype)?;
|
||||
if features.len() != media_parts_len {
|
||||
bail_multimodal!(
|
||||
"number of built multimodal features {} does not match number of media parts {}",
|
||||
features.len(),
|
||||
media_parts_len
|
||||
);
|
||||
}
|
||||
Ok(features)
|
||||
}
|
||||
|
||||
/// Fetch all image parts and preserve their request-order UUID metadata.
|
||||
async fn fetch_images(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedImageMedia> {
|
||||
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
|
||||
for part in media_parts {
|
||||
tracker.push_part(part).map_err(|error| multimodal!("{error}"))?;
|
||||
}
|
||||
|
||||
let tracker_output = tracker.finalize().await.map_err(|error| multimodal!("{error}"))?;
|
||||
let images = tracker_output.data.get(&Modality::Image).cloned().unwrap_or_default();
|
||||
let uuids = tracker_output.uuids.get(&Modality::Image).cloned().unwrap_or_default();
|
||||
|
||||
let frames = images
|
||||
.into_iter()
|
||||
.map(|media| match media {
|
||||
TrackedMedia::Image(frame) => Ok(frame),
|
||||
_ => Err(Error::UnsupportedMultimodalContent("non-image")),
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
Ok(FetchedImageMedia { frames, uuids })
|
||||
}
|
||||
|
||||
/// Preprocess fetched image frames with the model's resolved image
|
||||
/// processor.
|
||||
///
|
||||
/// The processor work is CPU-heavy relative to request wiring, so it runs
|
||||
/// in a blocking task and returns owned tensors ready for wire
|
||||
/// conversion.
|
||||
async fn preprocess_images(
|
||||
&self,
|
||||
image_frames: &[Arc<llm_multimodal::ImageFrame>],
|
||||
) -> Result<PreprocessedImages> {
|
||||
let config = self.image_processor.config.clone();
|
||||
let processor = self.image_processor.raw;
|
||||
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
processor.preprocess(&images, &config).map_err(|error| multimodal!("{error}"))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
||||
}
|
||||
|
||||
/// Replace rendered placeholder markers with model-specific replacement
|
||||
/// tokens.
|
||||
///
|
||||
/// Replacements are consumed in order, matching the original media-part
|
||||
/// order. The returned ranges point into the already-expanded prompt.
|
||||
fn expand_prompt_tokens(
|
||||
&self,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
replacements: Vec<PromptReplacement>,
|
||||
) -> Result<Vec<PlaceholderRange>> {
|
||||
let mut cursor = 0;
|
||||
let mut ranges = Vec::with_capacity(replacements.len());
|
||||
for replacement in replacements {
|
||||
if replacement.modality != Modality::Image {
|
||||
bail_multimodal!(
|
||||
"unsupported prompt replacement modality `{}`",
|
||||
replacement.modality
|
||||
);
|
||||
}
|
||||
let offset = find_next_token(
|
||||
prompt_token_ids,
|
||||
self.spec.placeholder_marker_token_id,
|
||||
cursor,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
multimodal!(
|
||||
"placeholder token `{}` was not found in tokenized prompt",
|
||||
self.spec.placeholder_token
|
||||
)
|
||||
})?;
|
||||
|
||||
if replacement.tokens.is_empty() {
|
||||
bail_multimodal!(
|
||||
"placeholder token `{}` expanded to no tokens",
|
||||
self.spec.placeholder_token
|
||||
);
|
||||
}
|
||||
let replacement_len = replacement.tokens.len();
|
||||
let replacement_tokens =
|
||||
replacement.tokens.iter().map(|&token| token as u32).collect::<Vec<_>>();
|
||||
let is_embed = {
|
||||
let mask = replacement_tokens
|
||||
.iter()
|
||||
.map(|&token| token == self.spec.placeholder_embed_token_id)
|
||||
.collect::<Vec<_>>();
|
||||
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
|
||||
};
|
||||
|
||||
prompt_token_ids.splice(offset..offset + 1, replacement_tokens);
|
||||
ranges.push(PlaceholderRange {
|
||||
offset,
|
||||
length: replacement_len,
|
||||
is_embed: Some(is_embed),
|
||||
});
|
||||
cursor = offset + replacement_len;
|
||||
}
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Convert preprocessed image tensors into engine-core multimodal features.
|
||||
///
|
||||
/// One `MmFeatureSpec` is produced per image. Tensor fields are
|
||||
/// sliced according to the model spec's field layout declarations.
|
||||
fn build_features(
|
||||
&self,
|
||||
preprocessed: PreprocessedImages,
|
||||
images: FetchedImageMedia,
|
||||
ranges: Vec<PlaceholderRange>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<MmFeatures> {
|
||||
let len = images.frames.len();
|
||||
let tensors = tensor::collect_tensors(preprocessed, model_dtype)?;
|
||||
|
||||
let mut features = Vec::with_capacity(images.frames.len());
|
||||
for (index, (frame, uuid, range)) in izip!(images.frames, images.uuids, ranges).enumerate()
|
||||
{
|
||||
let mut data = MmKwargsItem::new();
|
||||
for (key, tensor) in &tensors {
|
||||
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
|
||||
let (value, field) = match self.spec.field_layouts.get(key) {
|
||||
Some(FieldLayout::Batched) => (
|
||||
tensor.batched_value_at(index)?,
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu }),
|
||||
),
|
||||
Some(FieldLayout::Flat { sizes_key }) => {
|
||||
let sizes = tensors.get(sizes_key).ok_or_else(|| {
|
||||
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
|
||||
})?;
|
||||
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
|
||||
(
|
||||
tensor.flat_value_range(start, end)?,
|
||||
MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some((end - start) as isize),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
)
|
||||
}
|
||||
None => (
|
||||
tensor.clone(),
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size: len,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
data.insert(
|
||||
key.clone(),
|
||||
MmFieldElem {
|
||||
data: Some(value.try_into()?),
|
||||
field,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let hash = frame.hash.clone();
|
||||
features.push(MmFeatureSpec {
|
||||
data: Some(data),
|
||||
modality: "image".to_string(),
|
||||
identifier: uuid.unwrap_or_else(|| hash.clone()),
|
||||
mm_position: range,
|
||||
mm_hash: Some(hash),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(features)
|
||||
}
|
||||
}
|
||||
|
||||
/// Find `needle` in `haystack`, starting at `start`.
|
||||
///
|
||||
/// This is intentionally order-preserving rather than a global replace: each
|
||||
/// image consumes the next placeholder occurrence.
|
||||
fn find_next_token(haystack: &[u32], needle: u32, start: usize) -> Option<usize> {
|
||||
haystack
|
||||
.get(start..)?
|
||||
.iter()
|
||||
.position(|token| *token == needle)
|
||||
.map(|offset| start + offset)
|
||||
}
|
||||
|
||||
/// Adapter from the frontend tokenizer trait to `llm-multimodal`.
|
||||
#[derive(Clone)]
|
||||
struct TokenizerResolver(DynTokenizer);
|
||||
|
||||
impl TokenResolver for TokenizerResolver {
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
self.0.token_to_id(token)
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.0.id_to_token(id)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use llm_multimodal::TokenId;
|
||||
use vllm_engine_core_client::protocol::tensor::WireArrayData;
|
||||
use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
|
||||
|
||||
use super::*;
|
||||
|
||||
const LLAMA4_IMAGE_START_ID: u32 = 200088;
|
||||
const LLAMA4_IMAGE_END_ID: u32 = 200089;
|
||||
const LLAMA4_IMAGE_ID: u32 = 200090;
|
||||
const LLAMA4_PATCH_ID: u32 = 200092;
|
||||
const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
|
||||
const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
|
||||
|
||||
struct TestTokenizer;
|
||||
|
||||
impl Tokenizer for TestTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> std::result::Result<Vec<u32>, TokenizerError> {
|
||||
Ok(match text {
|
||||
"<|image|>" => vec![LLAMA4_IMAGE_ID],
|
||||
text => text.bytes().map(u32::from).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> std::result::Result<String, TokenizerError> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
|
||||
"<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
|
||||
"<|image|>" => Some(LLAMA4_IMAGE_ID),
|
||||
"<|patch|>" => Some(LLAMA4_PATCH_ID),
|
||||
"<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
|
||||
"<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match id {
|
||||
LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
|
||||
LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
|
||||
LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
|
||||
LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
|
||||
LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
|
||||
LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_decode_stream(
|
||||
&self,
|
||||
_prompt_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
_min_bytes_to_buffer: usize,
|
||||
) -> Box<dyn IncrementalDecoder + '_> {
|
||||
unreachable!("not used")
|
||||
}
|
||||
}
|
||||
|
||||
fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
|
||||
let context = MultimodalModelContext {
|
||||
model_id: format!("{model_type}-test"),
|
||||
model_type: Some(model_type.to_string()),
|
||||
config,
|
||||
tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
|
||||
};
|
||||
let spec = context
|
||||
.resolve_model_spec()
|
||||
.unwrap_or_else(|| panic!("{model_type} spec should match"));
|
||||
let spec = ResolvedMultimodalSpec::new(spec, &context).unwrap();
|
||||
let raw_image_processor = context
|
||||
.resolve_image_processor()
|
||||
.unwrap_or_else(|| panic!("{model_type} image processor should match"));
|
||||
let media_connector = Arc::new(
|
||||
MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default()).unwrap(),
|
||||
);
|
||||
|
||||
MultimodalModelInfo {
|
||||
context,
|
||||
spec,
|
||||
image_processor: ResolvedImageProcessor {
|
||||
raw: raw_image_processor,
|
||||
config: PreProcessorConfig::default(),
|
||||
},
|
||||
media_connector,
|
||||
}
|
||||
}
|
||||
|
||||
fn llama4_info() -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
"model_type": "llama4",
|
||||
"image_token_index": LLAMA4_PATCH_ID,
|
||||
"vision_config": {"image_size": 336, "patch_size": 14}
|
||||
});
|
||||
test_info("llama4", config)
|
||||
}
|
||||
|
||||
fn llama4_single_tile_replacement() -> PromptReplacement {
|
||||
PromptReplacement::sequence(
|
||||
Modality::Image,
|
||||
"<|image|>",
|
||||
vec![
|
||||
LLAMA4_IMAGE_START_ID as TokenId,
|
||||
LLAMA4_IMAGE_ID as TokenId,
|
||||
LLAMA4_PATCH_ID as TokenId,
|
||||
LLAMA4_PATCH_ID as TokenId,
|
||||
LLAMA4_IMAGE_END_ID as TokenId,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn llama4_multi_tile_replacement() -> PromptReplacement {
|
||||
PromptReplacement::sequence(
|
||||
Modality::Image,
|
||||
"<|image|>",
|
||||
vec![
|
||||
LLAMA4_IMAGE_START_ID as TokenId,
|
||||
LLAMA4_PATCH_ID as TokenId,
|
||||
LLAMA4_TILE_X_SEPARATOR_ID as TokenId,
|
||||
LLAMA4_PATCH_ID as TokenId,
|
||||
LLAMA4_TILE_Y_SEPARATOR_ID as TokenId,
|
||||
LLAMA4_IMAGE_ID as TokenId,
|
||||
LLAMA4_PATCH_ID as TokenId,
|
||||
LLAMA4_IMAGE_END_ID as TokenId,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) {
|
||||
let tensor = range.is_embed.as_ref().expect("is_embed mask");
|
||||
assert_eq!(tensor.dtype, "bool");
|
||||
assert_eq!(tensor.shape, vec![expected.len()]);
|
||||
assert_eq!(
|
||||
tensor.data,
|
||||
WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() {
|
||||
let info = llama4_info();
|
||||
let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
|
||||
let replacements = vec![llama4_multi_tile_replacement()];
|
||||
|
||||
let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
LLAMA4_IMAGE_START_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_TILE_X_SEPARATOR_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_TILE_Y_SEPARATOR_ID,
|
||||
LLAMA4_IMAGE_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_IMAGE_END_ID,
|
||||
2,
|
||||
]
|
||||
);
|
||||
assert_eq!(ranges[0].offset, 1);
|
||||
assert_eq!(ranges[0].length, 8);
|
||||
assert_bool_mask(
|
||||
&ranges[0],
|
||||
&[false, true, false, true, false, false, true, false],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_errors_when_placeholder_missing() {
|
||||
let info = llama4_info();
|
||||
let mut prompt_token_ids = vec![1, 2, 3];
|
||||
let replacements = vec![llama4_single_tile_replacement()];
|
||||
|
||||
let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
|
||||
|
||||
assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() {
|
||||
let info = llama4_info();
|
||||
let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3];
|
||||
let replacements = vec![
|
||||
llama4_single_tile_replacement(),
|
||||
llama4_single_tile_replacement(),
|
||||
];
|
||||
|
||||
let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
LLAMA4_IMAGE_START_ID,
|
||||
LLAMA4_IMAGE_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_IMAGE_END_ID,
|
||||
2,
|
||||
LLAMA4_IMAGE_START_ID,
|
||||
LLAMA4_IMAGE_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_IMAGE_END_ID,
|
||||
3,
|
||||
]
|
||||
);
|
||||
assert_eq!(ranges[0].offset, 1);
|
||||
assert_eq!(ranges[0].length, 5);
|
||||
assert_bool_mask(&ranges[0], &[false, false, true, true, false]);
|
||||
assert_eq!(ranges[1].offset, 7);
|
||||
assert_eq!(ranges[1].length, 5);
|
||||
assert_bool_mask(&ranges[1], &[false, false, true, true, false]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use half::{bf16, f16};
|
||||
use llm_multimodal::{ModelSpecificValue, PreprocessedImages};
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
|
||||
use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
|
||||
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Representation for multimodal kwarg values for transformation.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) enum KwargValue {
|
||||
/// Float tensor with row-major flat data and shape.
|
||||
F32Tensor { data: Vec<f32>, shape: Vec<usize> },
|
||||
/// Float16 tensor with row-major flat data and shape.
|
||||
F16Tensor { data: Vec<f16>, shape: Vec<usize> },
|
||||
/// BFloat16 tensor with row-major flat data and shape.
|
||||
Bf16Tensor { data: Vec<bf16>, shape: Vec<usize> },
|
||||
/// Signed integer tensor with row-major flat data and shape.
|
||||
I64Tensor { data: Vec<i64>, shape: Vec<usize> },
|
||||
/// Unsigned integer tensor with row-major flat data and shape.
|
||||
U32Tensor { data: Vec<u32>, shape: Vec<usize> },
|
||||
/// Non-tensor kwarg value that is shared or copied as-is.
|
||||
Passthrough(ProtocolKwargValue),
|
||||
}
|
||||
|
||||
/// Collect `pixel_values` and model-specific outputs into one tensor map.
|
||||
pub(super) fn collect_tensors(
|
||||
preprocessed: PreprocessedImages,
|
||||
float_dtype: ModelDtype,
|
||||
) -> Result<HashMap<String, KwargValue>> {
|
||||
let PreprocessedImages {
|
||||
pixel_values,
|
||||
model_specific,
|
||||
..
|
||||
} = preprocessed;
|
||||
|
||||
let pixel_values = {
|
||||
let shape = pixel_values.shape().to_vec();
|
||||
let data = pixel_values.into_iter().collect();
|
||||
KwargValue::from_f32_tensor(data, shape, float_dtype)?
|
||||
};
|
||||
|
||||
let mut tensors = HashMap::new();
|
||||
tensors.insert("pixel_values".to_string(), pixel_values);
|
||||
for (key, value) in model_specific {
|
||||
tensors.insert(key, KwargValue::from_model_specific(value, float_dtype)?);
|
||||
}
|
||||
Ok(tensors)
|
||||
}
|
||||
|
||||
impl KwargValue {
|
||||
fn from_model_specific(value: ModelSpecificValue, float_dtype: ModelDtype) -> Result<Self> {
|
||||
use ProtocolKwargValue::*;
|
||||
|
||||
Ok(match value {
|
||||
ModelSpecificValue::Tensor { data, shape } => {
|
||||
Self::from_f32_tensor(data, shape, float_dtype)?
|
||||
}
|
||||
ModelSpecificValue::IntTensor { data, shape } => Self::I64Tensor { data, shape },
|
||||
ModelSpecificValue::UintTensor { data, shape } => Self::U32Tensor { data, shape },
|
||||
ModelSpecificValue::Int(value) => Self::Passthrough(Int(value)),
|
||||
ModelSpecificValue::Float(value) => Self::Passthrough(Float(value)),
|
||||
ModelSpecificValue::IntVec(values) => {
|
||||
Self::Passthrough(List(values.into_iter().map(Int).collect()))
|
||||
}
|
||||
ModelSpecificValue::UintVec(values) => Self::Passthrough(List(
|
||||
values.into_iter().map(|value| Int(value as i64)).collect(),
|
||||
)),
|
||||
ModelSpecificValue::FloatVec(values) => Self::Passthrough(List(
|
||||
values.into_iter().map(|value| Float(value as f64)).collect(),
|
||||
)),
|
||||
ModelSpecificValue::TupleVec(values) => Self::Passthrough(List(
|
||||
values
|
||||
.into_iter()
|
||||
.map(|(height, width)| List(vec![Int(height as i64), Int(width as i64)]))
|
||||
.collect(),
|
||||
)),
|
||||
ModelSpecificValue::Bool(value) => Self::Passthrough(Int(i64::from(value))),
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a float tensor to the target float dtype if needed, keeping the
|
||||
/// same shape.
|
||||
fn from_f32_tensor(data: Vec<f32>, shape: Vec<usize>, float_dtype: ModelDtype) -> Result<Self> {
|
||||
match float_dtype {
|
||||
ModelDtype::Float16 => Ok(Self::F16Tensor {
|
||||
data: data.into_iter().map(f16::from_f32).collect(),
|
||||
shape,
|
||||
}),
|
||||
ModelDtype::BFloat16 => Ok(Self::Bf16Tensor {
|
||||
data: data.into_iter().map(bf16::from_f32).collect(),
|
||||
shape,
|
||||
}),
|
||||
ModelDtype::Float32 => Ok(Self::F32Tensor { data, shape }),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<KwargValue> for ProtocolKwargValue {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(value: KwargValue) -> Result<Self> {
|
||||
match value {
|
||||
KwargValue::F32Tensor { data, shape } => Ok(Self::Tensor(
|
||||
WireTensor::from_f32(shape, data).map_err(Error::Multimodal)?,
|
||||
)),
|
||||
KwargValue::F16Tensor { data, shape } => Ok(Self::Tensor(
|
||||
WireTensor::from_f16(shape, data).map_err(Error::Multimodal)?,
|
||||
)),
|
||||
KwargValue::Bf16Tensor { data, shape } => Ok(Self::Tensor(
|
||||
WireTensor::from_bf16(shape, data).map_err(Error::Multimodal)?,
|
||||
)),
|
||||
KwargValue::I64Tensor { data, shape } => Ok(Self::Tensor(
|
||||
WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?,
|
||||
)),
|
||||
KwargValue::U32Tensor { data, shape } => Ok(Self::Tensor(
|
||||
WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?,
|
||||
)),
|
||||
KwargValue::Passthrough(value) => Ok(value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl KwargValue {
|
||||
/// Extract one image from a batched tensor field.
|
||||
///
|
||||
/// Batched fields use their first axis as image index and drop that axis in
|
||||
/// the per-feature value, matching vLLM's batched-field semantics.
|
||||
pub(super) fn batched_value_at(&self, index: usize) -> Result<Self> {
|
||||
match self {
|
||||
Self::F32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?;
|
||||
Ok(Self::F32Tensor { data, shape })
|
||||
}
|
||||
Self::F16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?;
|
||||
Ok(Self::F16Tensor { data, shape })
|
||||
}
|
||||
Self::Bf16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?;
|
||||
Ok(Self::Bf16Tensor { data, shape })
|
||||
}
|
||||
Self::I64Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?;
|
||||
Ok(Self::I64Tensor { data, shape })
|
||||
}
|
||||
Self::U32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?;
|
||||
Ok(Self::U32Tensor { data, shape })
|
||||
}
|
||||
Self::Passthrough(value) => Ok(Self::Passthrough(value.clone())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract one image's variable-length range from a flat tensor field.
|
||||
///
|
||||
/// Flat fields keep the first axis as the sliced length for this image.
|
||||
pub(super) fn flat_value_range(&self, start: usize, end: usize) -> Result<Self> {
|
||||
match self {
|
||||
Self::F32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?;
|
||||
Ok(Self::F32Tensor { data, shape })
|
||||
}
|
||||
Self::F16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?;
|
||||
Ok(Self::F16Tensor { data, shape })
|
||||
}
|
||||
Self::Bf16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?;
|
||||
Ok(Self::Bf16Tensor { data, shape })
|
||||
}
|
||||
Self::I64Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?;
|
||||
Ok(Self::I64Tensor { data, shape })
|
||||
}
|
||||
Self::U32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?;
|
||||
Ok(Self::U32Tensor { data, shape })
|
||||
}
|
||||
Self::Passthrough(value) => Ok(Self::Passthrough(value.clone())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the first-axis range for one image in a flat tensor.
|
||||
///
|
||||
/// `sizes_key` names a companion tensor whose entries are cumulative slice
|
||||
/// sizes per image.
|
||||
pub(super) fn flat_range_for_index(
|
||||
sizes: &KwargValue,
|
||||
sizes_key: &str,
|
||||
index: usize,
|
||||
) -> Result<(usize, usize)> {
|
||||
let sizes = tensor_as_usize_vec(sizes)?;
|
||||
let size = *sizes.get(index).ok_or_else(|| {
|
||||
multimodal!("flat tensor sizes key `{sizes_key}` has no entry for image {index}")
|
||||
})?;
|
||||
let start = sizes[..index].iter().sum::<usize>();
|
||||
Ok((start, start + size))
|
||||
}
|
||||
|
||||
/// Read a tensor value as per-image sizes for flat slicing.
|
||||
fn tensor_as_usize_vec(tensor: &KwargValue) -> Result<Vec<usize>> {
|
||||
match tensor {
|
||||
KwargValue::I64Tensor { data, .. } => data
|
||||
.iter()
|
||||
.map(|value| {
|
||||
usize::try_from(*value)
|
||||
.map_err(|_| multimodal!("negative flat tensor size `{value}`"))
|
||||
})
|
||||
.collect(),
|
||||
KwargValue::U32Tensor { data, .. } => {
|
||||
Ok(data.iter().map(|value| *value as usize).collect())
|
||||
}
|
||||
_ => Err(multimodal!("flat tensor sizes must be int64 or uint32")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Slice a flat row-major tensor along its first axis.
|
||||
fn slice_first_axis_range<T: Clone>(
|
||||
shape: &[usize],
|
||||
data: &[T],
|
||||
start: usize,
|
||||
end: usize,
|
||||
drop_axis: bool,
|
||||
) -> Result<(Vec<usize>, Vec<T>)> {
|
||||
let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?;
|
||||
if start > end || end > first_dim {
|
||||
bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}");
|
||||
}
|
||||
let expected_len = shape
|
||||
.checked_numel()
|
||||
.ok_or_else(|| multimodal!("tensor shape {shape:?} has too many elements"))?;
|
||||
if expected_len != data.len() {
|
||||
bail_multimodal!(
|
||||
"tensor shape {shape:?} expects {expected_len} elements, got {}",
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
let stride = shape[1..].iter().product::<usize>();
|
||||
let data_start = start * stride;
|
||||
let data_end = end * stride;
|
||||
let out_shape = if drop_axis {
|
||||
shape[1..].to_vec()
|
||||
} else {
|
||||
let mut shape = shape.to_vec();
|
||||
shape[0] = end - start;
|
||||
shape
|
||||
};
|
||||
Ok((out_shape, data[data_start..data_end].to_vec()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn batched_value_at_drops_first_axis() {
|
||||
let value = KwargValue::F32Tensor {
|
||||
data: vec![1.0, 2.0, 3.0, 4.0],
|
||||
shape: vec![2, 2],
|
||||
};
|
||||
|
||||
let value = value.batched_value_at(1).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
value,
|
||||
KwargValue::F32Tensor { data, shape }
|
||||
if shape == vec![2] && data == vec![3.0, 4.0]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_value_range_keeps_first_axis() {
|
||||
let value = KwargValue::U32Tensor {
|
||||
data: (0..10).collect(),
|
||||
shape: vec![5, 2],
|
||||
};
|
||||
|
||||
let value = value.flat_value_range(1, 3).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
value,
|
||||
KwargValue::U32Tensor { data, shape }
|
||||
if shape == vec![2, 2] && data == vec![2, 3, 4, 5]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_range_for_index_uses_size_tensor() {
|
||||
let sizes = KwargValue::I64Tensor {
|
||||
data: vec![2, 3, 4],
|
||||
shape: vec![3],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
flat_range_for_index(&sizes, "image_grid_thw", 1).unwrap(),
|
||||
(2, 5)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slice_first_axis_range_errors_on_shape_data_mismatch() {
|
||||
let error = slice_first_axis_range(&[2, 2], &[1.0_f32, 2.0, 3.0], 0, 1, true).unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(error, Error::Multimodal(message) if message.contains("expects 4 elements"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bfloat16_tensor_wire_uses_bfloat16_dtype() {
|
||||
let value =
|
||||
KwargValue::from_f32_tensor(vec![1.0, -1.0], vec![2], ModelDtype::BFloat16).unwrap();
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(value).unwrap()
|
||||
else {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
|
||||
assert_eq!(tensor.dtype, "bfloat16");
|
||||
assert_eq!(tensor.shape, vec![2]);
|
||||
assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float16_tensor_wire_uses_float16_dtype() {
|
||||
let value =
|
||||
KwargValue::from_f32_tensor(vec![1.0, -1.0], vec![2], ModelDtype::Float16).unwrap();
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(value).unwrap()
|
||||
else {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
|
||||
assert_eq!(tensor.dtype, "float16");
|
||||
assert_eq!(tensor.shape, vec![2]);
|
||||
assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
//! Default output processing pipeline.
|
||||
|
||||
mod reasoning;
|
||||
mod tool;
|
||||
|
||||
use std::sync::Once;
|
||||
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use tracing::info;
|
||||
use trait_set::trait_set;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
|
||||
use self::reasoning::reasoning_event_stream;
|
||||
use self::tool::tool_event_stream;
|
||||
use super::structured::structured_chat_event_stream;
|
||||
use crate::error::Result;
|
||||
use crate::output::{
|
||||
AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream,
|
||||
DynDecodedTextEventStream,
|
||||
};
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
trait_set! {
|
||||
trait ContentEventStream = Stream<Item = Result<ContentEvent>> + Send + 'static;
|
||||
}
|
||||
|
||||
/// Default request-scoped output processor used by Hugging Face style chat
|
||||
/// backends.
|
||||
///
|
||||
/// This implementation assumes the backend already emitted decoded text deltas,
|
||||
/// then optionally layers reasoning parsing and tool-call parsing before
|
||||
/// assembling final structured chat events.
|
||||
pub struct DefaultChatOutputProcessor {
|
||||
reasoning_parser: Option<Box<dyn ReasoningParser>>,
|
||||
tool_parser: Option<Box<dyn ToolParser>>,
|
||||
}
|
||||
|
||||
impl DefaultChatOutputProcessor {
|
||||
/// Build the default output processor and apply any parser-specific request
|
||||
/// adjustments.
|
||||
///
|
||||
/// Parser resolution happens here so that request validation, prompt
|
||||
/// rendering, and streaming all observe the same parser-adjusted
|
||||
/// request state.
|
||||
pub fn new(
|
||||
request: &mut ChatRequest,
|
||||
model_id: &str,
|
||||
tokenizer: DynTokenizer,
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
) -> ChatResult<Self> {
|
||||
let tool_parsing_enabled =
|
||||
matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty();
|
||||
let tool_parser = if tool_parsing_enabled {
|
||||
Some(Self::resolve_tool_parser(
|
||||
request,
|
||||
model_id,
|
||||
tool_call_parser,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let reasoning_parser = Self::resolve_optional_reasoning_parser(
|
||||
request,
|
||||
model_id,
|
||||
tokenizer,
|
||||
reasoning_parser,
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
reasoning_parser,
|
||||
tool_parser,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build the plain-text-only default output processor.
|
||||
///
|
||||
/// This keeps the default structured chat-event assembly but disables both
|
||||
/// reasoning parsing and tool-call parsing completely, so that all
|
||||
/// content is treated as opaque text.
|
||||
pub fn plain_text_only() -> Self {
|
||||
Self {
|
||||
reasoning_parser: None,
|
||||
tool_parser: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_tool_parser(
|
||||
request: &mut ChatRequest,
|
||||
model_id: &str,
|
||||
selection: &ParserSelection,
|
||||
) -> ChatResult<Box<dyn ToolParser>> {
|
||||
let factory = ToolParserFactory::global();
|
||||
let parser_name = match selection {
|
||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id).ok_or_else(|| {
|
||||
Error::ParserUnavailableForModel {
|
||||
kind: "tool",
|
||||
model_id: model_id.to_string(),
|
||||
}
|
||||
})?,
|
||||
ParserSelection::None => return Err(Error::ParserDisabled { kind: "tool" }),
|
||||
ParserSelection::Explicit(name) => name.as_str(),
|
||||
};
|
||||
|
||||
let parser = factory.create(parser_name, &request.tools)?;
|
||||
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
}
|
||||
|
||||
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
|
||||
Ok(parser)
|
||||
}
|
||||
|
||||
fn resolve_optional_reasoning_parser(
|
||||
request: &mut ChatRequest,
|
||||
model_id: &str,
|
||||
tokenizer: DynTokenizer,
|
||||
selection: &ParserSelection,
|
||||
) -> ChatResult<Option<Box<dyn ReasoningParser>>> {
|
||||
let factory = ReasoningParserFactory::global();
|
||||
let parser_name = match selection {
|
||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id),
|
||||
ParserSelection::None => None,
|
||||
ParserSelection::Explicit(name) => Some(name.as_str()),
|
||||
};
|
||||
|
||||
let Some(parser_name) = parser_name else {
|
||||
REASONING_PARSER_LOG_ONCE.call_once(|| info!("reasoning parsing disabled"));
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let parser = factory.create(parser_name, tokenizer)?;
|
||||
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
}
|
||||
|
||||
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
|
||||
Ok(Some(parser))
|
||||
}
|
||||
}
|
||||
|
||||
static TOOL_PARSER_LOG_ONCE: Once = Once::new();
|
||||
static REASONING_PARSER_LOG_ONCE: Once = Once::new();
|
||||
|
||||
impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||
/// Transforms a raw generate-output token stream into structured chat
|
||||
/// events through three sequential stages once text decoding has
|
||||
/// already happened:
|
||||
///
|
||||
/// 1. [`reasoning_event_stream`] — reasoning/content separation
|
||||
/// 2. [`tool_event_stream`] — tool-call parsing
|
||||
/// 3. [`structured_chat_event_stream`] — final block assembly
|
||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
||||
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
|
||||
let tool = tool_event_stream(reasoning, self.tool_parser);
|
||||
let structured = structured_chat_event_stream(tool);
|
||||
|
||||
Ok(structured.boxed())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
//! Adapts decoded text updates into reasoning-aware assistant deltas.
|
||||
//!
|
||||
//! This stage sits between low-level token decoding and final block assembly.
|
||||
//! It is the only place in the new pipeline that understands reasoning
|
||||
//! separation: `decoded.rs` still only produces plain text deltas, while later
|
||||
//! stages consume the semantic `Text` / `Reasoning` split emitted here.
|
||||
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::{StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport;
|
||||
use tracing::warn;
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
|
||||
use super::ContentEvent;
|
||||
use crate::Result;
|
||||
use crate::error::Error;
|
||||
use crate::event::AssistantBlockKind;
|
||||
use crate::output::DecodedTextEventStream;
|
||||
use crate::parser::reasoning::{ReasoningDelta, ReasoningParser};
|
||||
|
||||
/// Per-stream reasoning parsing state.
|
||||
struct ReasoningState {
|
||||
/// Reasoning parser for the current model family.
|
||||
parser: Box<dyn ReasoningParser>,
|
||||
/// Whether reasoning parsing has already failed for this stream.
|
||||
parser_failed: bool,
|
||||
}
|
||||
|
||||
impl ReasoningState {
|
||||
/// Create one fresh reasoning-adaptation state for a new streamed response.
|
||||
fn new(parser: Box<dyn ReasoningParser>) -> Self {
|
||||
Self {
|
||||
parser,
|
||||
parser_failed: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one decoded text delta into zero or more semantic assistant
|
||||
/// deltas.
|
||||
fn process_delta(&mut self, delta: String) -> Vec<ContentEvent> {
|
||||
// If the parser has already failed, skip parsing and return plain text deltas.
|
||||
if self.parser_failed {
|
||||
return vec![ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta,
|
||||
}];
|
||||
}
|
||||
|
||||
let mut events = Vec::new();
|
||||
|
||||
match self.parser.push(&delta) {
|
||||
Ok(result) => {
|
||||
push_reasoning_delta(&mut events, result);
|
||||
}
|
||||
Err(error) => {
|
||||
if !self.parser_failed {
|
||||
warn!(
|
||||
error = %error.as_report(),
|
||||
"reasoning parser failed; falling back to plain text deltas"
|
||||
);
|
||||
self.parser_failed = true;
|
||||
}
|
||||
push_text_delta(&mut events, AssistantBlockKind::Text, delta);
|
||||
}
|
||||
}
|
||||
|
||||
events
|
||||
}
|
||||
|
||||
/// Initialize parser state once prompt token IDs are available.
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) {
|
||||
if self.parser_failed {
|
||||
return;
|
||||
}
|
||||
|
||||
match self.parser.initialize(prompt_token_ids) {
|
||||
Ok(()) => {}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error.as_report(),
|
||||
"failed to initialize reasoning parser; falling back to plain text deltas"
|
||||
);
|
||||
self.parser_failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Flush any parser-held partial delimiter state at end of stream.
|
||||
fn finish(&mut self) -> Vec<ContentEvent> {
|
||||
if self.parser_failed {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
match self.parser.finish() {
|
||||
Ok(result) => {
|
||||
let mut events = Vec::new();
|
||||
push_reasoning_delta(&mut events, result);
|
||||
events
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error.as_report(), "failed to flush reasoning parser state");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one semantic text delta if it is non-empty.
|
||||
fn push_text_delta(events: &mut Vec<ContentEvent>, kind: AssistantBlockKind, delta: String) {
|
||||
if delta.is_empty() {
|
||||
return;
|
||||
}
|
||||
events.push(ContentEvent::TextDelta { kind, delta });
|
||||
}
|
||||
|
||||
/// Convert one parsed reasoning delta into zero or more content events.
|
||||
fn push_reasoning_delta(events: &mut Vec<ContentEvent>, delta: ReasoningDelta) {
|
||||
if let Some(reasoning) = delta.reasoning {
|
||||
push_text_delta(events, AssistantBlockKind::Reasoning, reasoning);
|
||||
}
|
||||
if let Some(content) = delta.content {
|
||||
push_text_delta(events, AssistantBlockKind::Text, content);
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap one decoded-text stream into the internal reasoning event stream.
|
||||
#[try_stream]
|
||||
pub(crate) async fn reasoning_event_stream(
|
||||
decoded_stream: impl DecodedTextEventStream,
|
||||
reasoning_parser: Option<Box<dyn ReasoningParser>>,
|
||||
mut y: TryYielder<ContentEvent, Error>,
|
||||
) -> Result<()> {
|
||||
pin_mut!(decoded_stream);
|
||||
|
||||
// Without a parser, pass through as plain text deltas.
|
||||
let Some(reasoning_parser) = reasoning_parser else {
|
||||
while let Some(event) = decoded_stream.next().await.transpose()? {
|
||||
for next in ContentEvent::from_decoded_plain_text(event) {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut state = ReasoningState::new(reasoning_parser);
|
||||
|
||||
while let Some(event) = decoded_stream.next().await.transpose()? {
|
||||
match event {
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
} => {
|
||||
state.initialize(&prompt_token_ids);
|
||||
y.yield_ok(ContentEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
} => {
|
||||
for next in state.process_delta(delta) {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
if logprobs.is_some() || !token_ids.is_empty() {
|
||||
y.yield_ok(ContentEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
if let Some(finished) = finished {
|
||||
for next in state.finish() {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
y.yield_ok(ContentEvent::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use vllm_llm::FinishReason;
|
||||
use vllm_text::output::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
|
||||
};
|
||||
use vllm_tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use super::super::ContentEvent;
|
||||
use super::reasoning_event_stream;
|
||||
use crate::event::AssistantBlockKind;
|
||||
use crate::parser::reasoning::{
|
||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names,
|
||||
};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<think>" => Some(1),
|
||||
"</think>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingReasoningParser {
|
||||
fail_next: bool,
|
||||
}
|
||||
|
||||
impl ReasoningParser for FailingReasoningParser {
|
||||
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>, ReasoningError>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self { fail_next: true }))
|
||||
}
|
||||
|
||||
fn push(&mut self, _text: &str) -> Result<ReasoningDelta, ReasoningError> {
|
||||
if self.fail_next {
|
||||
self.fail_next = false;
|
||||
return Err(ReasoningError::MissingToken {
|
||||
token: "<think>".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(ReasoningDelta::default())
|
||||
}
|
||||
}
|
||||
|
||||
fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box<dyn ReasoningParser> {
|
||||
factory.register_parser::<FailingReasoningParser>("failing");
|
||||
|
||||
factory.create("failing", Arc::new(FakeTokenizer)).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_parser_failure_falls_back_to_plain_text() {
|
||||
let mut factory = ReasoningParserFactory::new();
|
||||
let events = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1, 2, 3].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "abc".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "def".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: Some(vllm_text::Finished {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory)))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let events = collected
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.expect("reasoning stream should not fail");
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
ContentEvent::Start {
|
||||
prompt_token_ids: vec![1, 2, 3].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "abc".to_string(),
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "def".to_string(),
|
||||
},
|
||||
ContentEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_stream_preserves_logprobs_delta() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "abc".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
finished: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let collected = reasoning_event_stream(events, None)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
collected,
|
||||
vec![
|
||||
ContentEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "abc".to_string(),
|
||||
},
|
||||
ContentEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let events = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "thought ".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "done</think>OK".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let factory = ReasoningParserFactory::new();
|
||||
let collected = reasoning_event_stream(
|
||||
events,
|
||||
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let events = collected
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.expect("reasoning stream should not fail");
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
ContentEvent::Start {
|
||||
prompt_token_ids: vec![2].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "thought ".to_string(),
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "done</think>OK".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn qwen3_parser_tolerates_prompt_prefill_reasoning() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let events = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "thought ".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "done</think>OK".to_string(),
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let factory = ReasoningParserFactory::new();
|
||||
let collected = reasoning_event_stream(
|
||||
events,
|
||||
Some(factory.create(names::QWEN3, tokenizer).unwrap()),
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let events = collected
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.expect("reasoning stream should not fail");
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
ContentEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "thought ".to_string(),
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "done".to_string(),
|
||||
},
|
||||
ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "OK".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
//! Adapts plain assistant text deltas into tool-call-aware assistant updates.
|
||||
//!
|
||||
//! This stage runs after reasoning separation and before final block assembly.
|
||||
//! It only inspects normal assistant text, leaves reasoning deltas untouched,
|
||||
//! and translates incremental tool parsing output into internal tool-call
|
||||
//! events while preserving plain-text fallback behavior.
|
||||
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::{StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport;
|
||||
use tracing::warn;
|
||||
|
||||
use super::{AssistantEvent, ContentEvent, ContentEventStream};
|
||||
use crate::Result;
|
||||
use crate::error::Error;
|
||||
use crate::event::AssistantBlockKind;
|
||||
use crate::output::generate_tool_call_id;
|
||||
use crate::parser::tool::{ToolCallDelta, ToolParseResult, ToolParser};
|
||||
|
||||
/// Per-stream tool parsing state.
|
||||
struct ToolState {
|
||||
/// Parser for the current model family.
|
||||
parser: Box<dyn ToolParser>,
|
||||
/// Whether tool parsing has already failed for this stream.
|
||||
parser_failed: bool,
|
||||
/// The parser-local index of the currently open tool call, if any.
|
||||
// NOTE: We only allow single open tool call at a time right now, since that's what all
|
||||
// supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple
|
||||
// interleaved calls in the future.
|
||||
open_call_index: Option<usize>,
|
||||
}
|
||||
|
||||
impl ToolState {
|
||||
/// Create one fresh tool-parsing state for a new streamed response.
|
||||
fn new(parser: Box<dyn ToolParser>) -> Self {
|
||||
Self {
|
||||
parser,
|
||||
parser_failed: false,
|
||||
open_call_index: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one semantic assistant text delta into zero or more tool-aware
|
||||
/// internal events.
|
||||
fn process_text_delta(
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
) -> Result<Vec<AssistantEvent>> {
|
||||
let mut events = Vec::new();
|
||||
|
||||
// Only normal assistant text is eligible for tool parsing. Reasoning
|
||||
// blocks and plain-text fallback should pass through unchanged.
|
||||
if kind != AssistantBlockKind::Text || self.parser_failed {
|
||||
self.open_call_index = None;
|
||||
events.push(AssistantEvent::TextDelta { kind, delta });
|
||||
return Ok(events);
|
||||
}
|
||||
|
||||
let parse_result = self.parser.push(&delta);
|
||||
|
||||
match parse_result {
|
||||
Ok(result) => self.process_parse_result(kind, result, &mut events)?,
|
||||
Err(error) => {
|
||||
if !self.parser_failed {
|
||||
warn!(
|
||||
error = %error.as_report(),
|
||||
"tool parser failed; falling back to plain text deltas"
|
||||
);
|
||||
self.parser_failed = true;
|
||||
}
|
||||
self.open_call_index = None;
|
||||
events.push(AssistantEvent::TextDelta { kind, delta });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Apply one parsed tool result to the current stream state.
|
||||
fn process_parse_result(
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
result: ToolParseResult,
|
||||
events: &mut Vec<AssistantEvent>,
|
||||
) -> Result<()> {
|
||||
// When we are not currently streaming a tool call, preserve plain
|
||||
// text first and then surface any new tool call items.
|
||||
if self.open_call_index.is_none() {
|
||||
push_text_delta(events, kind, result.normal_text);
|
||||
self.process_tool_items(result.calls, events)?;
|
||||
} else {
|
||||
// Once a tool call is open, prioritize tool deltas first. If the
|
||||
// parser emits normal text again, close the tool call and resume
|
||||
// plain text output.
|
||||
self.process_tool_items(result.calls, events)?;
|
||||
if !result.normal_text.is_empty() {
|
||||
self.open_call_index = None;
|
||||
push_text_delta(events, kind, result.normal_text);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one batch of parsed tool-call deltas emitted by the parser.
|
||||
fn process_tool_items(
|
||||
&mut self,
|
||||
items: Vec<ToolCallDelta>,
|
||||
events: &mut Vec<AssistantEvent>,
|
||||
) -> Result<()> {
|
||||
for item in items {
|
||||
if let Some(name) = item.name {
|
||||
let is_new_tool = match self.open_call_index {
|
||||
Some(open_call_index) => open_call_index != item.tool_index,
|
||||
None => true,
|
||||
};
|
||||
if is_new_tool {
|
||||
let id = generate_tool_call_id();
|
||||
self.open_call_index = Some(item.tool_index);
|
||||
events.push(AssistantEvent::ToolCallStart { id, name });
|
||||
}
|
||||
}
|
||||
|
||||
if item.arguments.is_empty() {
|
||||
// No arguments delta to apply.
|
||||
continue;
|
||||
}
|
||||
let Some(open_call_index) = self.open_call_index else {
|
||||
return Err(Error::ToolCallStreamInvariant {
|
||||
message: format!(
|
||||
"received arguments for tool index {} before any tool-call start",
|
||||
item.tool_index
|
||||
),
|
||||
});
|
||||
};
|
||||
if open_call_index != item.tool_index {
|
||||
return Err(Error::ToolCallStreamInvariant {
|
||||
message: format!(
|
||||
"received arguments for tool index {} while tool index {} is open",
|
||||
item.tool_index, open_call_index
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
events.push(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: item.arguments,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush parser state at end-of-stream and close any remaining open calls.
|
||||
fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
|
||||
let mut events = Vec::new();
|
||||
|
||||
if self.parser_failed {
|
||||
return Ok(events);
|
||||
}
|
||||
|
||||
match self.parser.finish() {
|
||||
Ok(result) => {
|
||||
self.process_parse_result(AssistantBlockKind::Text, result, &mut events)?
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error.as_report(),
|
||||
"tool parser finish failed; closing open tool calls with buffered state"
|
||||
);
|
||||
self.parser_failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one plain-text delta if it is non-empty.
|
||||
fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
|
||||
if delta.is_empty() {
|
||||
return;
|
||||
}
|
||||
events.push(AssistantEvent::TextDelta { kind, delta });
|
||||
}
|
||||
|
||||
/// Wrap one semantic assistant stream into the internal tool-aware assistant
|
||||
/// stream.
|
||||
#[try_stream]
|
||||
pub(crate) async fn tool_event_stream(
|
||||
stream: impl ContentEventStream,
|
||||
parser: Option<Box<dyn ToolParser>>,
|
||||
mut y: TryYielder<AssistantEvent, Error>,
|
||||
) -> Result<()> {
|
||||
// Without a parser, pass through the input stream unchanged.
|
||||
let Some(parser) = parser else {
|
||||
pin_mut!(stream);
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
y.yield_ok(event.into()).await;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
pin_mut!(stream);
|
||||
let mut state = ToolState::new(parser);
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
ContentEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
} => {
|
||||
y.yield_ok(AssistantEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
ContentEvent::TextDelta { kind, delta } => {
|
||||
for next in state.process_text_delta(kind, delta)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
ContentEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
} => {
|
||||
y.yield_ok(AssistantEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
ContentEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
for next in state.finish()? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use vllm_llm::FinishReason;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
use vllm_tool_parser::Result;
|
||||
|
||||
use super::super::{AssistantEvent, ContentEvent};
|
||||
use super::tool_event_stream;
|
||||
use crate::error::Error;
|
||||
use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
|
||||
use crate::output::structured::structured_chat_event_stream;
|
||||
use crate::parser::tool::{ToolParseResult, ToolParser, ToolParserError};
|
||||
use crate::request::ChatTool;
|
||||
use crate::stream::ChatEventStream;
|
||||
|
||||
struct FailingParser {
|
||||
fail_next: bool,
|
||||
}
|
||||
|
||||
struct ScriptedParser {
|
||||
push_results: Vec<ToolParseResult>,
|
||||
finish_result: ToolParseResult,
|
||||
}
|
||||
|
||||
impl ToolParser for FailingParser {
|
||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self { fail_next: false }))
|
||||
}
|
||||
|
||||
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
|
||||
if self.fail_next {
|
||||
self.fail_next = false;
|
||||
return Err(ToolParserError::ParsingFailed {
|
||||
message: "boom".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ToolParseResult::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolParser for ScriptedParser {
|
||||
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self {
|
||||
push_results: Vec::new(),
|
||||
finish_result: ToolParseResult::default(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
|
||||
Ok(self.push_results.pop().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ToolParseResult> {
|
||||
Ok(std::mem::take(&mut self.finish_result))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_parser_failure_falls_back_to_plain_text() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(ContentEvent::Start {
|
||||
prompt_token_ids: vec![1, 2, 3].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "abc".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "def".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let collected =
|
||||
tool_event_stream(events, Some(Box::new(FailingParser { fail_next: true })))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let events = collected
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.expect("tool stream should not fail");
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
AssistantEvent::Start {
|
||||
prompt_token_ids: vec![1, 2, 3].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "abc".to_string(),
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "def".to_string(),
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
let message = ChatEventStream::new(
|
||||
"req_fallback".to_string(),
|
||||
Box::pin(structured_chat_event_stream(stream::iter(
|
||||
events.into_iter().map(Ok),
|
||||
))),
|
||||
)
|
||||
.collect_message()
|
||||
.await
|
||||
.expect("collect_message should succeed");
|
||||
assert_eq!(message.message.text(), "abcdef");
|
||||
assert!(message.message.tool_calls().next().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_stream_preserves_logprobs_delta() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(ContentEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ContentEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
let events = tool_event_stream(events, Some(Box::new(FailingParser { fail_next: false })))
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
AssistantEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
AssistantEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_stream_rejects_interleaved_tool_indices() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "ignored".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let parser = ScriptedParser {
|
||||
push_results: vec![ToolParseResult {
|
||||
normal_text: String::new(),
|
||||
calls: vec![
|
||||
crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: Some("first".to_string()),
|
||||
arguments: String::new(),
|
||||
},
|
||||
crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 1,
|
||||
name: None,
|
||||
arguments: "{}".to_string(),
|
||||
},
|
||||
],
|
||||
}],
|
||||
finish_result: ToolParseResult::default(),
|
||||
};
|
||||
|
||||
let err = tool_event_stream(events, Some(Box::new(parser)))
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.find_map(|result| result.err())
|
||||
.expect("expected invariant error");
|
||||
|
||||
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "start".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "text".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "args".to_string(),
|
||||
}),
|
||||
]);
|
||||
|
||||
let parser = ScriptedParser {
|
||||
push_results: vec![
|
||||
ToolParseResult {
|
||||
normal_text: String::new(),
|
||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: None,
|
||||
arguments: "}".to_string(),
|
||||
}],
|
||||
},
|
||||
ToolParseResult {
|
||||
normal_text: "plain text".to_string(),
|
||||
calls: Vec::new(),
|
||||
},
|
||||
ToolParseResult {
|
||||
normal_text: String::new(),
|
||||
calls: vec![crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: Some("first".to_string()),
|
||||
arguments: "{".to_string(),
|
||||
}],
|
||||
},
|
||||
],
|
||||
finish_result: ToolParseResult::default(),
|
||||
};
|
||||
|
||||
let err = tool_event_stream(events, Some(Box::new(parser)))
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.find_map(|result| result.err())
|
||||
.expect("expected invariant error");
|
||||
|
||||
assert!(matches!(
|
||||
err,
|
||||
Error::ToolCallStreamInvariant { message }
|
||||
if message == "received arguments for tool index 0 before any tool-call start"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_stream_emits_start_and_args_for_terminal_text() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(ContentEvent::Start {
|
||||
prompt_token_ids: vec![1].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ContentEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "ignored".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let parser = ScriptedParser {
|
||||
push_results: vec![ToolParseResult {
|
||||
normal_text: String::new(),
|
||||
calls: vec![
|
||||
crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: Some("first".to_string()),
|
||||
arguments: r#"{"a":1}"#.to_string(),
|
||||
},
|
||||
crate::parser::tool::ToolCallDelta {
|
||||
tool_index: 1,
|
||||
name: Some("second".to_string()),
|
||||
arguments: r#"{"b":2}"#.to_string(),
|
||||
},
|
||||
],
|
||||
}],
|
||||
finish_result: ToolParseResult::default(),
|
||||
};
|
||||
|
||||
let events = tool_event_stream(events, Some(Box::new(parser)))
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. }));
|
||||
assert!(matches!(
|
||||
events[2],
|
||||
AssistantEvent::ToolCallArgumentsDelta { .. }
|
||||
));
|
||||
assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. }));
|
||||
assert!(matches!(
|
||||
events[4],
|
||||
AssistantEvent::ToolCallArgumentsDelta { .. }
|
||||
));
|
||||
let collected = ChatEventStream::new(
|
||||
"req_final_only".to_string(),
|
||||
Box::pin(structured_chat_event_stream(stream::iter(
|
||||
events.into_iter().map(Ok),
|
||||
))),
|
||||
)
|
||||
.collect_message()
|
||||
.await
|
||||
.unwrap();
|
||||
let tool_calls = collected.message.tool_calls().collect::<Vec<_>>();
|
||||
assert_eq!(tool_calls.len(), 2);
|
||||
assert_eq!(tool_calls[0].name, "first");
|
||||
assert_eq!(tool_calls[1].name, "second");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
//! Native Harmony output processing for `gpt_oss`.
|
||||
//!
|
||||
//! Unlike the default text-first pipeline, this processor consumes
|
||||
//! `DecodedTextEvent` token IDs directly and lets the official `openai-harmony`
|
||||
//! parser recover the structured assistant message shape at token granularity.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::Context;
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::StreamExt as _;
|
||||
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
|
||||
use openai_harmony::{
|
||||
HarmonyEncoding, HarmonyEncodingName, StreamableParser, load_harmony_encoding,
|
||||
};
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
|
||||
use crate::Result as ChatResult;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::AssistantBlockKind;
|
||||
use crate::output::{
|
||||
AssistantEvent, ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream,
|
||||
generate_tool_call_id,
|
||||
};
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Request-scoped Harmony output processor used for `model_type == "gpt_oss"`.
|
||||
///
|
||||
/// This processor keeps the existing northbound `ChatEvent` shape, but swaps
|
||||
/// the parsed-assistant backend from generic text/reasoning/tool parsers to the
|
||||
/// official Harmony token parser.
|
||||
#[derive(Debug)]
|
||||
pub struct HarmonyChatOutputProcessor {
|
||||
encoding: &'static HarmonyEncoding,
|
||||
tool_calls_enabled: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct HarmonyGroupKey {
|
||||
serial: usize,
|
||||
channel: Option<String>,
|
||||
recipient: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HarmonyGroup {
|
||||
key: HarmonyGroupKey,
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OpenHarmonyToolCall {
|
||||
recipient: String,
|
||||
}
|
||||
|
||||
struct HarmonyState {
|
||||
/// Incremental Harmony parser over assistant token IDs.
|
||||
parser: StreamableParser,
|
||||
/// Whether tool-call content should surface as structured tool events.
|
||||
tool_calls_enabled: bool,
|
||||
/// Count of completed visible assistant messages for newline insertion.
|
||||
completed_visible_messages: usize,
|
||||
/// Count of completed reasoning messages for newline insertion.
|
||||
completed_reasoning_messages: usize,
|
||||
/// The current visible text/reasoning group, if any.
|
||||
current_text_group: Option<HarmonyGroupKey>,
|
||||
/// The currently open Harmony tool recipient, if any.
|
||||
open_tool_call: Option<OpenHarmonyToolCall>,
|
||||
}
|
||||
|
||||
impl HarmonyChatOutputProcessor {
|
||||
/// Build one request-scoped Harmony processor after backend policy checks.
|
||||
pub fn new(request: &ChatRequest) -> ChatResult<Self> {
|
||||
Ok(Self {
|
||||
encoding: harmony_encoding()?,
|
||||
tool_calls_enabled: request.tool_parsing_enabled(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that the generic parser selections are compatible with native
|
||||
/// Harmony output parsing.
|
||||
///
|
||||
/// `gpt_oss` uses a model-specific token-level parser, so any generic
|
||||
/// reasoning/tool parser override is rejected instead of being silently
|
||||
/// ignored.
|
||||
pub(crate) fn validate_harmony_parser_overrides(
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
) -> ChatResult<()> {
|
||||
validate_harmony_override("tool", tool_call_parser)?;
|
||||
validate_harmony_override("reasoning", reasoning_parser)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_harmony_override(kind: &'static str, selection: &ParserSelection) -> ChatResult<()> {
|
||||
if matches!(selection, ParserSelection::Auto) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(Error::HarmonyParserOverrideUnsupported {
|
||||
kind,
|
||||
selection: selection.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
impl ChatOutputProcessor for HarmonyChatOutputProcessor {
|
||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
||||
let assistant =
|
||||
harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled);
|
||||
Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed())
|
||||
}
|
||||
}
|
||||
|
||||
impl HarmonyState {
|
||||
/// Create one fresh Harmony streaming state for a new assistant response.
|
||||
fn new(encoding: HarmonyEncoding, tool_calls_enabled: bool) -> Result<Self> {
|
||||
Ok(Self {
|
||||
parser: StreamableParser::new(encoding, Some(Role::Assistant))
|
||||
.map_err(harmony_output_parsing_error)?,
|
||||
tool_calls_enabled,
|
||||
completed_visible_messages: 0,
|
||||
completed_reasoning_messages: 0,
|
||||
current_text_group: None,
|
||||
open_tool_call: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn process_token_ids(&mut self, token_ids: &[u32]) -> Result<Vec<AssistantEvent>> {
|
||||
let mut events = Vec::new();
|
||||
let mut pending_group: Option<HarmonyGroup> = None;
|
||||
|
||||
for &token_id in token_ids {
|
||||
let completed_before = self.parser.messages().len();
|
||||
self.parser.process(token_id).map_err(harmony_output_parsing_error)?;
|
||||
let completed_after = self.parser.messages().len();
|
||||
|
||||
if let Some(delta) = self
|
||||
.parser
|
||||
.last_content_delta()
|
||||
.map_err(harmony_output_parsing_error)?
|
||||
.filter(|delta| !delta.is_empty())
|
||||
{
|
||||
let key = HarmonyGroupKey {
|
||||
serial: completed_after,
|
||||
channel: self.parser.current_channel(),
|
||||
recipient: self.parser.current_recipient(),
|
||||
};
|
||||
|
||||
match pending_group.as_mut() {
|
||||
Some(group) if group.key == key => group.text.push_str(&delta),
|
||||
_ => {
|
||||
if let Some(group) = pending_group.take() {
|
||||
self.emit_group(group, &mut events);
|
||||
}
|
||||
pending_group = Some(HarmonyGroup { key, text: delta });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if completed_after > completed_before {
|
||||
if let Some(group) = pending_group.take() {
|
||||
self.emit_group(group, &mut events);
|
||||
}
|
||||
|
||||
for serial in completed_before..completed_after {
|
||||
let key = {
|
||||
let message = &self.parser.messages()[serial];
|
||||
HarmonyGroupKey {
|
||||
serial,
|
||||
channel: message.channel.clone(),
|
||||
recipient: message.recipient.clone(),
|
||||
}
|
||||
};
|
||||
self.handle_completed_message(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(group) = pending_group {
|
||||
self.emit_group(group, &mut events);
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Flush Harmony parser state at EOS and emit any newly finalized assistant
|
||||
/// events.
|
||||
fn process_eos(&mut self) -> Result<Vec<AssistantEvent>> {
|
||||
let completed_before = self.parser.messages().len();
|
||||
let pending_key = HarmonyGroupKey {
|
||||
serial: completed_before,
|
||||
channel: self.parser.current_channel(),
|
||||
recipient: self.parser.current_recipient(),
|
||||
};
|
||||
let pending_content =
|
||||
self.parser.current_content().map_err(harmony_output_parsing_error)?;
|
||||
|
||||
self.parser.process_eos().map_err(harmony_output_parsing_error)?;
|
||||
|
||||
let completed_after = self.parser.messages().len();
|
||||
let mut events = Vec::new();
|
||||
|
||||
if completed_after == completed_before {
|
||||
return Ok(events);
|
||||
}
|
||||
|
||||
let final_message = &self.parser.messages()[completed_before];
|
||||
let final_text = harmony_message_text(final_message);
|
||||
let tail = final_text.strip_prefix(&pending_content).unwrap_or(final_text).to_string();
|
||||
if !tail.is_empty() {
|
||||
self.emit_group(
|
||||
HarmonyGroup {
|
||||
key: pending_key,
|
||||
text: tail,
|
||||
},
|
||||
&mut events,
|
||||
);
|
||||
}
|
||||
|
||||
for serial in completed_before..completed_after {
|
||||
let key = {
|
||||
let message = &self.parser.messages()[serial];
|
||||
HarmonyGroupKey {
|
||||
serial,
|
||||
channel: message.channel.clone(),
|
||||
recipient: message.recipient.clone(),
|
||||
}
|
||||
};
|
||||
self.handle_completed_message(key);
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Flush one coalesced Harmony content group into internal assistant
|
||||
/// events.
|
||||
fn emit_group(&mut self, group: HarmonyGroup, events: &mut Vec<AssistantEvent>) {
|
||||
let channel = group.key.channel.as_deref();
|
||||
let recipient = group.key.recipient.as_deref();
|
||||
|
||||
if let Some(kind) = text_block_kind(channel, recipient) {
|
||||
self.open_tool_call = None;
|
||||
|
||||
if self.current_text_group.as_ref() != Some(&group.key) {
|
||||
let needs_newline = match kind {
|
||||
AssistantBlockKind::Text => self.completed_visible_messages > 0,
|
||||
AssistantBlockKind::Reasoning => self.completed_reasoning_messages > 0,
|
||||
AssistantBlockKind::ToolCall => false,
|
||||
};
|
||||
|
||||
if needs_newline {
|
||||
events.push(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta: "\n".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
self.current_text_group = Some(group.key.clone());
|
||||
}
|
||||
|
||||
events.push(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta: group.text,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
self.current_text_group = None;
|
||||
|
||||
let Some(tool_name) = tool_name(channel, recipient) else {
|
||||
return;
|
||||
};
|
||||
if !self.tool_calls_enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
let recipient = recipient.expect("tool groups always have recipient").to_string();
|
||||
let opens_same_call = match self.open_tool_call.as_ref() {
|
||||
Some(open_call) => open_call.recipient == recipient,
|
||||
None => false,
|
||||
};
|
||||
if !opens_same_call {
|
||||
let id = generate_tool_call_id();
|
||||
self.open_tool_call = Some(OpenHarmonyToolCall { recipient });
|
||||
events.push(AssistantEvent::ToolCallStart {
|
||||
id,
|
||||
name: tool_name.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if !group.text.is_empty() {
|
||||
events.push(AssistantEvent::ToolCallArgumentsDelta { delta: group.text });
|
||||
}
|
||||
}
|
||||
|
||||
/// Update newline and open-tool state after one Harmony message completes.
|
||||
fn handle_completed_message(&mut self, key: HarmonyGroupKey) {
|
||||
if self.current_text_group.as_ref() == Some(&key) {
|
||||
self.current_text_group = None;
|
||||
}
|
||||
|
||||
let channel = key.channel.as_deref();
|
||||
let recipient = key.recipient.as_deref();
|
||||
let kind = text_block_kind(channel, recipient);
|
||||
|
||||
if kind == Some(AssistantBlockKind::Text) {
|
||||
self.completed_visible_messages += 1;
|
||||
} else if kind == Some(AssistantBlockKind::Reasoning) {
|
||||
self.completed_reasoning_messages += 1;
|
||||
} else if tool_name(channel, recipient).is_some() {
|
||||
self.open_tool_call = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert decoded token updates into internal assistant events with Harmony
|
||||
/// parsing.
|
||||
#[try_stream]
|
||||
async fn harmony_assistant_event_stream(
|
||||
decoded: DynDecodedTextEventStream,
|
||||
encoding: &'static HarmonyEncoding,
|
||||
tool_calls_enabled: bool,
|
||||
mut y: TryYielder<AssistantEvent, Error>,
|
||||
) -> Result<()> {
|
||||
let mut state = HarmonyState::new(encoding.clone(), tool_calls_enabled)?;
|
||||
futures::pin_mut!(decoded);
|
||||
|
||||
while let Some(event) = decoded.next().await.transpose()? {
|
||||
match event {
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
} => {
|
||||
y.yield_ok(AssistantEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
} => {
|
||||
for event in state.process_token_ids(&token_ids)? {
|
||||
y.yield_ok(event).await;
|
||||
}
|
||||
|
||||
if finished.is_some() {
|
||||
for event in state.process_eos()? {
|
||||
y.yield_ok(event).await;
|
||||
}
|
||||
}
|
||||
|
||||
if logprobs.is_some() || !token_ids.is_empty() {
|
||||
y.yield_ok(AssistantEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
if let Some(finished) = finished {
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Lazily load the shared GPT-OSS Harmony encoding once per process.
|
||||
fn harmony_encoding() -> Result<&'static HarmonyEncoding> {
|
||||
static ENCODING: LazyLock<anyhow::Result<HarmonyEncoding>> = LazyLock::new(|| {
|
||||
load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
|
||||
.context("failed to load harmony encoding for gpt-oss")
|
||||
});
|
||||
|
||||
ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing {
|
||||
error: error.to_report_string().into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn harmony_output_parsing_error(
|
||||
error: impl Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
) -> Error {
|
||||
Error::HarmonyOutputParsing {
|
||||
error: error.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the decoded text payload from one parsed Harmony message.
|
||||
fn harmony_message_text(message: &HarmonyMessage) -> &str {
|
||||
let [HarmonyContent::Text(text)] = message.content.as_slice() else {
|
||||
unreachable!("Harmony parser emits one text content block per parsed message")
|
||||
};
|
||||
&text.text
|
||||
}
|
||||
|
||||
/// Map one Harmony `(channel, recipient)` pair to a visible assistant block
|
||||
/// kind.
|
||||
fn text_block_kind(channel: Option<&str>, recipient: Option<&str>) -> Option<AssistantBlockKind> {
|
||||
match (channel, recipient) {
|
||||
(Some("final"), _) => Some(AssistantBlockKind::Text),
|
||||
(Some("analysis"), None) => Some(AssistantBlockKind::Reasoning),
|
||||
(Some("commentary"), None) => Some(AssistantBlockKind::Text),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the tool name from a Harmony tool-recipient field, if present.
|
||||
fn tool_name<'a>(channel: Option<&str>, recipient: Option<&'a str>) -> Option<&'a str> {
|
||||
match (channel, recipient) {
|
||||
(Some("commentary" | "analysis"), Some(recipient)) => recipient.strip_prefix("functions."),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,351 @@
|
||||
//! Harmony output tests share the upstream `openai-harmony` tiktoken cache.
|
||||
//!
|
||||
//! Use a file lock for tests that load the encoding so `cargo nextest` cannot
|
||||
//! start multiple processes that concurrently populate the same cache file.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::executor::block_on;
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use openai_harmony::chat::{Message, Role};
|
||||
use serial_test::file_serial;
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
|
||||
|
||||
use super::*;
|
||||
use crate::output::ChatOutputProcessor;
|
||||
use crate::request::{ChatRequest, ChatTool, ChatToolChoice};
|
||||
use crate::{AssistantMessageExt, ChatEvent, FinishReason};
|
||||
|
||||
fn assistant_prefix() -> Vec<u32> {
|
||||
harmony_encoding()
|
||||
.unwrap()
|
||||
.render_conversation_for_completion(std::iter::empty::<&Message>(), Role::Assistant, None)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn completion_tokens(messages: &[Message]) -> Vec<u32> {
|
||||
let encoding = harmony_encoding().unwrap();
|
||||
let prefix = assistant_prefix();
|
||||
let rendered = encoding.render_conversation(messages.iter(), None).unwrap();
|
||||
assert!(rendered.starts_with(&prefix));
|
||||
rendered[prefix.len()..].to_vec()
|
||||
}
|
||||
|
||||
fn text_message(channel: &str, text: &str) -> Message {
|
||||
Message::from_role_and_content(Role::Assistant, text).with_channel(channel)
|
||||
}
|
||||
|
||||
fn tool_message(name: &str, arguments: &str, channel: &str) -> Message {
|
||||
Message::from_role_and_content(Role::Assistant, arguments)
|
||||
.with_channel(channel)
|
||||
.with_recipient(format!("functions.{name}"))
|
||||
.with_content_type("json")
|
||||
}
|
||||
|
||||
fn decoded_start() -> DecodedTextEvent {
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids: Arc::<[u32]>::from([]),
|
||||
prompt_logprobs: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn finished() -> Finished {
|
||||
Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_events(
|
||||
processor: HarmonyChatOutputProcessor,
|
||||
events: Vec<DecodedTextEvent>,
|
||||
) -> Vec<ChatEvent> {
|
||||
Box::new(processor)
|
||||
.process(Box::pin(stream::iter(events.into_iter().map(Ok))))
|
||||
.unwrap()
|
||||
.try_collect()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn request_with_tools() -> ChatRequest {
|
||||
ChatRequest {
|
||||
tool_choice: ChatToolChoice::Auto,
|
||||
tools: vec![ChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"]
|
||||
}),
|
||||
strict: None,
|
||||
}],
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn interrupted_final_message_is_preserved() {
|
||||
let tokens = completion_tokens(&[text_message("final", "hello")]);
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
events.last(),
|
||||
Some(&ChatEvent::Done {
|
||||
message: crate::AssistantMessage {
|
||||
content: vec![crate::AssistantContentBlock::Text {
|
||||
text: "hello".to_string(),
|
||||
}],
|
||||
},
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn eos_flush_preserves_trailing_replacement_text() {
|
||||
let mut tokens = completion_tokens(&[text_message("final", "Hi")]);
|
||||
tokens.pop();
|
||||
tokens.push(u32::MAX);
|
||||
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
let ChatEvent::Done { message, .. } = events.last().unwrap() else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(message.text(), format!("Hi{}", char::REPLACEMENT_CHARACTER));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn interrupted_analysis_message_is_preserved() {
|
||||
let tokens = completion_tokens(&[text_message("analysis", "think")]);
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
events.last(),
|
||||
Some(&ChatEvent::Done {
|
||||
message: crate::AssistantMessage {
|
||||
content: vec![crate::AssistantContentBlock::Reasoning {
|
||||
text: "think".to_string(),
|
||||
}],
|
||||
},
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
|
||||
let tokens = completion_tokens(&[
|
||||
text_message("commentary", "Let me check."),
|
||||
tool_message("get_weather", r#"{"city":"Paris"}"#, "commentary"),
|
||||
]);
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
let done = events.last().unwrap();
|
||||
let ChatEvent::Done { message, .. } = done else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(message.text(), "Let me check.");
|
||||
assert_eq!(message.tool_calls().count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn multiple_messages_get_newline_separators() {
|
||||
let tokens = completion_tokens(&[
|
||||
text_message("analysis", "first think"),
|
||||
text_message("analysis", "second think"),
|
||||
text_message("final", "first answer"),
|
||||
text_message("final", "second answer"),
|
||||
]);
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
let ChatEvent::Done { message, .. } = events.last().unwrap() else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(
|
||||
message.reasoning().as_deref(),
|
||||
Some("first think\nsecond think")
|
||||
);
|
||||
assert_eq!(message.text(), "first answer\nsecond answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
|
||||
let tokens = completion_tokens(&[tool_message(
|
||||
"get_weather",
|
||||
r#"{"city":"Paris"}"#,
|
||||
"commentary",
|
||||
)]);
|
||||
let midpoint = tokens.len() / 2;
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..midpoint].to_vec(),
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
},
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[midpoint..].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
let mut saw_start = None;
|
||||
let mut saw_args = String::new();
|
||||
let mut saw_end = None;
|
||||
for event in &events {
|
||||
match event {
|
||||
ChatEvent::ToolCallStart { id, name, .. } => {
|
||||
assert!(id.starts_with("call_"));
|
||||
assert_eq!(name, "get_weather");
|
||||
saw_start = Some(id.clone());
|
||||
}
|
||||
ChatEvent::ToolCallArgumentsDelta { delta, .. } => saw_args.push_str(delta),
|
||||
ChatEvent::ToolCallEnd { call, .. } => {
|
||||
saw_end = Some(call.clone());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let start_id = saw_start.expect("tool start");
|
||||
assert_eq!(saw_args, r#"{"city":"Paris"}"#);
|
||||
let end = saw_end.expect("tool end");
|
||||
assert_eq!(end.id, start_id);
|
||||
assert_eq!(end.arguments, r#"{"city":"Paris"}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn semantic_events_precede_same_update_logprobs() {
|
||||
let tokens = completion_tokens(&[text_message("final", "hello")]);
|
||||
let events = block_on(collect_events(
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs { entries: vec![] }],
|
||||
}),
|
||||
finished: Some(finished()),
|
||||
},
|
||||
],
|
||||
));
|
||||
|
||||
let block_delta_index = events
|
||||
.iter()
|
||||
.position(|event| matches!(event, ChatEvent::BlockDelta { .. }))
|
||||
.unwrap();
|
||||
let logprobs_index = events
|
||||
.iter()
|
||||
.position(|event| matches!(event, ChatEvent::LogprobsDelta { .. }))
|
||||
.unwrap();
|
||||
assert!(block_delta_index < logprobs_index);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_generic_parser_overrides() {
|
||||
let reasoning_error =
|
||||
validate_harmony_parser_overrides(&ParserSelection::Auto, &ParserSelection::None)
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
reasoning_error.to_string(),
|
||||
"gpt_oss uses native Harmony output parsing; generic reasoning parser override `none` is not supported"
|
||||
);
|
||||
|
||||
let tool_error = validate_harmony_parser_overrides(
|
||||
&ParserSelection::Explicit("json".to_string()),
|
||||
&ParserSelection::Auto,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
tool_error.to_string(),
|
||||
"gpt_oss uses native Harmony output parsing; generic tool parser override `json` is not supported"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[file_serial(harmony_tiktoken_cache)]
|
||||
fn allows_auto_auto_only() {
|
||||
validate_harmony_parser_overrides(&ParserSelection::Auto, &ParserSelection::Auto).unwrap();
|
||||
let _ = HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap();
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::Stream;
|
||||
use subenum::subenum;
|
||||
use trait_set::trait_set;
|
||||
use uuid::Uuid;
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
|
||||
|
||||
use crate::FinishReason;
|
||||
use crate::error::Result;
|
||||
use crate::event::{AssistantBlockKind, ChatEvent};
|
||||
|
||||
mod default;
|
||||
mod harmony;
|
||||
mod structured;
|
||||
|
||||
pub use default::DefaultChatOutputProcessor;
|
||||
pub use harmony::HarmonyChatOutputProcessor;
|
||||
pub(crate) use harmony::validate_harmony_parser_overrides;
|
||||
|
||||
/// Internal assistant event before final assembly.
|
||||
///
|
||||
/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content.
|
||||
/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants.
|
||||
#[subenum(ContentEvent)]
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) enum AssistantEvent {
|
||||
#[subenum(ContentEvent)]
|
||||
Start {
|
||||
prompt_token_ids: Arc<[u32]>,
|
||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
},
|
||||
#[subenum(ContentEvent)]
|
||||
TextDelta {
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
},
|
||||
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
||||
#[subenum(ContentEvent)]
|
||||
LogprobsDelta {
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
token_ids: Vec<u32>,
|
||||
},
|
||||
/// The start of a new tool call, with its declared name and generated ID.
|
||||
ToolCallStart { id: String, name: String },
|
||||
/// A delta for the arguments of the currently open tool call. Must follow a
|
||||
/// `ToolCallStart`.
|
||||
ToolCallArgumentsDelta { delta: String },
|
||||
#[subenum(ContentEvent)]
|
||||
Done {
|
||||
prompt_token_count: usize,
|
||||
output_token_count: usize,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ContentEvent {
|
||||
/// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values
|
||||
/// by treating all text as plain (non-reasoning) content.
|
||||
fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec<Self> {
|
||||
match event {
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
} => vec![Self::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
}],
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished,
|
||||
} => {
|
||||
let mut events = Vec::new();
|
||||
if !delta.is_empty() {
|
||||
events.push(Self::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta,
|
||||
});
|
||||
}
|
||||
if logprobs.is_some() || !token_ids.is_empty() {
|
||||
events.push(Self::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
});
|
||||
}
|
||||
if let Some(finished) = finished {
|
||||
events.push(Self::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
});
|
||||
}
|
||||
events
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Boxed stream of decoded text events coming from [`vllm_text`].
|
||||
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
|
||||
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
|
||||
pub type DynChatEventStream = Pin<Box<dyn Stream<Item = Result<ChatEvent>> + Send>>;
|
||||
|
||||
/// Request-scoped output processor from decoded text events into structured
|
||||
/// chat events.
|
||||
pub trait ChatOutputProcessor: Send {
|
||||
/// Consume decoded text stream and return the structured chat-event stream.
|
||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream>;
|
||||
}
|
||||
|
||||
/// Trait-object form of [`ChatOutputProcessor`].
|
||||
pub type DynChatOutputProcessor = Box<dyn ChatOutputProcessor>;
|
||||
|
||||
trait_set! {
|
||||
/// Boxed-stream constraint for decoded text updates.
|
||||
pub(crate) trait DecodedTextEventStream = Stream<Item = Result<DecodedTextEvent>> + Send + 'static;
|
||||
/// Boxed-stream constraint for internal assistant events.
|
||||
pub(crate) trait AssistantEventStream = Stream<Item = Result<AssistantEvent>> + Send + 'static;
|
||||
/// Boxed-stream constraint for public chat events.
|
||||
pub(crate) trait ChatEventStream = Stream<Item = Result<ChatEvent>> + Send + 'static;
|
||||
}
|
||||
|
||||
/// Generate the northbound tool-call ID using the OpenAI-style `call_<id>`
|
||||
/// format.
|
||||
// TODO: support other ID scheme like Kimi-K2's
|
||||
// `functions.{name}:{global_index}`.
|
||||
pub(crate) fn generate_tool_call_id() -> String {
|
||||
format!("call_{}", &Uuid::new_v4().simple().to_string()[..24])
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
//! Adapts parsed assistant updates into structured chat events.
|
||||
//!
|
||||
//! This module remains the final assembly stage in `vllm-chat`. Token-to-text
|
||||
//! decoding still lives in `decoded.rs`, while reasoning separation and tool
|
||||
//! parsing are handled earlier by their own adapters. This stage consumes those
|
||||
//! parsed deltas and assembles higher-level assistant content blocks.
|
||||
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::{StreamExt as _, pin_mut};
|
||||
use vllm_text::DecodedLogprobs;
|
||||
|
||||
use super::{AssistantEvent, AssistantEventStream};
|
||||
use crate::error::Error;
|
||||
use crate::event::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantToolCall, ChatEvent,
|
||||
};
|
||||
use crate::{FinishReason, Result};
|
||||
|
||||
/// One currently open assistant text-like block being assembled from streamed
|
||||
/// deltas.
|
||||
struct OpenTextBlock {
|
||||
/// Stable position of this block in the final assistant message.
|
||||
index: usize,
|
||||
/// Semantic kind of the block being assembled.
|
||||
kind: AssistantBlockKind,
|
||||
/// Accumulated text payload for the block.
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// One currently open assistant tool call being assembled from streamed deltas.
|
||||
struct OpenToolCall {
|
||||
/// Stable ordinal of this tool call in the assistant tool-call list.
|
||||
index: usize,
|
||||
/// Stable tool-call ID exposed northbound.
|
||||
id: String,
|
||||
/// Function name.
|
||||
name: String,
|
||||
/// Incremental JSON arguments accumulated so far.
|
||||
arguments: String,
|
||||
}
|
||||
|
||||
/// Per-stream block assembly state.
|
||||
///
|
||||
/// The adapter maintains at most one open text block and one open tool call,
|
||||
/// and appends deltas to them until the semantic kind changes or the stream
|
||||
/// terminates.
|
||||
struct StructuredEventState {
|
||||
/// Final assistant message assembled so far.
|
||||
message: AssistantMessage,
|
||||
/// Currently open text or reasoning block, if any.
|
||||
open_text_block: Option<OpenTextBlock>,
|
||||
/// Currently open tool call, if any.
|
||||
open_tool_call: Option<OpenToolCall>,
|
||||
/// Next OpenAI-compatible tool-call ordinal.
|
||||
next_tool_call_index: usize,
|
||||
}
|
||||
|
||||
impl StructuredEventState {
|
||||
/// Create one fresh assembly state for a new streamed response.
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
message: AssistantMessage::default(),
|
||||
open_text_block: None,
|
||||
open_tool_call: None,
|
||||
next_tool_call_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one parsed text delta into zero or more structured chat events.
|
||||
fn process_text_delta(
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
) -> Result<Vec<ChatEvent>> {
|
||||
let mut events = Vec::new();
|
||||
self.close_open_tool_call(&mut events);
|
||||
self.push_text_delta(kind, delta, &mut events);
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Forward per-update sample metadata without attaching it to text blocks.
|
||||
fn process_logprobs_delta(
|
||||
&mut self,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
token_ids: Vec<u32>,
|
||||
) -> Result<Vec<ChatEvent>> {
|
||||
Ok(vec![ChatEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
}])
|
||||
}
|
||||
|
||||
/// Start one new tool call, closing any incompatible open block first.
|
||||
fn start_tool_call(&mut self, id: String, name: String) -> Result<Vec<ChatEvent>> {
|
||||
let mut events = Vec::new();
|
||||
self.close_open_text_block(&mut events);
|
||||
self.close_open_tool_call(&mut events);
|
||||
|
||||
let index = self.next_tool_call_index;
|
||||
self.next_tool_call_index += 1;
|
||||
self.open_tool_call = Some(OpenToolCall {
|
||||
index,
|
||||
id: id.clone(),
|
||||
name: name.clone(),
|
||||
arguments: String::new(),
|
||||
});
|
||||
events.push(ChatEvent::ToolCallStart { index, id, name });
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Append one incremental tool-call arguments delta.
|
||||
fn push_tool_call_arguments(&mut self, delta: String) -> Result<Vec<ChatEvent>> {
|
||||
let mut events = Vec::new();
|
||||
let Some(open_tool_call) = self.open_tool_call.as_mut() else {
|
||||
return Err(Error::ToolCallStreamInvariant {
|
||||
message: "received tool-call arguments delta without an open tool call".to_string(),
|
||||
});
|
||||
};
|
||||
open_tool_call.arguments.push_str(&delta);
|
||||
events.push(ChatEvent::ToolCallArgumentsDelta {
|
||||
index: open_tool_call.index,
|
||||
delta,
|
||||
});
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Close any open block and emit the terminal `Done` event.
|
||||
fn finish(
|
||||
&mut self,
|
||||
prompt_token_count: usize,
|
||||
output_token_count: usize,
|
||||
finish_reason: FinishReason,
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
) -> Result<Vec<ChatEvent>> {
|
||||
let mut events = Vec::new();
|
||||
self.close_open_text_block(&mut events);
|
||||
self.close_open_tool_call(&mut events);
|
||||
events.push(ChatEvent::Done {
|
||||
message: self.message.clone(),
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
});
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// Append one semantic text delta to the current block, or open a new block
|
||||
/// when the semantic kind changes.
|
||||
fn push_text_delta(
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
events: &mut Vec<ChatEvent>,
|
||||
) {
|
||||
if delta.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
match self.open_text_block.as_mut() {
|
||||
// If there's a currently open block of the same kind, append to it.
|
||||
Some(open_block) if open_block.kind == kind => {
|
||||
open_block.text.push_str(&delta);
|
||||
events.push(ChatEvent::BlockDelta {
|
||||
index: open_block.index,
|
||||
kind,
|
||||
delta,
|
||||
});
|
||||
}
|
||||
// Otherwise, close the currently open block (if any) and start a
|
||||
// new one.
|
||||
_ => {
|
||||
self.close_open_text_block(events);
|
||||
let index = self.message.content.len();
|
||||
self.open_text_block = Some(OpenTextBlock {
|
||||
index,
|
||||
kind,
|
||||
text: delta.clone(),
|
||||
});
|
||||
events.push(ChatEvent::BlockStart { index, kind });
|
||||
events.push(ChatEvent::BlockDelta { index, kind, delta });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Finalize the currently open text block, if present.
|
||||
fn close_open_text_block(&mut self, events: &mut Vec<ChatEvent>) {
|
||||
let Some(open_block) = self.open_text_block.take() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let block = match open_block.kind {
|
||||
AssistantBlockKind::Text => AssistantContentBlock::Text {
|
||||
text: open_block.text,
|
||||
},
|
||||
AssistantBlockKind::Reasoning => AssistantContentBlock::Reasoning {
|
||||
text: open_block.text,
|
||||
},
|
||||
AssistantBlockKind::ToolCall => {
|
||||
unreachable!("tool calls must not be assembled as text blocks")
|
||||
}
|
||||
};
|
||||
self.message.push_block(block.clone());
|
||||
events.push(ChatEvent::BlockEnd {
|
||||
index: open_block.index,
|
||||
block,
|
||||
});
|
||||
}
|
||||
|
||||
/// Finalize the currently open tool call, if present.
|
||||
fn close_open_tool_call(&mut self, events: &mut Vec<ChatEvent>) {
|
||||
let Some(open_tool_call) = self.open_tool_call.take() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let call = AssistantToolCall {
|
||||
id: open_tool_call.id,
|
||||
name: open_tool_call.name,
|
||||
arguments: open_tool_call.arguments,
|
||||
};
|
||||
self.message.push_block(AssistantContentBlock::ToolCall(call.clone()));
|
||||
events.push(ChatEvent::ToolCallEnd {
|
||||
index: open_tool_call.index,
|
||||
call,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap one parsed assistant stream into the public structured chat event
|
||||
/// stream.
|
||||
#[try_stream]
|
||||
pub(crate) async fn structured_chat_event_stream(
|
||||
stream: impl AssistantEventStream,
|
||||
mut y: TryYielder<ChatEvent, Error>,
|
||||
) -> Result<()> {
|
||||
pin_mut!(stream);
|
||||
|
||||
let mut state = StructuredEventState::new();
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
AssistantEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
} => {
|
||||
y.yield_ok(ChatEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
AssistantEvent::TextDelta { kind, delta } => {
|
||||
for next in state.process_text_delta(kind, delta)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
AssistantEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids,
|
||||
} => {
|
||||
for next in state.process_logprobs_delta(logprobs, token_ids)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
AssistantEvent::ToolCallStart { id, name } => {
|
||||
for next in state.start_tool_call(id, name)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
AssistantEvent::ToolCallArgumentsDelta { delta } => {
|
||||
for next in state.push_tool_call_arguments(delta)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
for next in state.finish(
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
|
||||
use super::structured_chat_event_stream;
|
||||
use crate::FinishReason;
|
||||
use crate::error::Error;
|
||||
use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent};
|
||||
use crate::output::AssistantEvent;
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_closes_tool_call_on_done() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(events[0], ChatEvent::ToolCallStart { .. }));
|
||||
assert!(matches!(
|
||||
events[1],
|
||||
ChatEvent::ToolCallArgumentsDelta { .. }
|
||||
));
|
||||
let ChatEvent::ToolCallEnd { call, .. } = &events[2] else {
|
||||
panic!("expected tool call end");
|
||||
};
|
||||
assert_eq!(call.name, "get_weather");
|
||||
assert_eq!(call.arguments, r#"{"city":"Paris"}"#);
|
||||
let ChatEvent::Done { message, .. } = &events[3] else {
|
||||
panic!("expected done");
|
||||
};
|
||||
let tool_calls = message.tool_calls().collect::<Vec<_>>();
|
||||
assert_eq!(tool_calls.len(), 1);
|
||||
assert_eq!(tool_calls[0].id, "call_1");
|
||||
assert_eq!(tool_calls[0].arguments, r#"{"city":"Paris"}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_closes_previous_tool_call_on_next_start() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
name: "first".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"a":1}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_2".to_string(),
|
||||
name: "second".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"b":2}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(events[0], ChatEvent::ToolCallStart { .. }));
|
||||
assert!(matches!(
|
||||
events[1],
|
||||
ChatEvent::ToolCallArgumentsDelta { .. }
|
||||
));
|
||||
let ChatEvent::ToolCallEnd { call, .. } = &events[2] else {
|
||||
panic!("expected first tool call end");
|
||||
};
|
||||
assert_eq!(call.name, "first");
|
||||
assert!(matches!(events[3], ChatEvent::ToolCallStart { .. }));
|
||||
let ChatEvent::Done { message, .. } = &events[6] else {
|
||||
panic!("expected done");
|
||||
};
|
||||
let tool_calls = message.tool_calls().collect::<Vec<_>>();
|
||||
assert_eq!(tool_calls.len(), 2);
|
||||
assert_eq!(tool_calls[0].name, "first");
|
||||
assert_eq!(tool_calls[1].name, "second");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_numbers_tool_calls_independent_of_text_blocks() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "before".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
events[0],
|
||||
ChatEvent::BlockStart {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
}
|
||||
));
|
||||
assert!(matches!(events[2], ChatEvent::BlockEnd { index: 0, .. }));
|
||||
assert!(matches!(
|
||||
events[3],
|
||||
ChatEvent::ToolCallStart { index: 0, .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
events[4],
|
||||
ChatEvent::ToolCallArgumentsDelta { index: 0, .. }
|
||||
));
|
||||
assert!(matches!(events[5], ChatEvent::ToolCallEnd { index: 0, .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_closes_tool_call_before_text() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "done".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(events[2], ChatEvent::ToolCallEnd { .. }));
|
||||
assert!(matches!(
|
||||
events[3],
|
||||
ChatEvent::BlockStart {
|
||||
kind: AssistantBlockKind::Text,
|
||||
..
|
||||
}
|
||||
));
|
||||
let ChatEvent::Done { message, .. } = &events[6] else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(message.text(), "done");
|
||||
assert_eq!(message.tool_calls().count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_rejects_arguments_without_open_tool_call() {
|
||||
let events = stream::iter(vec![Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: "{}".to_string(),
|
||||
})]);
|
||||
|
||||
let err = structured_chat_event_stream(events)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("expected one event")
|
||||
.expect_err("expected invariant error");
|
||||
|
||||
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
pub mod reasoning;
|
||||
pub mod tool;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde_with::DeserializeFromStr;
|
||||
|
||||
/// Specify which reasoning or tool-call parser implementation to use.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
pub enum ParserSelection {
|
||||
/// Use model-based auto-detection.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Disable the parser entirely.
|
||||
None,
|
||||
/// Force one specific parser implementation by name.
|
||||
Explicit(String),
|
||||
}
|
||||
|
||||
impl ParserSelection {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const NONE_LITERAL: &str = "none";
|
||||
}
|
||||
|
||||
impl FromStr for ParserSelection {
|
||||
type Err = Infallible;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
Ok(if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Self::Auto
|
||||
} else if value.eq_ignore_ascii_case(Self::NONE_LITERAL) {
|
||||
Self::None
|
||||
} else {
|
||||
Self::Explicit(value.to_owned())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ParserSelection {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Auto => f.write_str(Self::AUTO_LITERAL),
|
||||
Self::None => f.write_str(Self::NONE_LITERAL),
|
||||
Self::Explicit(name) => f.write_str(name),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Registry and model matcher for reasoning and tool parsers.
|
||||
#[derive(Clone)]
|
||||
pub struct ParserFactory<C> {
|
||||
creators: HashMap<String, C>,
|
||||
patterns: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl<C> Default for ParserFactory<C> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
creators: HashMap::new(),
|
||||
patterns: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<C> ParserFactory<C> {
|
||||
/// Register a creator for a parser by an exact name.
|
||||
pub fn register_creator(&mut self, name: &str, creator: C) -> &mut Self {
|
||||
self.creators.insert(name.to_string(), creator);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a case-insensitive substring match from model ID to parser name.
|
||||
pub fn register_pattern(&mut self, pattern: &str, parser_name: &str) -> &mut Self {
|
||||
self.patterns.push((pattern.to_lowercase(), parser_name.to_string()));
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the first registered parser name matching the given model ID.
|
||||
pub fn resolve_name_for_model(&self, model_id: &str) -> Option<&str> {
|
||||
let model_lower = model_id.to_lowercase();
|
||||
self.patterns
|
||||
.iter()
|
||||
.find(|(pattern, _)| model_lower.contains(pattern))
|
||||
.map(|(_, parser_name)| parser_name.as_str())
|
||||
}
|
||||
|
||||
/// Return true if the exact parser name is registered.
|
||||
pub fn contains(&self, name: &str) -> bool {
|
||||
self.creators.contains_key(name)
|
||||
}
|
||||
|
||||
/// Return all registered parser names sorted for stable display.
|
||||
pub fn list(&self) -> Vec<String> {
|
||||
let mut names: Vec<_> = self.creators.keys().cloned().collect();
|
||||
names.sort_unstable();
|
||||
names
|
||||
}
|
||||
|
||||
/// Get the constructor for a parser by its exact registered name, or return
|
||||
/// None if not found.
|
||||
pub fn creator(&self, name: &str) -> Option<&C> {
|
||||
self.creators.get(name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Reasoning parser registration and selection boundary for `vllm-chat`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_reasoning_parser::{
|
||||
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
|
||||
KimiReasoningParser, MiniMaxM2ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser,
|
||||
ReasoningDelta, ReasoningError, ReasoningParser, Step3ReasoningParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
|
||||
/// Canonical public names for registered reasoning parsers.
|
||||
pub mod names {
|
||||
pub const COHERE_CMD: &str = "cohere_cmd";
|
||||
pub const DEEPSEEK_R1: &str = "deepseek_r1";
|
||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const KIMI: &str = "kimi";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||
pub const NEMOTRON_V3: &str = "nemotron_v3";
|
||||
pub const QWEN3: &str = "qwen3";
|
||||
pub const STEP3: &str = "step3";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered reasoning parser implementation.
|
||||
type ReasoningParserCreator =
|
||||
fn(DynTokenizer) -> vllm_reasoning_parser::Result<Box<dyn ReasoningParser>>;
|
||||
|
||||
/// Registry and model matcher for reasoning parsers.
|
||||
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
|
||||
|
||||
impl ReasoningParserFactory {
|
||||
/// Get the global reasoning parser factory with built-in registrations and
|
||||
/// model mappings.
|
||||
pub fn global() -> &'static Self {
|
||||
static INSTANCE: LazyLock<ReasoningParserFactory> =
|
||||
LazyLock::new(ReasoningParserFactory::new);
|
||||
&INSTANCE
|
||||
}
|
||||
|
||||
/// Create the default registry with built-in parser names and model
|
||||
/// mappings.
|
||||
pub fn new() -> Self {
|
||||
let mut factory = Self::default();
|
||||
|
||||
factory
|
||||
.register_parser::<CohereCmdReasoningParser>(names::COHERE_CMD)
|
||||
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<Gemma4ReasoningParser>(names::GEMMA4)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
.register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
|
||||
.register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
|
||||
.register_parser::<Qwen3ReasoningParser>(names::QWEN3)
|
||||
.register_parser::<Step3ReasoningParser>(names::STEP3);
|
||||
|
||||
factory
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_R1)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("qwen", names::QWEN3)
|
||||
.register_pattern("glm-5", names::GLM45)
|
||||
.register_pattern("glm-4.7", names::GLM45)
|
||||
.register_pattern("glm-4.6", names::GLM45)
|
||||
.register_pattern("glm-4.5", names::GLM45)
|
||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||
.register_pattern("kimi", names::KIMI)
|
||||
.register_pattern("step3", names::STEP3)
|
||||
.register_pattern("minimax", names::MINIMAX_M2)
|
||||
.register_pattern("mm-m2", names::MINIMAX_M2)
|
||||
.register_pattern("cohere", names::COHERE_CMD)
|
||||
.register_pattern("command", names::COHERE_CMD)
|
||||
.register_pattern("nano", names::NEMOTRON_V3)
|
||||
.register_pattern("nemotron", names::NEMOTRON_V3);
|
||||
|
||||
factory
|
||||
}
|
||||
|
||||
/// Register one parser type that exposes a static `create()` constructor.
|
||||
pub fn register_parser<T>(&mut self, name: &str) -> &mut Self
|
||||
where
|
||||
T: ReasoningParser + 'static,
|
||||
{
|
||||
self.register_creator(name, T::create)
|
||||
}
|
||||
|
||||
/// Construct a parser from an exact name.
|
||||
pub fn create(
|
||||
&self,
|
||||
name: &str,
|
||||
tokenizer: DynTokenizer,
|
||||
) -> crate::Result<Box<dyn ReasoningParser>> {
|
||||
let creator = self.creator(name).ok_or_else(|| crate::Error::ParserUnavailableByName {
|
||||
kind: "reasoning",
|
||||
name: name.to_string(),
|
||||
available_names: self.list(),
|
||||
})?;
|
||||
|
||||
creator(tokenizer).map_err(|error| crate::Error::ParserInitialization {
|
||||
kind: "reasoning",
|
||||
name: name.to_string(),
|
||||
error: error.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,61 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::{ReasoningParserFactory, names};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_contains_and_lists_registered_parsers() {
|
||||
let factory = ReasoningParserFactory::new();
|
||||
assert!(factory.contains(names::QWEN3));
|
||||
assert!(factory.contains(names::DEEPSEEK_V4));
|
||||
assert!(factory.list().contains(&names::QWEN3.to_string()));
|
||||
assert!(factory.list().contains(&names::DEEPSEEK_V4.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_resolves_deepseek_v4_to_qwen3_alias() {
|
||||
let factory = ReasoningParserFactory::new();
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_rejects_unknown_parser_names() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let factory = ReasoningParserFactory::new();
|
||||
let error = match factory.create("missing", tokenizer) {
|
||||
Ok(_) => panic!("expected parser lookup to fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("choose from"));
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
//! Tool parser registration and selection boundary for `vllm-chat`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_tool_parser::{
|
||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, HermesToolParser, KimiK2ToolParser,
|
||||
Llama3JsonToolParser, MinimaxM2ToolParser, MistralToolParser, Qwen3CoderToolParser,
|
||||
Qwen3XmlToolParser, ToolCallDelta, ToolParseResult, ToolParser, ToolParserError,
|
||||
};
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
use crate::request::ChatTool;
|
||||
|
||||
/// Canonical public names for registered tool parsers.
|
||||
pub mod names {
|
||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||
pub const DEEPSEEK_V31: &str = "deepseek_v31";
|
||||
pub const DEEPSEEK_V32: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const HERMES: &str = "hermes";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const LLAMA3_JSON: &str = "llama3_json";
|
||||
pub const LLAMA4_JSON: &str = "llama4_json";
|
||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||
pub const MISTRAL: &str = "mistral";
|
||||
pub const QWEN3_CODER: &str = "qwen3_coder";
|
||||
pub const QWEN3_XML: &str = "qwen3_xml";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered tool parser implementation.
|
||||
type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>;
|
||||
|
||||
/// Registry and model matcher for tool parsers.
|
||||
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
|
||||
|
||||
impl ToolParserFactory {
|
||||
/// Get the global tool parser factory with built-in registrations and model
|
||||
/// mappings.
|
||||
pub fn global() -> &'static Self {
|
||||
static INSTANCE: LazyLock<ToolParserFactory> = LazyLock::new(ToolParserFactory::new);
|
||||
&INSTANCE
|
||||
}
|
||||
|
||||
/// Create the default registry with built-in parser names and model
|
||||
/// mappings.
|
||||
pub fn new() -> Self {
|
||||
let mut factory = Self::default();
|
||||
|
||||
factory
|
||||
.register_parser::<DeepSeekV3ToolParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV31ToolParser>(names::DEEPSEEK_V31)
|
||||
.register_parser::<DeepSeekV32ToolParser>(names::DEEPSEEK_V32)
|
||||
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||
.register_parser::<Gemma4ToolParser>(names::GEMMA4)
|
||||
.register_parser::<HermesToolParser>(names::HERMES)
|
||||
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
|
||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
|
||||
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
|
||||
.register_parser::<MistralToolParser>(names::MISTRAL)
|
||||
.register_parser::<Qwen3XmlToolParser>(names::QWEN3_XML)
|
||||
.register_parser::<Qwen3CoderToolParser>(names::QWEN3_CODER);
|
||||
|
||||
factory
|
||||
.register_pattern("mistral-", names::MISTRAL)
|
||||
.register_pattern("mixtral-", names::MISTRAL)
|
||||
.register_pattern("qwen3-coder", names::QWEN3_CODER)
|
||||
.register_pattern("qwen2.5-coder", names::QWEN3_CODER)
|
||||
.register_pattern("qwen3.5", names::QWEN3_CODER)
|
||||
.register_pattern("qwen", names::QWEN3_XML)
|
||||
.register_pattern("hermes", names::HERMES)
|
||||
.register_pattern("llama-4", names::LLAMA4_JSON)
|
||||
.register_pattern("llama-3.2", names::LLAMA3_JSON)
|
||||
.register_pattern("llama-3.1", names::LLAMA3_JSON)
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_V3)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3.2", names::DEEPSEEK_V32)
|
||||
.register_pattern("deepseek-v3.1", names::DEEPSEEK_V31)
|
||||
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
||||
.register_pattern("glm-5", names::GLM47)
|
||||
.register_pattern("glm-4.7", names::GLM47)
|
||||
.register_pattern("glm-4.6", names::GLM45)
|
||||
.register_pattern("glm-4.5", names::GLM45)
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||
.register_pattern("minimax", names::MINIMAX_M2)
|
||||
.register_pattern("mm-m2", names::MINIMAX_M2);
|
||||
|
||||
factory
|
||||
}
|
||||
|
||||
/// Register one parser type that exposes a static `create()` constructor.
|
||||
pub fn register_parser<T>(&mut self, name: &str) -> &mut Self
|
||||
where
|
||||
T: ToolParser + 'static,
|
||||
{
|
||||
self.register_creator(name, T::create)
|
||||
}
|
||||
|
||||
/// Construct a parser from an exact name.
|
||||
pub fn create(&self, name: &str, tools: &[ChatTool]) -> crate::Result<Box<dyn ToolParser>> {
|
||||
let creator = self.creator(name).ok_or_else(|| crate::Error::ParserUnavailableByName {
|
||||
kind: "tool",
|
||||
name: name.to_string(),
|
||||
available_names: self.list(),
|
||||
})?;
|
||||
|
||||
creator(tools).map_err(|error| crate::Error::ParserInitialization {
|
||||
kind: "tool",
|
||||
name: name.to_string(),
|
||||
error: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve a parser from model ID and then construct it.
|
||||
pub fn create_for_model(
|
||||
&self,
|
||||
model_id: &str,
|
||||
tools: &[ChatTool],
|
||||
) -> crate::Result<Box<dyn ToolParser>> {
|
||||
let name = self.resolve_name_for_model(model_id).ok_or_else(|| {
|
||||
crate::Error::ParserUnavailableForModel {
|
||||
kind: "tool",
|
||||
model_id: model_id.to_string(),
|
||||
}
|
||||
})?;
|
||||
self.create(name, tools)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,152 @@
|
||||
use vllm_tool_parser::Result;
|
||||
|
||||
use super::{ToolParseResult, ToolParser, ToolParserFactory, names};
|
||||
use crate::Error;
|
||||
use crate::request::ChatTool;
|
||||
|
||||
struct FakeToolParser;
|
||||
|
||||
impl ToolParser for FakeToolParser {
|
||||
fn create(_tools: &[ChatTool]) -> Result<Box<dyn ToolParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self))
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
|
||||
Ok(ToolParseResult::default())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_factory_starts_empty() {
|
||||
let factory = ToolParserFactory::default();
|
||||
assert!(factory.list().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_contains_and_creates_registered_parsers() {
|
||||
let mut factory = ToolParserFactory::default();
|
||||
factory.register_parser::<FakeToolParser>("fake");
|
||||
|
||||
assert!(factory.contains("fake"));
|
||||
assert!(factory.list().contains(&"fake".to_string()));
|
||||
factory.create("fake", &[]).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_rejects_unknown_parser_names() {
|
||||
let factory = ToolParserFactory::default();
|
||||
let error = match factory.create("missing", &[]) {
|
||||
Ok(_) => panic!("expected parser lookup to fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(error, Error::ParserUnavailableByName { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_rejects_unknown_models() {
|
||||
let factory = ToolParserFactory::default();
|
||||
let error = match factory.create_for_model("definitely-unknown-model", &[]) {
|
||||
Ok(_) => panic!("expected model lookup to fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(error, Error::ParserUnavailableForModel { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_creates_registered_parser_for_model() {
|
||||
let mut factory = ToolParserFactory::default();
|
||||
factory
|
||||
.register_parser::<FakeToolParser>("fake")
|
||||
.register_pattern("fake-model", "fake");
|
||||
|
||||
factory.create_for_model("my-fake-model-v1", &[]).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_new_resolves_default_patterns() {
|
||||
let factory = ToolParserFactory::new();
|
||||
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("Qwen/Qwen3.5-0.8B"),
|
||||
Some(names::QWEN3_CODER)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("Qwen/Qwen3-0.6B"),
|
||||
Some(names::QWEN3_XML)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("Qwen/Qwen3-Coder-30B"),
|
||||
Some(names::QWEN3_CODER)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("meta-llama-4-maverick"),
|
||||
Some(names::LLAMA4_JSON)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("meta-llama-3.2-3b-instruct"),
|
||||
Some(names::LLAMA3_JSON)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("meta-llama/Llama-3.1-8B-Instruct"),
|
||||
Some(names::LLAMA3_JSON)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V3.2-Exp"),
|
||||
Some(names::DEEPSEEK_V32)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4-Chat"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"),
|
||||
Some(names::DEEPSEEK_V3)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V3.1"),
|
||||
Some(names::DEEPSEEK_V31)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("zai-org/GLM-5-32B-Chat"),
|
||||
Some(names::GLM47)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("zai-org/GLM-5.1-32B-Instruct"),
|
||||
Some(names::GLM47)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("glm-4.7"),
|
||||
Some(names::GLM47)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("google/gemma-4-27b-it"),
|
||||
Some(names::GEMMA4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("NousResearch/Hermes-3-Llama-3.1-8B"),
|
||||
Some(names::HERMES)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("MiniMax/MiniMax-M2-01"),
|
||||
Some(names::MINIMAX_M2)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("org/mm-m2-base"),
|
||||
Some(names::MINIMAX_M2)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,555 @@
|
||||
//! DeepSeek V3.2 prompt renderer.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json_fmt::JsonFormat;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatRole, ChatTool};
|
||||
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
|
||||
|
||||
const BOS_TOKEN: &str = "<|begin▁of▁sentence|>";
|
||||
const EOS_TOKEN: &str = "<|end▁of▁sentence|>";
|
||||
const THINKING_START_TOKEN: &str = "<think>";
|
||||
const THINKING_END_TOKEN: &str = "</think>";
|
||||
const DSML_TOKEN: &str = "|DSML|";
|
||||
|
||||
/// DeepSeek uses `"chat"` vs `"thinking"` mode names. Keep the split explicit
|
||||
/// here so the render branches stay easy to read.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ThinkingMode {
|
||||
Chat,
|
||||
Thinking,
|
||||
}
|
||||
|
||||
/// Tool schema shape rendered inside the `<functions>` block.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RenderedToolSchema<'a> {
|
||||
name: &'a str,
|
||||
description: Option<&'a str>,
|
||||
parameters: &'a Value,
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
/// Render one chat request into the final prompt string.
|
||||
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
let thinking_mode = match request.enable_thinking()?.unwrap_or(false) {
|
||||
true => ThinkingMode::Thinking,
|
||||
false => ThinkingMode::Chat,
|
||||
};
|
||||
let drop_thinking = matches!(
|
||||
request.messages.last().map(ChatMessage::role),
|
||||
Some(ChatRole::User | ChatRole::Developer)
|
||||
);
|
||||
let render_offset = isize::from(request.tool_parsing_enabled());
|
||||
let last_user_render_index =
|
||||
find_last_user_render_index(request.messages.as_slice(), render_offset);
|
||||
let last_user_actual_index = find_last_user_actual_index(request.messages.as_slice());
|
||||
let mut prompt = String::from(BOS_TOKEN);
|
||||
|
||||
if request.tool_parsing_enabled() {
|
||||
render_system_message(&mut prompt, None, &request.tools)?;
|
||||
}
|
||||
|
||||
for (message_index, message) in request.messages.iter().enumerate() {
|
||||
render_message(
|
||||
&mut prompt,
|
||||
request.messages.as_slice(),
|
||||
message_index,
|
||||
message,
|
||||
render_offset,
|
||||
last_user_render_index,
|
||||
last_user_actual_index,
|
||||
thinking_mode,
|
||||
drop_thinking,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(prompt)
|
||||
}
|
||||
|
||||
/// Find the last user-like turn in render order.
|
||||
///
|
||||
/// `render_offset` is `1` when a synthetic tool-only system turn is rendered
|
||||
/// before the real request messages, and `0` otherwise.
|
||||
fn find_last_user_render_index(messages: &[ChatMessage], render_offset: isize) -> isize {
|
||||
messages
|
||||
.iter()
|
||||
.rposition(|message| matches!(message.role(), ChatRole::User | ChatRole::Developer))
|
||||
.map(|index| index as isize + render_offset)
|
||||
.unwrap_or(-1)
|
||||
}
|
||||
|
||||
/// Render one real request message, using `render_offset` to account for any
|
||||
/// synthetic tool-only system turn that was already emitted before the loop.
|
||||
fn render_message(
|
||||
out: &mut String,
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
message: &ChatMessage,
|
||||
render_offset: isize,
|
||||
last_user_render_index: isize,
|
||||
last_user_actual_index: usize,
|
||||
thinking_mode: ThinkingMode,
|
||||
drop_thinking: bool,
|
||||
) -> Result<()> {
|
||||
let render_index = message_index as isize + render_offset;
|
||||
let opens_thinking = render_index == last_user_render_index;
|
||||
let after_last_user_turn = render_index > last_user_render_index;
|
||||
let after_or_at_last_user_turn = render_index >= last_user_render_index;
|
||||
|
||||
match message {
|
||||
ChatMessage::System { content } => render_system_message(out, Some(content), &[]),
|
||||
ChatMessage::Developer { content, tools } => render_developer_message(
|
||||
out,
|
||||
content,
|
||||
tools.as_deref().unwrap_or(&[]),
|
||||
thinking_mode == ThinkingMode::Thinking && opens_thinking,
|
||||
),
|
||||
ChatMessage::User { content } => render_user_message(
|
||||
out,
|
||||
content,
|
||||
thinking_mode == ThinkingMode::Thinking && opens_thinking,
|
||||
),
|
||||
ChatMessage::Assistant { content } => render_assistant_message(
|
||||
out,
|
||||
thinking_mode == ThinkingMode::Thinking && after_last_user_turn,
|
||||
content,
|
||||
should_keep_assistant_reasoning(
|
||||
message_index,
|
||||
last_user_actual_index,
|
||||
thinking_mode,
|
||||
drop_thinking,
|
||||
),
|
||||
// TODO: Respect `continue_final_message` and map it to DeepSeek's
|
||||
// prefix-style final-assistant continuation behavior.
|
||||
false,
|
||||
),
|
||||
ChatMessage::ToolResponse { content, .. } => render_tool_message(
|
||||
out,
|
||||
messages,
|
||||
message_index,
|
||||
thinking_mode == ThinkingMode::Thinking && after_or_at_last_user_turn,
|
||||
content,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Historical assistant reasoning is dropped in thinking mode when the final
|
||||
/// request turn is a new user-like message.
|
||||
fn should_keep_assistant_reasoning(
|
||||
actual_index: usize,
|
||||
last_user_actual_index: usize,
|
||||
thinking_mode: ThinkingMode,
|
||||
drop_thinking: bool,
|
||||
) -> bool {
|
||||
!(thinking_mode == ThinkingMode::Thinking
|
||||
&& drop_thinking
|
||||
&& actual_index < last_user_actual_index)
|
||||
}
|
||||
|
||||
/// Return the last user/developer turn in the real request message list.
|
||||
fn find_last_user_actual_index(messages: &[ChatMessage]) -> usize {
|
||||
messages
|
||||
.iter()
|
||||
.rposition(|message| matches!(message.role(), ChatRole::User | ChatRole::Developer))
|
||||
.unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
/// Render a system turn, optionally followed by the tool preamble.
|
||||
fn render_system_message(
|
||||
out: &mut String,
|
||||
content: Option<&ChatContent>,
|
||||
tools: &[ChatTool],
|
||||
) -> Result<()> {
|
||||
if let Some(content) = content {
|
||||
write_chat_content(out, content)?;
|
||||
}
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Developer messages are wrapped into the same user-like turn shape as real
|
||||
/// user messages, but can also carry message-local tools.
|
||||
fn render_developer_message(
|
||||
out: &mut String,
|
||||
content: &ChatContent,
|
||||
tools: &[ChatTool],
|
||||
opens_thinking: bool,
|
||||
) -> Result<()> {
|
||||
if content.is_empty() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 developer message: empty content".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str("<|User|>");
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
}
|
||||
out.push_str("\n\n# The user's message is: ");
|
||||
write_chat_content(out, content)?;
|
||||
write_user_like_suffix(out, opens_thinking);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Plain user turns share the same wrapper shape as developer turns without the
|
||||
/// developer-specific preamble.
|
||||
fn render_user_message(
|
||||
out: &mut String,
|
||||
content: &ChatContent,
|
||||
opens_thinking: bool,
|
||||
) -> Result<()> {
|
||||
out.push_str("<|User|>");
|
||||
write_chat_content(out, content)?;
|
||||
write_user_like_suffix(out, opens_thinking);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shared trailing wrapper used by both real user turns and native developer
|
||||
/// turns after their content has already been written.
|
||||
// TODO: respect `add_generation_prompt` option
|
||||
fn write_user_like_suffix(out: &mut String, opens_thinking: bool) {
|
||||
out.push_str("<|Assistant|>");
|
||||
if opens_thinking {
|
||||
out.push_str(THINKING_START_TOKEN);
|
||||
} else {
|
||||
out.push_str(THINKING_END_TOKEN);
|
||||
}
|
||||
}
|
||||
|
||||
/// Render one tool result turn and decide whether it opens or closes the shared
|
||||
/// `<function_results>` block for the preceding assistant tool-call message.
|
||||
fn render_tool_message(
|
||||
out: &mut String,
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
resumes_thinking: bool,
|
||||
_content: &ChatContent,
|
||||
) -> Result<()> {
|
||||
let (block_start, block_end) = tool_response_block_bounds(messages, message_index);
|
||||
if message_index != block_start {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(prev_assistant_idx) = previous_assistant_actual_index(messages, block_start) else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 tool message: missing previous assistant message".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let ChatMessage::Assistant {
|
||||
content: assistant_content,
|
||||
} = &messages[prev_assistant_idx]
|
||||
else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 tool message: previous non-tool message is not assistant"
|
||||
.to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let assistant_tool_calls = assistant_content.tool_calls().collect::<Vec<_>>();
|
||||
if assistant_tool_calls.is_empty() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 tool message: previous assistant message has no tool calls"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut expected_tool_call_ids = HashSet::with_capacity(assistant_tool_calls.len());
|
||||
for tool_call in &assistant_tool_calls {
|
||||
if !expected_tool_call_ids.insert(tool_call.id.as_str()) {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 assistant tool calls: duplicate tool_call_id".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut tool_results_by_id = HashMap::with_capacity(assistant_tool_calls.len());
|
||||
for message in &messages[block_start..block_end] {
|
||||
let ChatMessage::ToolResponse {
|
||||
content,
|
||||
tool_call_id,
|
||||
} = message
|
||||
else {
|
||||
unreachable!("tool response block should only contain tool messages");
|
||||
};
|
||||
|
||||
if !expected_tool_call_ids.contains(tool_call_id.as_str()) {
|
||||
return Err(Error::ChatTemplate(format!(
|
||||
"invalid DeepSeek V3.2 tool message: unknown tool_call_id `{tool_call_id}`"
|
||||
)));
|
||||
}
|
||||
|
||||
if tool_results_by_id.insert(tool_call_id.as_str(), content).is_some() {
|
||||
return Err(Error::ChatTemplate(format!(
|
||||
"invalid DeepSeek V3.2 tool message: duplicate tool_call_id `{tool_call_id}`"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if tool_results_by_id.len() != assistant_tool_calls.len() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 tool messages: missing tool result for assistant tool call"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str("\n\n<function_results>");
|
||||
for tool_call in assistant_tool_calls {
|
||||
let content = tool_results_by_id
|
||||
.get(tool_call.id.as_str())
|
||||
.expect("validated tool_call_id set should be complete");
|
||||
out.push_str("\n<result>");
|
||||
write_chat_content(out, content)?;
|
||||
out.push_str("</result>");
|
||||
}
|
||||
|
||||
out.push_str("\n</function_results>");
|
||||
out.push_str("\n\n");
|
||||
if resumes_thinking {
|
||||
out.push_str(THINKING_START_TOKEN);
|
||||
} else {
|
||||
out.push_str(THINKING_END_TOKEN);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the contiguous tool-response block containing `actual_index`.
|
||||
fn tool_response_block_bounds(messages: &[ChatMessage], actual_index: usize) -> (usize, usize) {
|
||||
let mut block_start = actual_index;
|
||||
while block_start > 0 && matches!(messages[block_start - 1], ChatMessage::ToolResponse { .. }) {
|
||||
block_start -= 1;
|
||||
}
|
||||
|
||||
let mut block_end = actual_index + 1;
|
||||
while block_end < messages.len()
|
||||
&& matches!(messages[block_end], ChatMessage::ToolResponse { .. })
|
||||
{
|
||||
block_end += 1;
|
||||
}
|
||||
|
||||
(block_start, block_end)
|
||||
}
|
||||
|
||||
/// Return the most recent assistant turn before `actual_index`.
|
||||
fn previous_assistant_actual_index(messages: &[ChatMessage], actual_index: usize) -> Option<usize> {
|
||||
messages[..actual_index]
|
||||
.iter()
|
||||
.rposition(|message| matches!(message, ChatMessage::Assistant { .. }))
|
||||
}
|
||||
|
||||
/// Render one assistant turn, including optional reasoning, DSML tool calls,
|
||||
/// and the trailing EOS marker.
|
||||
fn render_assistant_message(
|
||||
out: &mut String,
|
||||
after_last_user_turn: bool,
|
||||
content: &[AssistantContentBlock],
|
||||
keep_reasoning: bool,
|
||||
prefix: bool,
|
||||
) -> Result<()> {
|
||||
let has_reasoning = keep_reasoning && content.has_reasoning();
|
||||
let has_tool_calls = content.has_tool_calls();
|
||||
|
||||
if !has_tool_calls && prefix {
|
||||
write_assistant_text(out, content);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if after_last_user_turn {
|
||||
if !has_reasoning && !has_tool_calls {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V3.2 assistant message after last user message: expected reasoning or tool calls"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if has_reasoning {
|
||||
write_assistant_reasoning(out, content);
|
||||
}
|
||||
out.push_str(THINKING_END_TOKEN);
|
||||
}
|
||||
|
||||
write_assistant_text(out, content);
|
||||
|
||||
if has_tool_calls {
|
||||
out.push_str("\n\n<|DSML|function_calls>\n");
|
||||
for (index, tool_call) in content.tool_calls().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_call(out, tool_call)?;
|
||||
}
|
||||
out.push_str("\n</|DSML|function_calls>");
|
||||
}
|
||||
|
||||
out.push_str(EOS_TOKEN);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render one assistant tool call in DSML XML-like format.
|
||||
fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call)?;
|
||||
write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convert one assistant tool-call arguments object into DSML parameter form.
|
||||
///
|
||||
/// String values are emitted raw with `string="true"`, while all other JSON
|
||||
/// values are rendered with JSON syntax and `string="false"`.
|
||||
fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek V3.2: {error}"
|
||||
))
|
||||
})?;
|
||||
let Some(arguments) = arguments.as_object() else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"assistant tool call arguments for DeepSeek V3.2 must be a JSON object".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let mut wrote_parameter = false;
|
||||
for (key, value) in arguments {
|
||||
if wrote_parameter {
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
let is_string = matches!(value, Value::String(_));
|
||||
write!(
|
||||
out,
|
||||
"<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
|
||||
if is_string { "true" } else { "false" }
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
|
||||
match value {
|
||||
Value::String(value) => out.push_str(value),
|
||||
value => out.push_str(&json_dumps(value)?),
|
||||
}
|
||||
|
||||
write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
|
||||
wrote_parameter = true;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render the full tool preamble shown to the model.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
|
||||
out.push_str(
|
||||
r#"## Tools
|
||||
|
||||
You have access to a set of tools you can use to answer the user's question.
|
||||
You can invoke functions by writing a "<|DSML|function_calls>" block like the following as part of your reply to the user:
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="$FUNCTION_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$FUNCTION_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls>
|
||||
|
||||
String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects).
|
||||
|
||||
If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example:
|
||||
|
||||
<|DSML|function_calls>
|
||||
...
|
||||
</|DSML|function_calls>
|
||||
|
||||
<function_results>
|
||||
...
|
||||
</function_results>
|
||||
|
||||
<think>...thinking about results</think>
|
||||
|
||||
Here are the functions available in JSONSchema format:
|
||||
<functions>
|
||||
"#,
|
||||
);
|
||||
|
||||
for (index, tool) in tools.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_schema(out, tool)?;
|
||||
}
|
||||
|
||||
out.push_str("\n</functions>\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialize one typed tool schema into the JSON shape embedded inside
|
||||
/// `<functions>`.
|
||||
fn render_tool_schema(out: &mut String, tool: &ChatTool) -> Result<()> {
|
||||
out.push_str(&json_dumps(&RenderedToolSchema {
|
||||
name: &tool.name,
|
||||
description: tool.description.as_deref(),
|
||||
parameters: &tool.parameters,
|
||||
strict: tool.strict,
|
||||
})?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write chat content directly into the destination buffer without flattening
|
||||
/// it into an intermediate `String`.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => out.push_str(text),
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
out.push_str(part.as_text()?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write all reasoning blocks in encounter order.
|
||||
fn write_assistant_reasoning(out: &mut String, content: &[AssistantContentBlock]) {
|
||||
for block in content {
|
||||
if let AssistantContentBlock::Reasoning { text } = block {
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write all visible assistant text blocks in encounter order.
|
||||
fn write_assistant_text(out: &mut String, content: &[AssistantContentBlock]) {
|
||||
for block in content {
|
||||
if let AssistantContentBlock::Text { text } = block {
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compact JSON serialization used by this renderer for exact prompt text.
|
||||
fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
|
||||
JsonFormat::new()
|
||||
.comma(", ")
|
||||
.expect("literal comma separator is valid JSON")
|
||||
.colon(": ")
|
||||
.expect("literal colon separator is valid JSON")
|
||||
.ascii(false)
|
||||
.format_to_string(value)
|
||||
.map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"failed to serialize DeepSeek V3.2 JSON payload: {error}"
|
||||
))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_datetime",
|
||||
"description": "Get the current date and time",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"timezone": {
|
||||
"type": "string",
|
||||
"description": "The timezone, e.g. Asia/Shanghai, UTC"
|
||||
}
|
||||
},
|
||||
"required": ["timezone"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a specific date and location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city name, e.g. Beijing, Hangzhou"
|
||||
},
|
||||
"date": {
|
||||
"type": "string",
|
||||
"description": "The date in YYYY-MM-DD format"
|
||||
}
|
||||
},
|
||||
"required": ["location", "date"]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful Assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "明天杭州和北京的天气怎么样?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "用户询问明天的天气,我需要先获取当前日期来计算明天的日期📅",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_xK9mN3pL2qR8vT5wY6hZ1aB4",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"arguments": "{\"timezone\": \"Asia/Shanghai\"}",
|
||||
"name": "get_datetime"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"tool_call_id": "call_xK9mN3pL2qR8vT5wY6hZ1aB4",
|
||||
"role": "tool",
|
||||
"content": "{\"current_date\": \"2024-01-15\", \"current_time\": \"14:30:00\", \"timezone\": \"Asia/Shanghai\"}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "现在知道今天是2024-01-15,明天就是2024-01-16。接下来查询杭州和北京明天的天气🌤️",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_bN7kR9mX3pQ2wL5vY8jZ4cD6",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"arguments": "{\"location\": \"Hangzhou\", \"date\": \"2024-01-16\"}",
|
||||
"name": "get_weather"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "call_dP9mL7kX5rT4yN3wZ2hV8eF1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"arguments": "{\"location\": \"Beijing\", \"date\": \"2024-01-16\"}",
|
||||
"name": "get_weather"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"tool_call_id": "call_bN7kR9mX3pQ2wL5vY8jZ4cD6",
|
||||
"role": "tool",
|
||||
"content": "{\"location\": \"Hangzhou\", \"date\": \"2024-01-16\", \"temperature_high\": \"12\", \"temperature_low\": \"5\", \"weather\": \"多云\", \"humidity\": \"65%\"}"
|
||||
},
|
||||
{
|
||||
"tool_call_id": "call_dP9mL7kX5rT4yN3wZ2hV8eF1",
|
||||
"role": "tool",
|
||||
"content": "{\"location\": \"Beijing\", \"date\": \"2024-01-16\", \"temperature_high\": \"-2\", \"temperature_low\": \"-8\", \"weather\": \"晴\", \"humidity\": \"30%\"}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "已获取两个城市明天的天气信息,现在整理给用户✨",
|
||||
"content": "根据查询结果,明天(2024年1月16日)的天气情况如下:\n\n**杭州**:\n- 天气:多云\n- 最高温度:12°C\n- 最低温度:5°C\n- 湿度:65%\n\n**北京**:\n- 天气:晴\n- 最高温度:-2°C\n- 最低温度:-8°C\n- 湿度:30%\n\n杭州明天会比较温暖但有些多云,而北京会很冷但是晴天。建议在北京的朋友要注意保暖!"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "谢谢!那后天呢?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "用户现在问后天的天气,后天是2024-01-17,我可以直接查询(因为已知今天日期)🗓️",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_fR3nK8mV7pL4xT2yW9jB5gH3",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"arguments": "{\"location\": \"Hangzhou\", \"date\": \"2024-01-17\"}",
|
||||
"name": "get_weather"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "call_hT5pN2kY9rV6zL3wX1mD7jK8",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"arguments": "{\"location\": \"Beijing\", \"date\": \"2024-01-17\"}",
|
||||
"name": "get_weather"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"tool_call_id": "call_fR3nK8mV7pL4xT2yW9jB5gH3",
|
||||
"role": "tool",
|
||||
"content": "{\"location\": \"Hangzhou\", \"date\": \"2024-01-17\", \"temperature_high\": \"15\", \"temperature_low\": \"8\", \"weather\": \"小雨\", \"humidity\": \"80%\"}"
|
||||
},
|
||||
{
|
||||
"tool_call_id": "call_hT5pN2kY9rV6zL3wX1mD7jK8",
|
||||
"role": "tool",
|
||||
"content": "{\"location\": \"Beijing\", \"date\": \"2024-01-17\", \"temperature_high\": \"0\", \"temperature_low\": \"-6\", \"weather\": \"多云\", \"humidity\": \"45%\"}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "获取到后天的天气数据,整理回复给用户📝",
|
||||
"content": "后天(2024年1月17日)的天气情况:\n\n**杭州**:\n- 天气:小雨\n- 最高温度:15°C\n- 最低温度:8°C\n- 湿度:80%\n\n**北京**:\n- 天气:多云\n- 最高温度:0°C\n- 最低温度:-6°C\n- 湿度:45%\n\n杭州后天会有小雨,温度略有回升,记得带伞。北京会稍微暖和一点,但依然很冷,请继续做好保暖措施。"
|
||||
}
|
||||
]
|
||||
}
|
||||
+732
File diff suppressed because one or more lines are too long
+533
@@ -0,0 +1,533 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "In a year before 2023, between December 1 and 15 (inclusive), a press release reported that as of the 5th of that month, a country had received a total of 16,201,670 vaccine doses, with 7,583,134 administered. That same year, a company's annual report revealed that its nearly 2.2 million members and supporters contributed over $49 million, with 12% of total revenue coming from contributed services. The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan to open most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling and that contributed services had decreased by 2% to 10%. Who was the emeritus board member that year? Name only, no titles.",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Searches for information related to query and displays topn results.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The search query string"
|
||||
},
|
||||
"topn": {
|
||||
"type": "integer",
|
||||
"description": "Number of top results to display",
|
||||
"default": 10
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Source to search within",
|
||||
"enum": [
|
||||
"web",
|
||||
"news"
|
||||
],
|
||||
"default": "web"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"description": "Opens the link id from the page indicated by cursor starting at line number loc, showing num_lines lines. Valid link ids are displayed with the formatting: 【{id}†.*】. If cursor is not provided, the most recent page is implied. If id is a string, it is treated as a fully qualified URL associated with source. If loc is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. Use this function without id to scroll to a new location of an opened page.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"description": "Link ID (number) or fully qualified URL (string)",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"default": -1
|
||||
},
|
||||
"cursor": {
|
||||
"type": "integer",
|
||||
"description": "Page cursor indicator",
|
||||
"default": -1
|
||||
},
|
||||
"loc": {
|
||||
"type": "integer",
|
||||
"description": "Starting line number",
|
||||
"default": -1
|
||||
},
|
||||
"num_lines": {
|
||||
"type": "integer",
|
||||
"description": "Number of lines to show",
|
||||
"default": -1
|
||||
},
|
||||
"view_source": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to view source",
|
||||
"default": false
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Source associated with the URL",
|
||||
"enum": [
|
||||
"web",
|
||||
"news"
|
||||
],
|
||||
"default": "web"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"description": "Finds exact matches of pattern in the current page, or the page given by cursor.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {
|
||||
"type": "string",
|
||||
"description": "The exact pattern to search for"
|
||||
},
|
||||
"cursor": {
|
||||
"type": "integer",
|
||||
"description": "Page cursor to search in",
|
||||
"default": -1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pattern"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "python",
|
||||
"description": "Use this tool to execute Python code. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string",
|
||||
"description": "The python code to execute in the jupyter environment."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"code"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "The user asks: \"In a year before 2023, between December 1 and 15 (inclusive), a press release reported that as of the 5th of that month, a country had received a total of 16,201,670 vaccine doses, with 7,583,134 administered. That same year, a company's annual report revealed that its nearly 2.2 million members and supporters contributed over $49 million, with 12% of total revenue coming from contributed services. The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan to open most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling and that contributed services had decreased by 2% to 10%. Who was the emeritus board member that year? Name only, no titles.\"\n\nWe need to parse the puzzle.\n\nWe have clues about a press release about vaccine doses for a country, dated between Dec 1-15 inclusive, reporting as of 5th of that month total doses received 16,201,670 and administered 7,583,134. That is likely a COVID-19 vaccine update for some country. Which country? Might be a smaller country. Could be something like \"Uganda\", \"Zimbabwe\", \"Kenya\", \"Philippines\", etc. Let's search.\n\nSearch for \"16,201,670 vaccine doses\" \"7,583,134 administered\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-1-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"\\\"16,201,670\\\" vaccine doses\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[0] \"16,201,670\" vaccine doses (Search_Results/\"16,201,670\" vaccine doses)\n**viewing lines [0 - 38] of 38**\n\nL0: \nL1: URL: Search_Results/\"16,201,670\" vaccine doses\nL2: # Search Results\nL3: \nL4: * 【0†Kenya's Economy is Showing Resilience as Output Rises ...; publish_date: \nL5: none†www.worldbank.org】 Dec 14, 2021 — As of December 5, 2021, Kenya had \nL6: received a total of 16,201,670 vaccines, with 7,583,134 administered. While \nL7: vaccine acceptance is ...\nL8: * 【1†MINISTRY OF HEALTH; publish_date: none†covidhub.mediacouncil.or.ke】 Dec \nL9: 1, 2021 — Total Doses Received 16,201,670. Table 10 gives the total vaccines \nL10: received since the start of Covid -19 vaccination exercise in the country.\nL11: * 【2†Output Result Page; publish_date: none†open.unicef.org】 ... 16,201,670 \nL12: doses of multiple vaccines nationwide and full vaccination of 15.5 per cent with\nL13: two doses of COVID-19 vaccine as of 31 December 2021.\nL14: * 【3†rebased GDP; publish_date: none†documents1.worldbank.org】 Dec 7, 2021 — \nL15: As of December 5, 2021,. Kenya had received a total of 16,201,670 vaccines, \nL16: with. 7,583,134 administered. Vaccine acceptance is reportedly high.\nL17: * 【4†Integrated Annual Report; publish_date: none†www.co-opbank.co.ke】 May 27,\nL18: 2022 — ... doses of Covid-19 vaccines and administered close to 17 million ... \nL19: 16,201,670, huku 7,583,134 zikiwa tayari zimedungwa watu. Bado kuna ...\nL20: * 【5†World Bank lifts Kenya's growth prospect to 5% in 2021; publish_date: \nL21: none†www.africa-press.net】 ... 16,201,670 vaccines, with 7,583,134 administered.\nL22: While vaccine acceptance is reportedly high there is still a long way to go \nL23: towards the government's ...\nL24: * 【6†2020 Annual Report United States; publish_date: none†www.hi-us.org】 \nL25: including medicine, vaccines, Covid-19 prevention kits, and kits of ... \nL26: 16,201,670. 9,630,584. Prepaid expenses. 56,707. 22,601. Total current assets.\nL27: * 【7†HOSPITAL CENTRAL DE LA DEFENSA; publish_date: \nL28: none†contrataciondelestado.es】 16201670 X. URPA-URPA. 0. HCD-000393/00. KONTRON.\nL29: 7000 ... LASER SHOT LBP-1120. CNEUR-C. NEUROFISIOLOGÍA. 0 ... VICTOREEN CT HEAD\nL30: DOSE PHANTON. 43650-014.\nL31: * 【8†body metal concentrations: Topics by ...; publish_date: \nL32: none†www.science.gov】 PMID:16201670. Design Characteristics and Tobacco Metal \nL33: Concentrations in Filtered Cigars. PubMed. Caruso, Rosalie V; O'Connor, Richard \nL34: J; Travers, Mark J ...\nL35: * 【9†Public Offices, Personal Demands: Capability in ...; publish_date: \nL36: none†dokumen.pub】 ... (16201670).9 The Institutiones Politicae consists of two \nL37: books. Book one deals with the origin, nature and goal of the commonwealth on \nL38: the one hand, and the ...",
|
||||
"tool_call_id": "fixture-tool-call-1-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Result shows Kenya. So the press release is about Kenya's vaccine doses as of Dec 5, 2021. The press release likely by World Bank or something. The search result shows \"World Bank lifts Kenya's growth prospect...\" maybe a press release dated Dec 14, 2021. But the clue says \"between December 1 and 15 (inclusive), a press release reported that as of the 5th of that month, a country had received a total of 16,201,670 vaccine doses, with 7,583,134 administered.\" That matches Kenya on Dec 5, 2021. So the year is 2021.\n\nNow the next clue: \"That same year, a company's annual report revealed that its nearly 2.2 million members and supporters contributed over $49 million, with 12% of total revenue coming from contributed services.\"\n\nThis sounds like an environmental or conservation organization. \"Nearly 2.2 million members and supporters contributed over $49 million.\" That might be something like \"The Nature Conservancy\", \"World Wildlife Fund\", \"Sierra Club\", \"Environmental Defense Fund\", \"Natural Resources Defense Council\", etc. Let's search.\n\nSearch for \"nearly 2.2 million members and supporters contributed over $49 million\" \"12% of total revenue coming from contributed services\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-3-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"\\\"nearly 2.2 million members\\\" \\\"contributed over $49 million\\\"\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[1] \"nearly 2.2 million members\" \"contributed over $49 million\" (Search_Results/\"nearly 2.2 million members\" \"contributed over $49 million\")\n**viewing lines [0 - 41] of 41**\n\nL0: \nL1: URL: Search_Results/\"nearly 2.2 million members\" \"contributed over $49 million\"\nL2: # Search Results\nL3: \nL4: * 【0†The Global Good Fund, Microsoft and Goodwill join forces ...; \nL5: publish_date: none†globalgoodfund.org】 Sep 28, 2022 — ... more than 2.2 million \nL6: people. They additionally contribute close to $49 billion in local, state and \nL7: federal tax revenues. When COVID hit ...\nL8: * 【1†Almost 22 billion American tax dollars spent to wipe out a ...; \nL9: publish_date: none†www.facebook.com】 US military funding for Israel's war crimes\nL10: in Lebanon and Gaza has now cost US taxpayers over $22 billion. When millions \nL11: struggle to afford the ...\nL12: * 【2†Corporate America has largely abandoned its post-January ...; \nL13: publish_date: none†www.citizensforethics.org】 Jul 29, 2025 — Since the January 6\nL14: insurrection, over 2,000 corporate and industry group PACs have given over $174\nL15: million to members of the Sedition ...\nL16: * 【3†Audit shows millions in questionable taxpayer spending at ...; \nL17: publish_date: none†www.aol.com】 18 hours ago — ... nearly doubled from 1.3 \nL18: million to about 2.2 million. That is more than one in four Washington state \nL19: residents receiving Medicaid, and the ...\nL20: * 【4†Incarceration and Poverty in the United States - AAF; publish_date: \nL21: none†www.americanactionforum.org】 Jun 30, 2020 — The United States currently \nL22: incarcerates 2.2 million people, nearly half of whom are non-violent drug \nL23: offenders, accused people held pre ...\nL24: * 【5†How the NRA Spends Revenue (2023); publish_date: none†paddockpost.com】 \nL25: Apr 25, 2025 — In 2023, the NRA reported total revenue of $176 million (compared\nL26: to $211 million in 2022, $227 million in 2021, $282 million in 2020, $292 ...\nL27: * 【6†NCA Newsroom; publish_date: none†www.ncausa.org】 ... 2.2 million U.S. \nL28: jobs—operating in every U.S. state and territory and contributing nearly $350 \nL29: billion to the U.S. economy every year. For media inquiries ...\nL30: * 【7†The Early Childhood Educator Workforce; publish_date: \nL31: none†cscce.berkeley.edu】 About 2.2 million adults are paid to care for and \nL32: educate more than 9.7 million children between birth and age five in center-\nL33: based and home-based settings.\nL34: * 【8†Bringing Out the Worst in Congress: CRA By the Numbers ...; publish_date:\nL35: none†progressivereform.org】 Jul 14, 2025 — In fact, the top ten recipients (out\nL36: of 25) account for almost $7 million, showing how these contributions are \nL37: targeted and strategic. Real ...\nL38: * 【9†Club 100: Over 100 Americans gave more than $100000 to ...; publish_date:\nL39: none†publicintegrity.org】 Jan 7, 2022 — Through Internet investments during the\nL40: 1990s, he attracted 2.2 million online accounts valued at $174 million. Schwab \nL41: has been a vocal ...",
|
||||
"tool_call_id": "fixture-tool-call-3-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Not that. Try search for \"2.2 million members and supporters contributed over $49 million\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-5-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"\\\"2.2 million members\\\" \\\"contributed over $49\\\"\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[2] \"2.2 million members\" \"contributed over $49\" (Search_Results/\"2.2 million members\" \"contributed over $49\")\n**viewing lines [0 - 2] of 2**\n\nL0: \nL1: URL: Search_Results/\"2.2 million members\" \"contributed over $49\"\nL2: # Search Results",
|
||||
"tool_call_id": "fixture-tool-call-5-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Maybe it's an annual report of a non-profit. Search for \"annual report 2.2 million members contributed $49 million\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-7-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"\\\"2.2 million members\\\" \\\"annual report\\\" $49 million\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[3] \"2.2 million members\" \"annual report\" $49 million (Search_Results/\"2.2 million members\" \"annual report\" $49 million)\n**viewing lines [0 - 36] of 36**\n\nL0: \nL1: URL: Search_Results/\"2.2 million members\" \"annual report\" $49 million\nL2: # Search Results\nL3: \nL4: * 【0†20-F; publish_date: none†www.sec.gov】 ANNUAL REPORT PURSUANT TO SECTION \nL5: ... Our membership grew from 2.1 million members as at December 31, 2023 to 2.2 \nL6: million members as at December 31, 2024.\nL7: * 【1†Oportun Reports Fourth Quarter and Full Year 2023 ...; publish_date: \nL8: none†investor.oportun.com】 Mar 12, 2024 — Oportun (Nasdaq: OPRT) is a mission-\nL9: driven fintech that puts its 2.2 million members' financial goals within reach. \nL10: ... annual report on ...\nL11: * 【2†2 0 21; publish_date: none†www.annualreports.com】 ANNUAL REPORT. 2. 0. \nL12: 21. 2. 0. 21. Page 2. 2. DEFENDERS OF WILDLIFE. 2. 0. 21. 2. 0. 21 ... In 2021, \nL13: Defenders of Wildlife's nearly 2.2 million members and.\nL14: * 【3†Annual report and accounts 2020; publish_date: none†www.3i.com】 \nL15: Disclaimer. The Annual report and accounts have been prepared solely to provide \nL16: information to shareholders. ... 2.2 million members. In December 2019, we sold \nL17: ...\nL18: * 【4†united states securities and exchange commission; publish_date: \nL19: none†s21.q4cdn.com】 Dec 14, 2018 — ☒ ANNUAL REPORT ... approximately 5.7 million\nL20: Members, including approximately 2.2 million Members from the acquisition of \nL21: Best Doctors.\nL22: * 【5†securities and exchange commission; publish_date: none†www.sec.gov】 ... \nL23: Annual Report on Form 10-K filed with the SEC on March 10, 2020. The ... 2.2 \nL24: million members at June 30, 2020. The following table highlights the ...\nL25: * 【6†2022-23 Annual Comprehensive Financial Report - CalPERS; publish_date: \nL26: none†www.calpers.ca.gov】 Nov 17, 2023 — ... annual report is consistent with \nL27: these accounting principles ... 2.2 million members and health benefit services \nL28: to over 1.5 million ...\nL29: * 【7†Medical Centers Report; publish_date: \nL30: none†regents.universityofcalifornia.edu】 Oct 8, 2019 — which will allow its 2.2 \nL31: million members to receive advanced and ... annual report of the University of \nL32: California. For the Medical ...\nL33: * 【8†Working TogeTher to make HealtH care better; publish_date: \nL34: none†www.unitedhealthgroup.com】 Feb 8, 2012 — ANNUAL REPORT PURSUANT TO SECTION \nL35: 13 OR 15(d) OF THE ... approximately 2.2 million members enrolled in its \nL36: Medicare. Advantage ...",
|
||||
"tool_call_id": "fixture-tool-call-7-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Result 2 shows \"Defenders of Wildlife\". That might be the organization. Let's open.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-9-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 3, \"id\": 2}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[4] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2021.pdf)\n**viewing lines [0 - 53] of 261**\n\nL0: \nL1: URL: https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-\nL2: wildlife_2021.pdf\nL3: ===== Page 1 ===== 2021 ANNUAL REPORT ===== Page 2 ===== # DEFENDERS OF WILDLIFE\nL4: made important progress for imperiled species and vital landscapes across the \nL5: United States in 2021. \\--- **LAWYERS** immediately **challenged** the premature\nL6: and reckless decision to strip **gray wolves** of federal **Endangered Species \nL7: Act (ESA)** protections. For many decades, Defenders has led the effort to \nL8: protect and restore the gray wolf, and we will continue to fight the \nL9: unscientific and hostile anti-wolf policies that impede conservation progress \nL10: and will carry on our unrelenting battle to restore federal protections for this\nL11: iconic keystone species. \\--- **LOBBYISTS** worked around the clock to keep \nL12: wildlife and climate priorities in the **Infrastructure Investment and Jobs \nL13: Act**. We also continue fighting to keep important wildlife and habitat funding \nL14: in relevant **appropriations bills**. \\--- 2 DEFENDERS OF WILDLIFE ===== Page 3 \nL15: ===== POLICY EXPERTS pushed forward on the urgent need for a National \nL16: Biodiversity Strategy (NBS), an all-of-government approach to address the \nL17: unprecedented loss of wildlife and habitat we are experiencing. We have coupled \nL18: this with our new campaign to expand the National Wildlife Refuge System to \nL19: preserve our nation’s only lands set aside for wildlife. By defending, funding \nL20: and expanding our national wildlife refuges, we will directly address \nL21: biodiversity loss and climate change while promoting increased equitable access \nL22: to nature. FIELD TEAMS were on the ground helping to recover imperiled species. \nL23: From panthers and sea turtles in Florida to wolves, bison and black-footed \nL24: ferrets in Montana, Defenders’ conservation experts were in the field saving \nL25: wildlife all over the country. CONSERVATION INNOVATION EXPERTS provided \nL26: comprehensive analyses to guide policy and inform conservation strategies to \nL27: reach the goal of protecting 30% of our terrestrial and marine systems by 2030 \nL28: (“30x30”). Defenders’ Center for Conservation Innovation (CCI) produced a report\nL29: which details actions we need to take to achieve 30x30 while protecting \nL30: biodiversity and addressing the climate crisis. DEFENDERS.ORG ===== Page 4 =====\nL31: WE HAVE ACCOMPLISHED MUCH THIS YEAR WORKING WITH AN ADMINISTRATION THAT VALUES \nL32: SCIENCE AND CONSERVATION. Our many successes include the return of protections \nL33: to the Tongass National Forest in Alaska and the suspension of oil and gas \nL34: leasing permits for the Arctic National Wildlife Refuge. Defenders also played a\nL35: lead role in successfully defending the Migratory Bird Protection Act from \nL36: attack and securing critical protections for migratory birds. Throughout 2021 \nL37: Defenders made important progress for the wildlife and wild places we all love, \nL38: yet our nation’s wildlife still face unprecedented challenges. More and more \nL39: species, including manatees, Mexican gray wolves, polar bears and North Atlantic\nL40: right whales, face the very real threat of extinction because of climate \nL41: change, habitat loss, pollution and inadequate protections. In our work we \nL42: continue to demonstrate success by following the science, building partnerships,\nL43: leading with innovative solutions and focusing on collaborative and practical \nL44: approaches in defense of nature. Together we can make a lasting difference for \nL45: wildlife and habitats. Your support is essential to our work and we couldn’t do \nL46: it without you. Thank you for being an important part of our efforts to protect \nL47: imperiled wildlife across our nation. Jamie Rappaport Clark President and CEO \nL48: John Dayton Board Chair OUR NEW TELEVISION SERIES, WILDLIFE NATION WITH JEFF \nL49: CORWIN, premiered in October and airs every weekend on ABC stations nationwide. \nL50: This groundbreaking series takes Jeff across the country to meet Defenders’ \nL51: experts along with our science, tribal and community partners to educate and \nL52: inspire viewers to join with Defenders to protect wildlife. While highlighting \nL53: some of our important conservation work and raising awareness of important ",
|
||||
"tool_call_id": "fixture-tool-call-9-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Search within PDF for \"2.2 million members\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-11-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"arguments": "{\"cursor\": 4, \"pattern\": \"2.2\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[5] Find results for text: `2.2` in `www.annualreports.com` (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2021.pdf/find?pattern=2.2)\n**viewing lines [0 - 4] of 4**\n\nL0: # 【0†match at L187】\nL1: 2021, Defenders of Wildlife’s nearly 2.2 million members and supporters \nL2: contributed more than $49 million for wildlife and wildlife habitat. This figure\nL3: includes all those who generously donated their time and expertise. The \nL4: steadfast support of our donors allows Defenders to sustain our program and ",
|
||||
"tool_call_id": "fixture-tool-call-11-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open around L180.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-13-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 4, \"loc\": 170}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[6] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2021.pdf)\n**viewing lines [170 - 215] of 261**\n\nL170: into the wild. **10 DEFENDERS OF WILDLIFE** ===== Page 11 ===== AFTER ANOTHER \nL171: SUCCESSFUL DEFENDERS LAWSUIT ON BEHALF OF THE CRITICALLY ENDANGERED RED WOLF, \nL172: FWS reversed its decision to limit the recovery area and committed to a robust \nL173: release strategy. After years of inaction, FWS released eight wolves to the wild\nL174: in North Carolina and plan to release nine more wolves in the spring of 2022. \nL175: This is an incredible win for this critically endangered species whose \nL176: population has dwindled down to single digits in the wild because of \nL177: mismanagement, vehicle strikes and poaching. DEFENDERS CONTINUED TO LEAD EFFORTS\nL178: TO PROTECT THE FLORIDA MANATEE, a beloved species that suffered the deadliest \nL179: year on record in 2021, tragically surpassing 1,000 deaths because of water \nL180: pollution and lack of warm water habitat. Defenders led advocacy and education \nL181: aimed at restoring the natural flow of the dammed Ocklawaha River, which would \nL182: provide critical warm-water habitat that manatees need to survive. Defenders’ \nL183: legal team continued to fight for manatees in the courts, holding government \nL184: agencies accountable for protecting critical habitat and addressing the \nL185: devastating water pollution that is killing the seagrass and causing manatees to\nL186: starve. DAVID TES | SAM FRENZY DRAW DEFENDERS.ORG 11 ===== Page 12 ===== In \nL187: 2021, Defenders of Wildlife’s nearly 2.2 million members and supporters \nL188: contributed more than $49 million for wildlife and wildlife habitat. This figure\nL189: includes all those who generously donated their time and expertise. The \nL190: steadfast support of our donors allows Defenders to sustain our program and \nL191: public education efforts in the field, the courts and on Capitol Hill. 2021 \nL192: SOURCES OF FUNDS Grants and contributions $29,057 Bequests, trusts and split \nL193: interests $7,692 Income from investments, annuity reserve funds and trusts \nL194: $3,354 Royalties and other income $3,576 Contributed services $6,140 Total \nL195: Revenue $49,819 USES OF FUNDS Biodiversity conservation $22,420 Constituency \nL196: mobilization $16,324 Fundraising $1,211 Management and general $5,865 Total \nL197: Expenses $45,820 Change in net assets $3,999 Net assets, start of year $41,145 \nL198: Net Assets, End of the Year $45,144 Dollars are in thousands. 12 DEFENDERS OF \nL199: WILDLIFE Grants and contributions 58% Income from investments 7% Requests, \nL200: trusts and split interests 15% Royalties and other income 7% Contributed \nL201: services 12% Program and support services 97% Fundraising 3% ===== Page 13 =====\nL202: DEFENDERS.ORG 13 ===== Page 14 ===== John Dayton, Esq.* Chair Texas Sheryl \nL203: Sachs* Vice Chair Connecticut Whitfield Bailey* Secretary Tennessee Mark Caylor*\nL204: Treasurer District of Columbia Sylvia Baca* California Donald Barry, Esq. \nL205: Oregon Dinah Bear, Esq. Arizona Kim O’Keefe Beck, Ph.D California Gabriela \nL206: Chavarria, Ph.D Washington Jamie Rappaport Clark* President & CEO Virginia \nL207: Caroline D. Gabel Maryland Keith M. Harper, Esq. Maryland John Donovan, Esq. \nL208: Montana and California James M. Hecker, Esq. Virginia Judith Ponsikoff, Ph.D* \nL209: California Terry L. Root, Ph.D Florida BOARD EMERITUS Terry C. Pelster, Esq. New\nL210: York *Executive Committee member* ST A F F & A D V I S O R S SCIENCE ADVISORS \nL211: Gabriela Chavarria, Ph.D Burke Museum, University of Washington Leah Gerber, \nL212: Ph.D Arizona State University Kathayoon Khalil, Ph.D Oregon Zoo Rurik List, Ph.D\nL213: Universidad Autónoma Metropolitana-Lerma Barry Noon, Ph.D Colorado State \nL214: University Terry L. Root, Ph.D Stanford University Mark Schwartz, Ph.D \nL215: University of California-Davis David S. Wilcox, Ph.D Princeton University Rae ",
|
||||
"tool_call_id": "fixture-tool-call-13-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Scrolling further for \"Contributed services 12%\". Already seen. So the annual report is Defenders of Wildlife 2021 annual report.\n\nNext clue: \"The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan to open most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling and that contributed services had decreased by 2% to 10%.\"\n\nThus, the following year (2022) annual report of Defenders of Wildlife includes mention of rescinding plan to open most of 22 million acres of NPR-A to oil and gas drilling. Also contributed services decreased to 10% (from 12% in 2021). So we need to find the 2022 annual report for Defenders of Wildlife.\n\nSearch for \"Defenders of Wildlife 2022 annual report National Petroleum Reserve Alaska\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-15-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"Defenders of Wildlife 2022 annual report National Petroleum Reserve Alaska\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[7] Defenders of Wildlife 2022 annual report National Petroleum Reserve Alaska (Search_Results/Defenders of Wildlife 2022 annual report National Petroleum Reserve Alaska)\n**viewing lines [0 - 37] of 37**\n\nL0: \nL1: URL: Search_Results/Defenders of Wildlife 2022 annual report National Petroleum \nL2: Reserve Alaska\nL3: # Search Results\nL4: \nL5: * 【0†CELEBRATING YEARS; publish_date: none†www.annualreports.com】 With less \nL6: than 340 right whales left in the wild, Defenders is fighting tirelessly to end \nL7: deadly lobster gear entanglements and vessel strikes that are driving ...\nL8: * 【1†Financials; publish_date: none†defenders.org】 We invite you to explore \nL9: the reports below to learn more about our activities and accomplishments, and \nL10: how we put your money to work for wildlife.\nL11: * 【2†Alaska Program Looks Back on 2022; publish_date: none†defenders.org】 Feb \nL12: 9, 2023 — Thanks to a lawsuit joined by Defenders, seven million acres were \nL13: returned to protection within the National Petroleum Reserve-Alaska (NPR-A), ...\nL14: * 【3†Defenders-of-Wildlife-2022-Financial-Statement. ...; publish_date: \nL15: none†defenders.org】 We have audited the accompanying consolidated financial \nL16: statements of Defenders of Wildlife and Affiliated Defenders of Wildlife Action \nL17: Fund (collectively, ...\nL18: * 【4†2022 Annual Report; publish_date: none†alaskaconservation.org】 Jun 13, \nL19: 2023 — In 2022, we focused on three landscapes: the Arctic. National Wildlife \nL20: Refuge, Bristol Bay, and the Tongass National Forest. In March 2022,.\nL21: * 【5†Assessment of ecological and cultural values within the ...; \nL22: publish_date: none†www.blm.gov】 This document was written to provide technical \nL23: information regarding the ecological importance of the National Petroleum \nL24: Reserve – Alaska (NPR-A). Several ...\nL25: * 【6†Accomplishments Report; publish_date: none†defenders.org】 National \nL26: Petroleum Reserve-Alaska. Identified for exceptional wildlife and cultural \nL27: values, including critical habitat for polar bears and other species ...\nL28: * 【7†2022 annual report; publish_date: none†dory-\nL29: plantain-s2zc.squarespace.com】 These projects are made possible through \nL30: collaborations with Defenders of. Wildlife, Cook Inletkeeper, Trustees for \nL31: Alaska,. Environmental Investigation Agency, ...\nL32: * 【8†23IMPACT REPORT; publish_date: none†www.annualreports.com】 Defenders of \nL33: Wildlife made incredible progress protecting wildlife and wild places in 2023, \nL34: helping shape a brighter future for imperiled species and vital ...\nL35: * 【9†From Leasing to Land Protections; publish_date: none†defenders-cci.org】 \nL36: Abstract. When you hear the name, National Petroleum Reserve - Alaska (NPR-A, or\nL37: Reserve), you might think of a barren land filled with oil wells and ...",
|
||||
"tool_call_id": "fixture-tool-call-15-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open result 0.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-17-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 7, \"id\": 0}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[8] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf)\n**viewing lines [0 - 53] of 289**\n\nL0: \nL1: URL: https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-\nL2: wildlife_2022.pdf\nL3: ===== Page 1 ===== 2022 Impact Report C E L E B R A T I N G Y E A R S ===== Page\nL4: 2 ===== 2022 Defenders of Wildlife made important progress for imperiled \nL5: species and vital landscapes across the United States in 2022. GRAY WOLF | JIM \nL6: GUMMERAL MAY STOCK PRIOR Lawyers successfully challenged the previous \nL7: administration’s decision to delist the gray wolf and restored critical federal \nL8: protections under the Endangered Species Act. This latest triumph in court is \nL9: part of our ongoing battle to protect and restore gray wolves throughout their \nL10: historical range and shield them from persecution by extremist legislators in \nL11: Idaho, Montana and Wyoming. TWO MORE FATALIZED GRAY SWALLETS TO SEA TO SHARE \nL12: ALLIANCE Lobbyists worked around the clock to expand funding for wildlife \nL13: conservation in the FY2022 federal spending bill, which included $31 million (a \nL14: 44% increase) for the Bureau of Land Management’s Threatened and Endangered \nL15: Species Program, $2.5 million (an 81% increase) for the U.S. Department of \nL16: Agriculture Wildlife Services’ Nonlethal Initiative to prevent human-wildlife \nL17: conflicts and $21 million (a 320% increase) for North Atlantic right whale \nL18: conservation. 2 DEFENDERS OF WILDLIFE ===== Page 3 ===== **Policy Experts** \nL19: played a crucial role in securing international trade protections for 100 \nL20: species of sharks and rays, all 158 species of glass frogs and 73 species of \nL21: reptiles, including 21 species of desert horned lizards, at the Convention on \nL22: International Trade in Endangered Species (CITES) in Panama. \\--- **Field \nL23: Teams** worked tirelessly to protect and restore imperiled species across the \nL24: country. From Florida manatees and red wolves in the Southeast to belugas and \nL25: grizzly bears in Alaska, Defenders’ conservation experts were on the ground \nL26: saving species that need our help to survive and thrive. \\--- **Conservation \nL27: Innovation Experts** published more than 10 peer-reviewed studies on topics that\nL28: include the Cook Inlet beluga whale, golden-cheeked warbler, global parrot \nL29: biodiversity, the Endangered Species Act, the effects of mountaintop removal \nL30: mining on endangered species, the ecological importance of panthers and the \nL31: implementation of “30x30” – the globally recognized goal to which President \nL32: Biden committed the U.S. to conserve 30% of our imperiled lands and waters by \nL33: 2030. \\--- **DEFENDERS.ORG** ===== Page 4 ===== THANK YOU Defenders celebrated \nL34: our 75th anniversary in 2022—an exciting milestone that we attribute to our \nL35: unwavering dedication to our wildlife conservation mission. From helping to pass\nL36: the Endangered Species Act and other bedrock conservation laws to leading the \nL37: advocacy on the reintroduction of wolves to Yellowstone National Park and other \nL38: parts of the West, Defenders is proud to inspire, lead and ultimately celebrate \nL39: some of the most consequential conservation wins for wildlife and wild places. \nL40: It is wonderfully fitting that we also mark 2022 as a year when we successfully \nL41: defended, yet again, the original focal species and symbol of our organization, \nL42: the gray wolf. While the future for gray wolves and many other species remains \nL43: uncertain, we will never stop fighting to protect the imperiled wildlife that \nL44: need us and to restore vibrant and balanced ecosystems. Our mission is more \nL45: important than ever due to the escalating biodiversity and climate crises. \nL46: Defenders is determined to build on our accomplishments and seize new \nL47: opportunities to address these challenges through our multifaceted approach to \nL48: conservation. We focus on developing innovative and science-based solutions, \nL49: coordinating on-the-ground species and habitat restoration, promoting human-\nL50: wildlife coexistence, protecting and strengthening our bedrock environmental \nL51: laws, investing in impactful collaborations and partnerships, building a more \nL52: diverse and powerful conservation movement and challenging attacks on wildlife \nL53: through advocacy, policy and litigation. Defenders’ continued success is made ",
|
||||
"tool_call_id": "fixture-tool-call-17-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Search within PDF for \"National Petroleum Reserve\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-19-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"arguments": "{\"cursor\": 8, \"pattern\": \"Petroleum\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[9] Find results for text: `petroleum` in `www.annualreports.com` (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf/find?pattern=petroleum)\n**viewing lines [0 - 4] of 4**\n\nL0: # 【0†match at L201】\nL1: the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas \nL2: drilling and removed protections for important wildlife habitat. Protections \nL3: have now been restored for nearly half of the reserve’s pristine lands, which \nL4: are vital habitat for shorebirds, denning polar bears and tens of thousands of ",
|
||||
"tool_call_id": "fixture-tool-call-19-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open around L190.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-21-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 8, \"loc\": 180}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[10] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf)\n**viewing lines [180 - 227] of 289**\n\nL180: the sixth successful transfer of bison to the Assiniboine and Sioux Tribes of \nL181: Fort Peck since 2019. \\--- **SWIFT FIX KITS | © ROSIMA PAELARINTSKIMMA MADDIAL \nL182: 200 AND CONSERVATION BIOLOGY INSTITUTE** \\--- **Celebrating our third year** of \nL183: a collaborative program with the Aaniih and Nakoda Tribes and others to restore \nL184: swift foxes to the Fort Belknap Indian Reservation in Montana, Defenders helped \nL185: with the release of 28 more swift foxes. With over 100 foxes reintroduced \nL186: through this program, monitoring efforts show that they are reproducing in the \nL187: wild—a critical measure of success for a self-sustaining population. \\--- \nL188: **Defenders continued to lead the way** for conserving and recovering the \nL189: endangered black-footed ferret, supporting the black-footed ferret survey for \nL190: the Fort Belknap Indian community. Thirty-six ferrets were vaccinated against \nL191: sylvatic plague and two dozen kits were released in the wild. \\--- **10 \nL192: DEFENDERS OF WILDLIFE** ===== Page 11 ===== Defenders helped to bring hope for \nL193: recovery for the endangered military macaw, adding 11 fledglings to a growing \nL194: wild population in Puerta Vallarta, Mexico, that is under pressure from habitat \nL195: loss and poachers for the illegal pet trade. Accord- ing to our recent report, \nL196: the 2008 parrot trade ban that Defenders fought to achieve is working. \nL197: Preventing more than 30,000 parrots from being illegally trapped each year, the \nL198: trade ban has resulted in a 47% decrease in the illegal trade of parrots and an \nL199: 88% decrease in U.S. seizures of Mexican parrots. As a result of a Defenders \nL200: lawsuit, BLM rescinded the previous administration’s plan that opened most of \nL201: the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas \nL202: drilling and removed protections for important wildlife habitat. Protections \nL203: have now been restored for nearly half of the reserve’s pristine lands, which \nL204: are vital habitat for shorebirds, denning polar bears and tens of thousands of \nL205: caribou in the Teshekpuk caribou herd. Our new storymap illustrates why the \nL206: reserve is so important to wildlife. Defenders marked the 20th anniversary of \nL207: our Sea Otter Awareness Week. In celebration of this milestone, we sponsored \nL208: state legislation, signed by Gov. Gavin Newsom, that formally recognizes the \nL209: event’s anniversary and acknowledges the vital role that sea otters play in \nL210: California’s nearshore ecosystems. This annual celebration provides a unique \nL211: opportunity to raise awareness of the important role sea otters play in \nL212: maintaining ecosystem health, our need to coexist with sea otters and the \nL213: efforts of Defenders and others in advancing sea otter conservation. \nL214: DEFENDERS.ORG ===== Page 12 ===== FINANCIAL REPORT In 2022, Defenders of \nL215: Wildlife’s nearly 2.2 million members and supporters contributed more than $41 \nL216: million for wildlife and wildlife habitat. This figure includes all those who \nL217: generously donated their time and expertise. The steadfast support of our donors\nL218: allows Defenders to sustain our program and public education efforts in the \nL219: field, the courts and on Capitol Hill. 2022 SOURCES OF FUNDS Grants and \nL220: contributions $28,946 Bequests, trusts and split interests $13,928 Income from \nL221: investments, annuity reserve funds and trusts (-$7,495) Royalties and other \nL222: income $744 Contributed services $5,047 Total Revenue $41,170 USES OF FUNDS \nL223: Biodiversity conservation $18,970 Constituency mobilization $16,528 Fundraising \nL224: $1,029 Management and general $6,354 Total Expenses $42,881 Change in net assets\nL225: (-$1,711) Net assets, start of year $45,144 Net Assets, End of the Year $43,433\nL226: Dollars are in thousands. 12 DEFENDERS OF WILDLIFE Grants and contributions 59%\nL227: Contributed services 10% Royalties and other income 2% Bequests, trusts and ",
|
||||
"tool_call_id": "fixture-tool-call-21-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Scrolling a bit more for \"Contributed services 10%\". Already seen. So indeed the 2022 annual report shows contributed services 10%, down from 12% in 2021.\n\nThus the organization is Defenders of Wildlife.\n\nThe question: \"Who was the emeritus board member that year? Name only, no titles.\"\n\nWe need to identify the emeritus board member listed in the 2022 annual report. From the 2021 annual report, we saw \"BOARD EMERITUS Terry C. Pelster, Esq.\" However, that is for 2021. The question asks \"Who was the emeritus board member that year?\" That year refers to the following year (2022). So we need to check the 2022 annual report for the emeritus board member.\n\nOpen the 2022 PDF near the board list. Search for \"Emeritus\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-23-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"arguments": "{\"cursor\": 8, \"pattern\": \"Emeritus\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[11] Find results for text: `emeritus` in `www.annualreports.com` (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf/find?pattern=emeritus)\n**viewing lines [0 - 10] of 10**\n\nL0: # 【0†match at L237】\nL1: Sisk, Ph.D British Columbia, Canada BOARD EMERITUS Terry C. Pelster, Esq. New \nL2: York *Executive Committee member* STAFF & ADVISORS SCIENCE ADVISORS Gabriela \nL3: Chavarria, Ph.D Burke Museum, University of Washington Leah Gerber, Ph.D Arizona\nL4: State University Kathayoon Khalil, Ph.D Oregon Zoo Rurik List, Ph.D Universidad\nL5: \nL6: # 【1†match at L243】\nL7: Davis Thomas D. Sisk, Ph.D Emeritus Professor Northern Arizona University David \nL8: S. Wilcox, Ph.D Princeton University Rae Wynn-Grant, Ph.D National Geographic \nL9: Society SENIOR STAFF Jamie Rappaport Clark President & CEO Thu Pham Chief of \nL10: Staff James Stofan Senior Vice President, Operations McCrystle Adams Vice ",
|
||||
"tool_call_id": "fixture-tool-call-23-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open around L230.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-25-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 8, \"loc\": 220}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[12] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf)\n**viewing lines [220 - 267] of 289**\n\nL220: contributions $28,946 Bequests, trusts and split interests $13,928 Income from \nL221: investments, annuity reserve funds and trusts (-$7,495) Royalties and other \nL222: income $744 Contributed services $5,047 Total Revenue $41,170 USES OF FUNDS \nL223: Biodiversity conservation $18,970 Constituency mobilization $16,528 Fundraising \nL224: $1,029 Management and general $6,354 Total Expenses $42,881 Change in net assets\nL225: (-$1,711) Net assets, start of year $45,144 Net Assets, End of the Year $43,433\nL226: Dollars are in thousands. 12 DEFENDERS OF WILDLIFE Grants and contributions 59%\nL227: Contributed services 10% Royalties and other income 2% Bequests, trusts and \nL228: split interests 29% TOTAL REVENUE Income from investments -15% Program and \nL229: support services 98% USE OF FUNDS Fundraising 2% ===== Page 13 ===== \nL230: DEFENDERS.ORG 13 ===== Page 14 ===== BOARD OF DIRECTORS John Dayton, Esq.* Chair\nL231: Texas Sheryl Sachs* Vice Chair Connecticut Dinah Bear, Esq.* Secretary Arizona \nL232: Mark Caylor* Treasurer District of Columbia Susan Alexander Connecticut Sylvia \nL233: Baca* California Whitefield Bailey Tennessee Donald Barry, Esq. Oregon Kim \nL234: O’Keefe Beck, Ph.D California Gabriela Chavarria, Ph.D Washington Jamie \nL235: Rappaport Clark* President & CEO Virginia John Donovan, Esq.* Montana and \nL236: California Caroline D. Gabel Maryland Terry L. Root, Ph.D New Mexico Thomas D. \nL237: Sisk, Ph.D British Columbia, Canada BOARD EMERITUS Terry C. Pelster, Esq. New \nL238: York *Executive Committee member* STAFF & ADVISORS SCIENCE ADVISORS Gabriela \nL239: Chavarria, Ph.D Burke Museum, University of Washington Leah Gerber, Ph.D Arizona\nL240: State University Kathayoon Khalil, Ph.D Oregon Zoo Rurik List, Ph.D Universidad\nL241: Autónoma Metropolitana-Lerma Barry Noon, Ph.D Colorado State University Terry \nL242: L. Root, Ph.D Stanford University Mark Schwartz, Ph.D University of California-\nL243: Davis Thomas D. Sisk, Ph.D Emeritus Professor Northern Arizona University David \nL244: S. Wilcox, Ph.D Princeton University Rae Wynn-Grant, Ph.D National Geographic \nL245: Society SENIOR STAFF Jamie Rappaport Clark President & CEO Thu Pham Chief of \nL246: Staff James Stofan Senior Vice President, Operations McCrystle Adams Vice \nL247: President, Conservation Law Matt Braughler Vice President, Philanthropy Rachel \nL248: Brittin Vice President, Communications Shawn Cantrell Vice President, Field \nL249: Conservation Programs Robert L. Dewey Vice President, Government Relations & \nL250: External Affairs Monica Goldberg Vice President, Landscape Conservation Sharon \nL251: Morris-Smith Vice President of Human Resources Elizabeth Nichols Vice President,\nL252: Finance/CFO Anthony Pegues Vice President, Diversity, Equity, Inclusion, \nL253: Justice Lindsay Rosa, Ph.D Vice President, Conservation Research and Innovation \nL254: Brandace Swimley Vice President, Integrated Marketing 14 DEFENDERS OF WILDLIFE \nL255: ===== Page 15 ===== There are many ways to support Defenders of Wildlife and \nL256: protect our imperiled wildlife. To learn more or donate, contact us at \nL257: 1-800-385-9712, visit Defenders.org/donate or choose one of the following \nL258: options. ➤ Transfer appreciated assets directly to Defenders and maximize \nL259: potential tax benefits. Defenders.org/ donate-stocks-help-save-wildlife ➤ Become\nL260: a Leadership Council member with a single donation of $1,000 or more and enjoy \nL261: more personal involvement with Defenders. Defenders.org/leadershipcouncil ➤ If \nL262: you are 70.5 or older, roll over a portion of your IRA to benefit wildlife. \nL263: Remember, these gifts must come directly from your IRA account. \nL264: Defenders.org/planned-giving/gifts-your-retirement-plan ➤ Get started on a \nL265: contribution from your donor-advised fund Defenders.org/donor-advised-funds ➤ \nL266: Check with your personnel office or that of your spouse/partner about your \nL267: company’s Corporate Matching Gifts program or visit us at ",
|
||||
"tool_call_id": "fixture-tool-call-25-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Thus the emeritus board member is Terry C. Pelster. However, the question might ask for \"Who was the emeritus board member that year?\" Possibly referring to the year of the annual report that mentioned the rescinded plan and contributed services decrease. That is 2022. So the answer is Terry C. Pelster.\n\nBut let's verify that the 2022 annual report indeed includes the mention of contributed services decreasing to 10%. We saw that. So answer is Terry C. Pelster.\n\nBut maybe the question is tricky: \"Who was the emeritus board member that year?\" Could be the emeritus board member listed in the 2021 annual report? However, the question says \"That same year, a company's annual report revealed that its nearly 2.2 million members and supporters contributed over $49 million, with 12% of total revenue coming from contributed services. The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan to open most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling and that contributed services had decreased by 2% to 10%.\"\n\nThus the \"following year\" annual report is 2022. The question likely asks: \"Who was the emeritus board member that year?\" Meaning the emeritus board member listed in that annual report (2022). So answer: Terry C. Pelster.\n\nBut let's confirm that the 2021 annual report also shows emeritus board member Terry C. Pelster. Indeed we saw that. But perhaps the 2022 report might have a different emeritus board member. We saw it's the same. So answer is Terry C. Pelster.\n\nHowever, the question might be about a different organization. Let's verify the details.\n\nFirst clue: Press release about Kenya vaccine doses on Dec 5, 2021. That is likely a World Bank press release. Search for \"World Bank Kenya vaccine doses December 2021 press release\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-27-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"As of December 5, 2021, Kenya had received a total of 16,201,670 vaccines\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[13] As of December 5, 2021, Kenya had received a total of 16,201,670 vaccines (Search_Results/As of December 5, 2021, Kenya had received a total of 16,201,670 vaccines)\n**viewing lines [0 - 41] of 41**\n\nL0: \nL1: URL: Search_Results/As of December 5, 2021, Kenya had received a total of \nL2: 16,201,670 vaccines\nL3: # Search Results\nL4: \nL5: * 【0†Kenya's Economy is Showing Resilience as Output Rises ...; publish_date: \nL6: none†www.worldbank.org】 Dec 14, 2021 — As of December 5, 2021, Kenya had \nL7: received a total of 16,201,670 vaccines, with 7,583,134 administered. While \nL8: vaccine acceptance is ...\nL9: * 【1†Unmet need for COVID-19 vaccination coverage in Kenya - PMC; \nL10: publish_date: none†pmc.ncbi.nlm.nih.gov】 by SK Muchiri · 2022 · Cited by 42 — As\nL11: of December 2021, six counties had a vaccination coverage of less than 5%. \nL12: These counties include Garissa, Mandera, Marsabit, Tana River, Turkana, and ...\nL13: * 【2†MINISTRY OF HEALTH; publish_date: none†covidhub.mediacouncil.or.ke】 Dec \nL14: 1, 2021 — • Total Covid-19 Vaccines Received to date- 16,201,670 ... Table 10: \nL15: Vaccine Logistics Received in the Country as at 5th, December 2021.\nL16: * 【3†COVID-19 vaccination refusal trends in Kenya over 2021 - PMC; \nL17: publish_date: none†pmc.ncbi.nlm.nih.gov】 by RT Rego · 2023 · Cited by 21 — We \nL18: assessed vaccine refusal over time in Kenya, and characterized factors \nL19: associated with changes in vaccination refusal.\nL20: * 【4†Ciheb-Kenya on the Front Lines of the COVID-19 Vaccine ...; publish_date:\nL21: none†www.ciheb.org】 Of the 721,509 persons vaccinated, 409,711 are aged 58 \nL22: years and above, 143,050 are healthcare workers, 109,593 are teachers, and \nL23: 59,155 are security officers.\nL24: * 【5†proportion of fully vaccinated adults 23.3pc.; publish_date: \nL25: none†www.facebook.com】 COVID-19: KENYA administers 150632 new vaccinations; \nL26: total jabs given now 13782963; proportion of fully vaccinated adults 23.3pc.\nL27: * 【6†National COVID-19 Vaccines Deployment and ...; publish_date: \nL28: none†www.parliament.go.ke】 Kenya has a well-established immunization program and\nL29: a robust disease surveillance and response unit (DSRU) which serves as an early\nL30: warning system, to.\nL31: * 【7†Kenya: Vaccine Requirements Violate Rights; publish_date: \nL32: none†www.hrw.org】 Dec 13, 2021 — Kenya, with an estimated adult population of \nL33: 27.2 million and a total population of 55 million, has received approximately 23\nL34: million doses as ...\nL35: * 【8†Government gives 20 million a month to get vaccinated; publish_date: \nL36: none†www.bbc.com】 Nov 22, 2021 — Although less than 10% of Kenyans are \nL37: vaccinated, the government wants to avoid a surge over Christmas.\nL38: * 【9†ministry of health kenya covid-19 immunization status ...; publish_date: \nL39: none†www.health.go.ke】 May 15, 2022 — By the end of 2021 Kenya had administered \nL40: 10.1 million doses of 5 types of Covid-19 vaccines. During this period 4.2 \nL41: million people were ...",
|
||||
"tool_call_id": "fixture-tool-call-27-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open result 0.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-29-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 13, \"id\": 0}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[14] www.worldbank.org (https://www.worldbank.org/en/news/press-release/2021/12/14/kenya-s-economy-is-showing-resilience-as-output-rises-above-pre-pandemic-levels-driven-by-a-rebound-in-the-services-sect)\n**viewing lines [0 - 52] of 61**\n\nL0: \nL1: URL: https://www.worldbank.org/en/news/press-release/2021/12/14/kenya-s-economy-\nL2: is-showing-resilience-as-output-rises-above-pre-pandemic-levels-driven-by-a-\nL3: rebound-in-the-services-sect\nL4: NAIROBI, December 14, 2021 — Kenya’s economy has demonstrated resilience to the \nL5: COVID-19 shock, with output in the first half of the year rising above pre-\nL6: pandemic levels. In 2021 as a whole, gross domestic product (GDP) is expected to\nL7: grow by 5%, one of the faster recoveries among Sub-Saharan African countries. \nL8: Overall economic performance is expected to be robust at 4.9% per year in \nL9: 2022-23, similar to the pre-pandemic pace (5% average annual growth from 2010 to\nL10: 2019). According to the 24th edition of the Kenya Economic Update, “From \nL11: Recovery to Better Jobs,” growth has been supported by rebounds in industry and,\nL12: especially, services. Agricultural output, however, fell by 0.5% year on year \nL13: in the first half of 2021 following a particularly strong performance in 2020, \nL14: partly due to below-average rains. Demand-side recovery has been supported by a \nL15: revival in private consumption, against a backdrop of improving employment \nL16: conditions and household incomes. “Kenya’s economy has shown considerable \nL17: resilience to the enormous shock of the pandemic, and this year is expected to \nL18: post one of the stronger growth rebounds in the region thanks to diversified \nL19: sources of growth and sound economic policies and management,” said Keith \nL20: Hansen, World Bank Country Director for Kenya. “However, poverty has increased, \nL21: and the buffers and coping mechanisms of households, firms, and the public \nL22: finances have been depleted.” Economic activity in Kenya has continued to adapt \nL23: to the pandemic and associated restrictions. A mix of containment measures, such\nL24: as a nightly curfew, were in effect through most of 2021, while more \nL25: economically disruptive measures such as lockdowns and travel restrictions were \nL26: phased, limiting the impact on economic activities. The vaccine rollout, which \nL27: had a slow start due to supply constraints, has picked up as new shipments of \nL28: vaccines have arrived, particularly since September. This has supported economic\nL29: recovery and growth through the third quarter of 2021. As of December 5, 2021, \nL30: Kenya had received a total of 16,201,670 vaccines, with 7,583,134 administered. \nL31: While vaccine acceptance is reportedly high there is still a long way to go \nL32: towards the government’s target of fully inoculating the adult population of \nL33: about 30 million by the end of 2022.As of December 6, 2021, about 10% of adults \nL34: (2.9 million people) had been fully vaccinated while another 16% (4.9 million \nL35: people) had received their first dose. The report also projects robust growth in\nL36: the medium-term. This outlook takes into account that some sub-sectors, such as\nL37: education, have bounced back strongly, but others such as international \nL38: tourism, have only partially and face a much more protracted recovery. The \nL39: anticipated further recovery of hotels and restaurants, trade, transport, and \nL40: other services, depends on substantial vaccination progress to help prevent new \nL41: waves of infections and associated containment measures. “Recent economic \nL42: performance has been strong and the outlook is positive, but in Kenya, as \nL43: everywhere, the future course of the pandemic remains a major source of \nL44: uncertainty, said Alex Sienaert, Senior Economist for Kenya. “To support further\nL45: recovery, reduce debt distress risks, and rebuild space for social and \nL46: development spending, it is critical that Kenya continues to implement medium-\nL47: term fiscal consolidation plans.” In addition to pandemic-related risks to the \nL48: outlook, a second key domestic risk factor stems from the drought conditions \nL49: which are affecting parts of the country and already causing severe hardship. \nL50: Should the drought intensify or spread, this would weigh on the near-term \nL51: economic outlook. Weaker global growth, higher-than-anticipated energy prices, \nL52: and tighter external financing conditions are the primary external risks. As ",
|
||||
"tool_call_id": "fixture-tool-call-29-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Thus first clue confirmed.\n\nSecond clue: Defenders of Wildlife 2021 annual report. Confirmed.\n\nThird clue: Defenders of Wildlife 2022 annual report. Confirmed.\n\nThus the answer is Terry C. Pelster.\n\nBut let's verify that the 2022 annual report indeed shows \"Contributed services 10%\". We saw that. Also confirm that 2021 report shows 12%. We saw that.\n\nThus the question: \"Who was the emeritus board member that year?\" Refers to 2022. Answer: Terry C. Pelster.\n\nHowever, the question might be asking for the emeritus board member in the 2021 report? Let's read the question carefully:\n\n\"In a year before 2023, between December 1 and 15 (inclusive), a press release reported that as of the 5th of that month, a country had received a total of 16,201,670 vaccine doses, with 7,583,134 administered. That same year, a company's annual report revealed that its nearly 2.2 million members and supporters contributed over $49 million, with 12% of total revenue coming from contributed services. The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan to open most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling and that contributed services had decreased by 2% to 10%. Who was the emeritus board member that year? Name only, no titles.\"\n\nThe phrase \"that year\" could refer to \"the following year\" i.e., the year of the annual report that mentioned the rescinded plan. Or could refer to \"that same year\" (the year of the press release and the first annual report). However, the phrase \"that year\" appears at the end: \"Who was the emeritus board member that year?\" It likely refers to the year of the annual report that is being discussed in the preceding sentence. The preceding sentence talks about the following year's annual report. So indeed \"that year\" is the following year (2022).\n\nThus answer: Terry C. Pelster.\n\nBut to be thorough, we should check the 2022 annual report for any other emeritus board members. The search only showed \"BOARD EMERITUS Terry C. Pelster, Esq.\" Maybe there are multiple emeritus board members. But likely only one.\n\nCheck the 2021 annual report for emeritus board member. It also shows Terry C. Pelster. So maybe the question is asking for the emeritus board member in the 2021 report. But why would they ask that? The question is about connecting the clues to identify the organization and then find the emeritus board member. The clues point to Defenders of Wildlife. The question likely asks for the emeritus board member as of the 2022 report.\n\nNevertheless, we should confirm that the 2022 report is indeed the one that mentions the rescinded plan. We saw that. Also confirm that contributed services decreased from 12% to 10%. We saw that.\n\nThus answer is Terry C. Pelster.\n\nBut let's search for \"Defenders of Wildlife emeritus board member 2022 Terry Pelster\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-31-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"arguments": "{\"query\": \"Terry C. Pelster Defenders of Wildlife emeritus board member\", \"topn\": 10, \"source\": \"web\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[15] Terry C. Pelster Defenders of Wildlife emeritus board member (Search_Results/Terry C. Pelster Defenders of Wildlife emeritus board member)\n**viewing lines [0 - 31] of 31**\n\nL0: \nL1: URL: Search_Results/Terry C. Pelster Defenders of Wildlife emeritus board member\nL2: # Search Results\nL3: \nL4: * 【0†2024 impact report; publish_date: none†defenders.org】 Terry Root, Ph.D. \nL5: New Mexico. Thomas D. Sisk, Ph.D. British Columbia, Canada. BOARD EMERITUS. \nL6: Terry C. Pelster, Esq. New York. SCIENCE ADVISORS. Leah Gerber, Ph ...\nL7: * 【1†Defenders of Wildlife; publish_date: none†www.wrongkindofgreen.org】 \nL8: Directors ; Victor M. Sher*, Chair – California ; Terry C. Pelster*, Vice Chair \nL9: – New York ; Richard Kopcho*, Treasurer – California ; Adelaide P. Gomer*, \nL10: Secretary ...\nL11: * 【2†Exhibit 12; publish_date: none†www.uschamber.com】 DECLARATION OF TERRY \nL12: PELSTER. I, Terry C. Pelster, declare as follows: 1. I am a current member of \nL13: Defenders of Wildlife (“Defenders”) and have been a member.\nL14: * 【3†2020 ANNUAL REPORT; publish_date: none†defenders.org】 Terry L. Root. \nL15: Florida. BOARD. EMERITUS. Terry C. Pelster. New York. Alan Steinberg. Florida. \nL16: SCIENCE. ADVISORS. Gabriela Chavarria,. Ph.D. Denver Museum of.\nL17: * 【4†2019 Annual Report; publish_date: none†defenders.org】 BOARD EMERITUS. \nL18: Terry C. Pelster. New York. Alan Steinberg. Florida. SCIENCE ADVISORS. Leah \nL19: Gerber, Ph.D. Arizona State University. Kathayoon Khalil, Ph.D.\nL20: * 【5†annual report; publish_date: none†defenders.org】 * Executive Committee \nL21: member. BoARD EMERITuS. Terry C. Pelster. New York. Alan Steinberg. Florida. \nL22: PRESIDENT. EMERITuS. Rodger Schlickeisen. Virginia. SCIENCE ...\nL23: * 【6†BEING THERE FOR BELUGAS; publish_date: none†wendeenicole.com】 Mar 2, 2023\nL24: — Defenders of Wildlife is a national, nonprofit member- ship ... Terry C. \nL25: Pelster, Esq. New York. *Executive Committee member outfront.\nL26: * 【7†Annual Reports; publish_date: none†www.annualreports.com】 WAYS TO GIVE \nL27: There are many ways to support Defenders of Wildlife and protect our imperiled \nL28: wildlife. ... Terry C. Pelster. New York. Alan Steinberg. Florida.\nL29: * 【8†2014 Annual Report; publish_date: none†defenders.org】 BOARD OF DIRECTORS \nL30: defenders of wildlife is a national, nonprofit ... BOARD EMERITUS. Terry C. \nL31: Pelster. New York. Alan Steinberg. Florida. Page 3. PO.",
|
||||
"tool_call_id": "fixture-tool-call-31-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Thus Terry C. Pelster is indeed the emeritus board member.\n\nNow the question might want just \"Terry C. Pelster\". Or maybe \"Terry Pelster\". Or \"Terry C. Pelster, Esq.\" but instructions say \"Name only, no titles.\" So likely \"Terry C. Pelster\". However, maybe they want \"Terry Pelster\". Better to include full name as listed: \"Terry C. Pelster\".\n\nLet's read the question again: \"Who was the emeritus board member that year? Name only, no titles.\" So we should answer just the name, no \"Esq.\" or \"Board Emeritus\". So \"Terry C. Pelster\". Or maybe \"Terry Pelster\". But the document shows \"Terry C. Pelster, Esq.\" So the name is \"Terry C. Pelster\". \n\nBetter to verify that the 2022 annual report indeed lists \"BOARD EMERITUS Terry C. Pelster, Esq.\" We saw that line.\n\nThus answer: Terry C. Pelster.\n\nBut let's double-check that the question might refer to the 2021 annual report. The phrase \"that year\" could be ambiguous. Let's parse the timeline:\n\n- Year 1: 2021. Press release in Dec 2021 about Kenya vaccine doses. Same year (2021) annual report of Defenders of Wildlife shows 12% contributed services.\n\n- Year 2: 2022. Annual report mentions rescinded plan and contributed services decreased to 10%.\n\nThe question says \"The following year, the annual report stated that the organization had played a role in rescinding a previous administration’s plan ... and that contributed services had decreased by 2% to 10%.\" This is year 2.\n\nThen asks \"Who was the emeritus board member that year?\" That likely refers to year 2.\n\nThus answer: Terry C. Pelster.\n\nHowever, perhaps the emeritus board member changed between 2021 and 2022. We need to check 2022 annual report for any other emeritus board member. We saw only Terry C. Pelster. Also check 2021 annual report: same. So likely unchanged.\n\nThus answer is Terry C. Pelster.\n\nNevertheless, we should confirm that the 2022 annual report is indeed the one that includes the phrase \"BLM rescinded the previous administration’s plan that opened most of the 22 million acres of the National Petroleum Reserve in Alaska to oil and gas drilling\". We saw that in line 200-202.\n\nThus answer is Terry C. Pelster.\n\nNow we need to provide the answer in the final message. Only the name.\n\nThus final answer: Terry C. Pelster.\n\nBut let's be thorough and search for \"Defenders of Wildlife 2022 annual report contributed services 10%\".",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-33-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"arguments": "{\"cursor\": 8, \"pattern\": \"Contributed services\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[16] Find results for text: `contributed services` in `www.annualreports.com` (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2022.pdf/find?pattern=contributed services)\n**viewing lines [0 - 10] of 10**\n\nL0: # 【0†match at L222】\nL1: income $744 Contributed services $5,047 Total Revenue $41,170 USES OF FUNDS \nL2: Biodiversity conservation $18,970 Constituency mobilization $16,528 Fundraising \nL3: $1,029 Management and general $6,354 Total Expenses $42,881 Change in net assets\nL4: (-$1,711) Net assets, start of year $45,144 Net Assets, End of the Year $43,433\nL5: \nL6: # 【1†match at L227】\nL7: Contributed services 10% Royalties and other income 2% Bequests, trusts and \nL8: split interests 29% TOTAL REVENUE Income from investments -15% Program and \nL9: support services 98% USE OF FUNDS Fundraising 2% ===== Page 13 ===== \nL10: DEFENDERS.ORG 13 ===== Page 14 ===== BOARD OF DIRECTORS John Dayton, Esq.* Chair",
|
||||
"tool_call_id": "fixture-tool-call-33-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Thus confirmed.\n\nNow we should verify that the 2021 annual report shows contributed services 12%.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-35-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find",
|
||||
"arguments": "{\"cursor\": 4, \"pattern\": \"Contributed services\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[17] Find results for text: `contributed services` in `www.annualreports.com` (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2021.pdf/find?pattern=contributed services)\n**viewing lines [0 - 4] of 4**\n\nL0: # 【0†match at L194】\nL1: $3,354 Royalties and other income $3,576 Contributed services $6,140 Total \nL2: Revenue $49,819 USES OF FUNDS Biodiversity conservation $22,420 Constituency \nL3: mobilization $16,324 Fundraising $1,211 Management and general $5,865 Total \nL4: Expenses $45,820 Change in net assets $3,999 Net assets, start of year $41,145 ",
|
||||
"tool_call_id": "fixture-tool-call-35-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "Open around L190-200 to see percentage.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fixture-tool-call-37-0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "open",
|
||||
"arguments": "{\"cursor\": 4, \"loc\": 185}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[18] www.annualreports.com (https://www.annualreports.com/HostedData/AnnualReportArchive/d/defenders-wildlife_2021.pdf)\n**viewing lines [185 - 232] of 261**\n\nL185: devastating water pollution that is killing the seagrass and causing manatees to\nL186: starve. DAVID TES | SAM FRENZY DRAW DEFENDERS.ORG 11 ===== Page 12 ===== In \nL187: 2021, Defenders of Wildlife’s nearly 2.2 million members and supporters \nL188: contributed more than $49 million for wildlife and wildlife habitat. This figure\nL189: includes all those who generously donated their time and expertise. The \nL190: steadfast support of our donors allows Defenders to sustain our program and \nL191: public education efforts in the field, the courts and on Capitol Hill. 2021 \nL192: SOURCES OF FUNDS Grants and contributions $29,057 Bequests, trusts and split \nL193: interests $7,692 Income from investments, annuity reserve funds and trusts \nL194: $3,354 Royalties and other income $3,576 Contributed services $6,140 Total \nL195: Revenue $49,819 USES OF FUNDS Biodiversity conservation $22,420 Constituency \nL196: mobilization $16,324 Fundraising $1,211 Management and general $5,865 Total \nL197: Expenses $45,820 Change in net assets $3,999 Net assets, start of year $41,145 \nL198: Net Assets, End of the Year $45,144 Dollars are in thousands. 12 DEFENDERS OF \nL199: WILDLIFE Grants and contributions 58% Income from investments 7% Requests, \nL200: trusts and split interests 15% Royalties and other income 7% Contributed \nL201: services 12% Program and support services 97% Fundraising 3% ===== Page 13 =====\nL202: DEFENDERS.ORG 13 ===== Page 14 ===== John Dayton, Esq.* Chair Texas Sheryl \nL203: Sachs* Vice Chair Connecticut Whitfield Bailey* Secretary Tennessee Mark Caylor*\nL204: Treasurer District of Columbia Sylvia Baca* California Donald Barry, Esq. \nL205: Oregon Dinah Bear, Esq. Arizona Kim O’Keefe Beck, Ph.D California Gabriela \nL206: Chavarria, Ph.D Washington Jamie Rappaport Clark* President & CEO Virginia \nL207: Caroline D. Gabel Maryland Keith M. Harper, Esq. Maryland John Donovan, Esq. \nL208: Montana and California James M. Hecker, Esq. Virginia Judith Ponsikoff, Ph.D* \nL209: California Terry L. Root, Ph.D Florida BOARD EMERITUS Terry C. Pelster, Esq. New\nL210: York *Executive Committee member* ST A F F & A D V I S O R S SCIENCE ADVISORS \nL211: Gabriela Chavarria, Ph.D Burke Museum, University of Washington Leah Gerber, \nL212: Ph.D Arizona State University Kathayoon Khalil, Ph.D Oregon Zoo Rurik List, Ph.D\nL213: Universidad Autónoma Metropolitana-Lerma Barry Noon, Ph.D Colorado State \nL214: University Terry L. Root, Ph.D Stanford University Mark Schwartz, Ph.D \nL215: University of California-Davis David S. Wilcox, Ph.D Princeton University Rae \nL216: Wynn-Grant, Ph.D National Geographic Society SENIOR STAFF Jamie Rappaport Clark \nL217: President & CEO Thu Pham Chief of Staff Renee Stone Senior Vice President, \nL218: Conservation Programs & General Counsel James Stefan Senior Vice President, \nL219: Operations Matt Braughler Vice President, Philanthropy Rachel Brittin Vice \nL220: President, Communications Shawn Cantrell Vice President, Field Conservation \nL221: Programs Robert L. Dewey Vice President, Government Relations & External Affairs\nL222: Monica Goldberg Vice President, Landscape Conservation Sharon Morris-Smith Vice\nL223: President of Human Resources Elizabeth Nichols Vice President, Finance/CFO \nL224: Anthony Pegues Vice President, Diversity, Equity, Inclusion, Justice Lindsay \nL225: Rosa, Ph.D Director, Center for Conservation Innovation Michael P. Senatore Vice\nL226: President, Conservation Law Brandace Swimley Vice President, Integrated \nL227: Marketing 14 DEFENDERS OF WILDLIFE ===== Page 15 ===== THERE ARE MANY WAYS TO \nL228: SUPPORT DEFENDERS OF WILDLIFE and protect our imperiled wildlife. To learn more \nL229: or donate, contact us at 1-800-385-9712, visit defenders.org/donate or choose \nL230: one of the following options. ➤ Transfer appreciated assets directly to \nL231: Defenders and maximize potential tax benefits. Defenders.org/donate-stocks-help-\nL232: save-wildlife ➤ Become a Leadership Council member with a single donation of ",
|
||||
"tool_call_id": "fixture-tool-call-37-0"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Terry C. Pelster",
|
||||
"reasoning_content": "Thus everything matches.\n\nThus answer: Terry C. Pelster.\n\nNow we need to output only the name.\n\nThus final answer: Terry C. Pelster.\n\nBut perhaps the question expects \"Terry Pelster\". However, the document includes middle initial \"C.\" It might be better to include the full name as listed: \"Terry C. Pelster\". The instruction \"Name only, no titles\" suggests we should not include \"Esq.\" or \"Board Emeritus\". But including middle initial is likely okay.\n\nTo be safe, we could output \"Terry C. Pelster\". Let's see similar puzzles: often they expect the full name as appears in the document.\n\nThus answer: Terry C. Pelster.\n\nNow we will provide final answer.",
|
||||
"tool_calls": []
|
||||
}
|
||||
]
|
||||
}
|
||||
+2455
File diff suppressed because it is too large
Load Diff
+1069
File diff suppressed because it is too large
Load Diff
+112
@@ -0,0 +1,112 @@
|
||||
<|begin▁of▁sentence|>
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools you can use to answer the user's question.
|
||||
You can invoke functions by writing a "<|DSML|function_calls>" block like the following as part of your reply to the user:
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="$FUNCTION_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$FUNCTION_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls>
|
||||
|
||||
String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects).
|
||||
|
||||
If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example:
|
||||
|
||||
<|DSML|function_calls>
|
||||
...
|
||||
</|DSML|function_calls>
|
||||
|
||||
<function_results>
|
||||
...
|
||||
</function_results>
|
||||
|
||||
<think>...thinking about results</think>
|
||||
|
||||
Here are the functions available in JSONSchema format:
|
||||
<functions>
|
||||
{"name": "get_datetime", "description": "Get the current date and time", "parameters": {"type": "object", "properties": {"timezone": {"type": "string", "description": "The timezone, e.g. Asia/Shanghai, UTC"}}, "required": ["timezone"]}}
|
||||
{"name": "get_weather", "description": "Get the weather for a specific date and location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name, e.g. Beijing, Hangzhou"}, "date": {"type": "string", "description": "The date in YYYY-MM-DD format"}}, "required": ["location", "date"]}}
|
||||
</functions>
|
||||
You are a helpful Assistant.<|User|>明天杭州和北京的天气怎么样?<|Assistant|></think>
|
||||
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="get_datetime">
|
||||
<|DSML|parameter name="timezone" string="true">Asia/Shanghai</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls><|end▁of▁sentence|>
|
||||
|
||||
<function_results>
|
||||
<result>{"current_date": "2024-01-15", "current_time": "14:30:00", "timezone": "Asia/Shanghai"}</result>
|
||||
</function_results>
|
||||
|
||||
</think>
|
||||
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Hangzhou</|DSML|parameter>
|
||||
<|DSML|parameter name="date" string="true">2024-01-16</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
|
||||
<|DSML|parameter name="date" string="true">2024-01-16</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls><|end▁of▁sentence|>
|
||||
|
||||
<function_results>
|
||||
<result>{"location": "Hangzhou", "date": "2024-01-16", "temperature_high": "12", "temperature_low": "5", "weather": "多云", "humidity": "65%"}</result>
|
||||
<result>{"location": "Beijing", "date": "2024-01-16", "temperature_high": "-2", "temperature_low": "-8", "weather": "晴", "humidity": "30%"}</result>
|
||||
</function_results>
|
||||
|
||||
</think>根据查询结果,明天(2024年1月16日)的天气情况如下:
|
||||
|
||||
**杭州**:
|
||||
- 天气:多云
|
||||
- 最高温度:12°C
|
||||
- 最低温度:5°C
|
||||
- 湿度:65%
|
||||
|
||||
**北京**:
|
||||
- 天气:晴
|
||||
- 最高温度:-2°C
|
||||
- 最低温度:-8°C
|
||||
- 湿度:30%
|
||||
|
||||
杭州明天会比较温暖但有些多云,而北京会很冷但是晴天。建议在北京的朋友要注意保暖!<|end▁of▁sentence|><|User|>谢谢!那后天呢?<|Assistant|><think>用户现在问后天的天气,后天是2024-01-17,我可以直接查询(因为已知今天日期)🗓️</think>
|
||||
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Hangzhou</|DSML|parameter>
|
||||
<|DSML|parameter name="date" string="true">2024-01-17</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
|
||||
<|DSML|parameter name="date" string="true">2024-01-17</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls><|end▁of▁sentence|>
|
||||
|
||||
<function_results>
|
||||
<result>{"location": "Hangzhou", "date": "2024-01-17", "temperature_high": "15", "temperature_low": "8", "weather": "小雨", "humidity": "80%"}</result>
|
||||
<result>{"location": "Beijing", "date": "2024-01-17", "temperature_high": "0", "temperature_low": "-6", "weather": "多云", "humidity": "45%"}</result>
|
||||
</function_results>
|
||||
|
||||
<think>获取到后天的天气数据,整理回复给用户📝</think>后天(2024年1月17日)的天气情况:
|
||||
|
||||
**杭州**:
|
||||
- 天气:小雨
|
||||
- 最高温度:15°C
|
||||
- 最低温度:8°C
|
||||
- 湿度:80%
|
||||
|
||||
**北京**:
|
||||
- 天气:多云
|
||||
- 最高温度:0°C
|
||||
- 最低温度:-6°C
|
||||
- 湿度:45%
|
||||
|
||||
杭州后天会有小雨,温度略有回升,记得带伞。北京会稍微暖和一点,但依然很冷,请继续做好保暖措施。<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,31 @@
|
||||
mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Dedicated DeepSeek V3.2 renderer.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct DeepSeekV32ChatRenderer;
|
||||
|
||||
impl DeepSeekV32ChatRenderer {
|
||||
/// Create the dedicated DeepSeek V3.2 renderer.
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for DeepSeekV32ChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,422 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use expect_test::{ExpectFile, expect, expect_file};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::DeepSeekV32ChatRenderer;
|
||||
use crate::error::Error;
|
||||
use crate::event::{AssistantContentBlock, AssistantToolCall};
|
||||
use crate::request::{
|
||||
ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
};
|
||||
use crate::{ChatRenderer, ChatRole};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureRequest {
|
||||
#[serde(default)]
|
||||
tools: Vec<FixtureTool>,
|
||||
messages: Vec<FixtureMessage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureTool {
|
||||
function: FixtureToolFunction,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolFunction {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
parameters: Value,
|
||||
#[serde(default)]
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "role", rename_all = "snake_case")]
|
||||
enum FixtureMessage {
|
||||
System {
|
||||
content: String,
|
||||
},
|
||||
Developer {
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
tools: Vec<FixtureTool>,
|
||||
},
|
||||
User {
|
||||
content: String,
|
||||
},
|
||||
Assistant {
|
||||
#[serde(default)]
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
reasoning_content: String,
|
||||
#[serde(default)]
|
||||
tool_calls: Vec<FixtureToolCall>,
|
||||
},
|
||||
Tool {
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
tool_call_id: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolCall {
|
||||
#[serde(default)]
|
||||
id: Option<String>,
|
||||
function: FixtureToolCallFunction,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolCallFunction {
|
||||
name: String,
|
||||
arguments: String,
|
||||
}
|
||||
|
||||
fn render_request(request: &ChatRequest) -> String {
|
||||
DeepSeekV32ChatRenderer::new()
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_text()
|
||||
.expect("deepseek renderer should return text prompt")
|
||||
}
|
||||
|
||||
fn render_result(request: &ChatRequest) -> Result<String, Error> {
|
||||
DeepSeekV32ChatRenderer::new().render(request).map(|rendered| {
|
||||
rendered
|
||||
.prompt
|
||||
.into_text()
|
||||
.expect("deepseek renderer should return text prompt")
|
||||
})
|
||||
}
|
||||
|
||||
fn thinking_request(messages: Vec<ChatMessage>) -> ChatRequest {
|
||||
let mut request = ChatRequest {
|
||||
request_id: "deepseek-v32-small-test".to_string(),
|
||||
messages,
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
if matches!(
|
||||
request.messages.last().map(ChatMessage::role),
|
||||
Some(ChatRole::Assistant)
|
||||
) {
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
}
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request
|
||||
}
|
||||
|
||||
fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
let fixture = fs::read_to_string(fixture_path(input_name)).unwrap();
|
||||
let fixture: FixtureRequest = serde_json::from_str(&fixture).unwrap();
|
||||
let mut request = ChatRequest {
|
||||
request_id: "deepseek-v32-fixture".to_string(),
|
||||
messages: fixture
|
||||
.messages
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, message)| match message {
|
||||
FixtureMessage::System { content } => ChatMessage::system(content),
|
||||
FixtureMessage::Developer { content, tools } => ChatMessage::developer(
|
||||
content,
|
||||
(!tools.is_empty()).then(|| to_chat_tools(&tools)),
|
||||
),
|
||||
FixtureMessage::User { content } => ChatMessage::user(content),
|
||||
FixtureMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
tool_calls,
|
||||
} => {
|
||||
let mut blocks = Vec::new();
|
||||
if !reasoning_content.is_empty() {
|
||||
blocks.push(AssistantContentBlock::Reasoning {
|
||||
text: reasoning_content,
|
||||
});
|
||||
}
|
||||
if !content.is_empty() {
|
||||
blocks.push(AssistantContentBlock::Text { text: content });
|
||||
}
|
||||
blocks.extend(tool_calls.into_iter().enumerate().map(
|
||||
|(tool_index, tool_call)| {
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: tool_call.id.unwrap_or_else(|| {
|
||||
format!("fixture-tool-call-{index}-{tool_index}")
|
||||
}),
|
||||
name: tool_call.function.name,
|
||||
arguments: tool_call.function.arguments,
|
||||
})
|
||||
},
|
||||
));
|
||||
ChatMessage::assistant_blocks(blocks)
|
||||
}
|
||||
FixtureMessage::Tool {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => ChatMessage::tool_response(
|
||||
content,
|
||||
tool_call_id.unwrap_or_else(|| format!("fixture-tool-response-{index}")),
|
||||
),
|
||||
})
|
||||
.collect(),
|
||||
tools: to_chat_tools(&fixture.tools),
|
||||
tool_choice: if fixture.tools.is_empty() {
|
||||
ChatToolChoice::None
|
||||
} else {
|
||||
ChatToolChoice::Auto
|
||||
},
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
if matches!(
|
||||
request.messages.last().map(ChatMessage::role),
|
||||
Some(ChatRole::Assistant)
|
||||
) {
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
}
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request
|
||||
}
|
||||
|
||||
fn to_chat_tools(tools: &[FixtureTool]) -> Vec<ChatTool> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| ChatTool {
|
||||
name: tool.function.name.clone(),
|
||||
description: tool.function.description.clone(),
|
||||
parameters: tool.function.parameters.clone(),
|
||||
strict: tool.function.strict,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("src/renderer/deepseek_v32")
|
||||
.join("fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn assert_fixture(input_name: &str, expected: ExpectFile) {
|
||||
let request = fixture_request(input_name);
|
||||
let rendered = render_request(&request);
|
||||
expected.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_vllm_parity_prompt_for_request_level_tools_fixture() {
|
||||
assert_fixture(
|
||||
"test_input.json",
|
||||
expect_file!["fixtures/test_output_vllm_parity.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_official_search_fixture_without_date() {
|
||||
assert_fixture(
|
||||
"test_input_search_wo_date.json",
|
||||
expect_file!["fixtures/test_output_search_wo_date.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_official_search_fixture_with_date() {
|
||||
assert_fixture(
|
||||
"test_input_search_w_date.json",
|
||||
expect_file!["fixtures/test_output_search_w_date.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_level_tools_are_lowered_as_synthetic_leading_system_message() {
|
||||
let mut request = ChatRequest {
|
||||
request_id: "deepseek-v32-tools".to_string(),
|
||||
messages: vec![
|
||||
ChatMessage::system("System prompt."),
|
||||
ChatMessage::text(ChatRole::User, "Hello"),
|
||||
],
|
||||
tools: vec![ChatTool {
|
||||
name: "lookup".to_string(),
|
||||
description: Some("Look things up".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}),
|
||||
strict: None,
|
||||
}],
|
||||
tool_choice: ChatToolChoice::Auto,
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.starts_with("<|begin▁of▁sentence|>\n\n## Tools\n"));
|
||||
assert!(rendered.contains("</functions>\nSystem prompt."));
|
||||
assert!(rendered.ends_with("<|User|>Hello<|Assistant|><think>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn developer_turn_is_treated_as_last_user_like_turn() {
|
||||
let request = thinking_request(vec![ChatMessage::developer("Follow policy.", None)]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains("# The user's message is: Follow policy."));
|
||||
assert!(rendered.ends_with("<|Assistant|><think>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_assistant_reasoning_is_dropped_before_final_user_turn() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "internal reasoning".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "Visible answer.".to_string(),
|
||||
},
|
||||
]),
|
||||
ChatMessage::user("What about the next one?"),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(!rendered.contains("internal reasoning"));
|
||||
assert!(rendered.contains("Visible answer.<|end▁of▁sentence|>"));
|
||||
assert!(rendered.ends_with("<|User|>What about the next one?<|Assistant|><think>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_assistant_reasoning_is_dropped_before_final_developer_turn() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "internal reasoning".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "Visible answer.".to_string(),
|
||||
},
|
||||
]),
|
||||
ChatMessage::developer("Follow the rubric.", None),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(!rendered.contains("internal reasoning"));
|
||||
assert!(rendered.contains("Visible answer.<|end▁of▁sentence|>"));
|
||||
assert!(rendered.ends_with(
|
||||
"<|User|>\n\n# The user's message is: Follow the rubric.<|Assistant|><think>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_results_after_last_user_resume_thinking() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::user("Check the weather."),
|
||||
ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "call-weather".to_string(),
|
||||
name: "weather".to_string(),
|
||||
arguments: "{\"city\":\"Hangzhou\"}".to_string(),
|
||||
})]),
|
||||
ChatMessage::tool_response("{\"ok\":true}", "call-weather"),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains(
|
||||
"<|User|>Check the weather.<|Assistant|><think></think>\n\n<|DSML|function_calls>"
|
||||
));
|
||||
assert!(rendered.ends_with("</function_results>\n\n<think>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_results_follow_assistant_tool_call_id_order() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::user("Check two cities."),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "call-hangzhou".to_string(),
|
||||
name: "weather".to_string(),
|
||||
arguments: "{\"city\":\"Hangzhou\"}".to_string(),
|
||||
}),
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "call-beijing".to_string(),
|
||||
name: "weather".to_string(),
|
||||
arguments: "{\"city\":\"Beijing\"}".to_string(),
|
||||
}),
|
||||
]),
|
||||
ChatMessage::tool_response("{\"city\":\"Beijing\"}", "call-beijing"),
|
||||
ChatMessage::tool_response("{\"city\":\"Hangzhou\"}", "call-hangzhou"),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains(
|
||||
"<function_results>\n<result>{\"city\":\"Hangzhou\"}</result>\n<result>{\"city\":\"Beijing\"}</result>\n</function_results>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_results_require_matching_tool_call_ids() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::user("Check the weather."),
|
||||
ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "call-weather".to_string(),
|
||||
name: "weather".to_string(),
|
||||
arguments: "{\"city\":\"Hangzhou\"}".to_string(),
|
||||
})]),
|
||||
ChatMessage::tool_response("{\"ok\":true}", "call-unknown"),
|
||||
]);
|
||||
|
||||
let error = render_result(&request).unwrap_err();
|
||||
|
||||
expect!["chat template error: invalid DeepSeek V3.2 tool message: unknown tool_call_id `call-unknown`"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_after_last_user_requires_reasoning_or_tool_calls() {
|
||||
let request = thinking_request(vec![
|
||||
ChatMessage::user("Hello"),
|
||||
ChatMessage::assistant_text("Hi there."),
|
||||
]);
|
||||
|
||||
let error = render_result(&request).unwrap_err();
|
||||
|
||||
expect!["chat template error: invalid DeepSeek V3.2 assistant message after last user message: expected reasoning or tool calls"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
#[test]
|
||||
fn render_rejects_multimodal_input() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::user(vec![ChatContentPart::image_url(
|
||||
"data:image/png;base64,test",
|
||||
)])],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = DeepSeekV32ChatRenderer::new().render(&request).unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::UnsupportedMultimodalContent("image_url")
|
||||
));
|
||||
}
|
||||
@@ -0,0 +1,558 @@
|
||||
//! DeepSeek V4 prompt renderer.
|
||||
//!
|
||||
//! Original Python implementation:
|
||||
//! <https://github.com/vllm-project/vllm/blob/main/vllm/tokenizers/deepseek_v4_encoding.py>
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json_fmt::JsonFormat;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort};
|
||||
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
|
||||
|
||||
const BOS_TOKEN: &str = "<|begin▁of▁sentence|>";
|
||||
const EOS_TOKEN: &str = "<|end▁of▁sentence|>";
|
||||
const THINKING_START_TOKEN: &str = "<think>";
|
||||
const THINKING_END_TOKEN: &str = "</think>";
|
||||
const DSML_TOKEN: &str = "|DSML|";
|
||||
const USER_SP_TOKEN: &str = "<|User|>";
|
||||
const ASSISTANT_SP_TOKEN: &str = "<|Assistant|>";
|
||||
const REASONING_EFFORT_MAX: &str = concat!(
|
||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n",
|
||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n",
|
||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n",
|
||||
);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ThinkingMode {
|
||||
Chat,
|
||||
Thinking,
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RenderedToolSchema<'a> {
|
||||
name: &'a str,
|
||||
description: Option<&'a str>,
|
||||
parameters: &'a Value,
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
/// Render one chat request into the final prompt string.
|
||||
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
let (thinking_mode, max_reasoning_effort) = resolve_thinking_options(request)?;
|
||||
let request_tools = request_tools(request);
|
||||
let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
|
||||
let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
|
||||
&& !rendered_tools_present(request, request_tools);
|
||||
let last_user_render_index =
|
||||
find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system);
|
||||
let mut out = String::from(BOS_TOKEN);
|
||||
if thinking_mode == ThinkingMode::Thinking && max_reasoning_effort {
|
||||
out.push_str(REASONING_EFFORT_MAX);
|
||||
}
|
||||
|
||||
let mut request_tools_attached = false;
|
||||
let mut render_index = 0isize;
|
||||
if synthetic_tool_system {
|
||||
render_system_message(&mut out, None, request_tools)?;
|
||||
request_tools_attached = true;
|
||||
render_index += 1;
|
||||
}
|
||||
|
||||
for (message_index, message) in request.messages.iter().enumerate() {
|
||||
if is_following_tool_response(request.messages.as_slice(), message_index) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let current_render_index = render_index;
|
||||
render_index += 1;
|
||||
|
||||
match message {
|
||||
ChatMessage::System { content } => {
|
||||
let tools = if !request_tools_attached {
|
||||
request_tools_attached = true;
|
||||
request_tools
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
render_system_message(&mut out, Some(content), tools)?;
|
||||
}
|
||||
ChatMessage::Developer { content, tools } => {
|
||||
render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?;
|
||||
}
|
||||
ChatMessage::User { content } => render_user_message(&mut out, content)?,
|
||||
ChatMessage::Assistant { content } => {
|
||||
// Mirror Python: thinking block (reasoning + </think>) is
|
||||
// emitted whenever thinking is active and reasoning isn't
|
||||
// dropped - i.e. drop_thinking is off OR this turn lies
|
||||
// strictly after the last user turn.
|
||||
let emit_thinking_block = thinking_mode == ThinkingMode::Thinking
|
||||
&& (!drop_thinking || current_render_index > last_user_render_index);
|
||||
let append_eos = !(message_index + 1 == request.messages.len()
|
||||
&& request.chat_options.continue_final_message());
|
||||
render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?;
|
||||
}
|
||||
ChatMessage::ToolResponse { .. } => {
|
||||
render_tool_response_block(&mut out, request.messages.as_slice(), message_index)?;
|
||||
}
|
||||
}
|
||||
|
||||
if is_user_like_entry(message)
|
||||
&& next_rendered_entry_is_assistant_or_end(request.messages.as_slice(), message_index)
|
||||
{
|
||||
write_assistant_transition(
|
||||
&mut out,
|
||||
thinking_mode,
|
||||
drop_thinking,
|
||||
current_render_index >= last_user_render_index,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Resolve DeepSeek V4's thinking controls. Unlike the Python tokenizer
|
||||
/// wrapper, the Rust renderer only consumes the typed top-level
|
||||
/// `reasoning_effort`; the generic template-kwargs map is left for HF
|
||||
/// templates.
|
||||
fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, bool)> {
|
||||
let mut thinking_mode = match request.enable_thinking()?.unwrap_or(false) {
|
||||
true => ThinkingMode::Thinking,
|
||||
false => ThinkingMode::Chat,
|
||||
};
|
||||
let mut max_reasoning_effort = false;
|
||||
|
||||
match request.chat_options.reasoning_effort {
|
||||
Some(ReasoningEffort::None) => thinking_mode = ThinkingMode::Chat,
|
||||
Some(ReasoningEffort::Max | ReasoningEffort::XHigh) => max_reasoning_effort = true,
|
||||
Some(_) | None => {}
|
||||
}
|
||||
|
||||
Ok((thinking_mode, max_reasoning_effort))
|
||||
}
|
||||
|
||||
/// Return request-level tools only when native tool parsing is enabled.
|
||||
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
|
||||
if request.tool_parsing_enabled() {
|
||||
request.tools.as_slice()
|
||||
} else {
|
||||
&[]
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether request tools need a synthetic leading system entry.
|
||||
fn needs_synthetic_tool_system(request: &ChatRequest, request_tools: &[ChatTool]) -> bool {
|
||||
!request_tools.is_empty()
|
||||
&& !request
|
||||
.messages
|
||||
.iter()
|
||||
.any(|message| matches!(message, ChatMessage::System { .. }))
|
||||
}
|
||||
|
||||
/// Return whether any rendered message carries tool schemas.
|
||||
fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) -> bool {
|
||||
!request_tools.is_empty()
|
||||
|| request.messages.iter().any(|message| {
|
||||
matches!(
|
||||
message,
|
||||
ChatMessage::Developer {
|
||||
tools: Some(tools),
|
||||
..
|
||||
} if !tools.is_empty()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Find the last user-like turn after inline tool-response merging.
|
||||
fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize {
|
||||
let mut render_index = isize::from(synthetic_tool_system);
|
||||
let mut last_user_index = -1;
|
||||
|
||||
for (message_index, message) in messages.iter().enumerate() {
|
||||
if is_following_tool_response(messages, message_index) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_user_like_entry(message) {
|
||||
last_user_index = render_index;
|
||||
}
|
||||
render_index += 1;
|
||||
}
|
||||
|
||||
last_user_index
|
||||
}
|
||||
|
||||
/// Return whether this tool message is already covered by a previous tool run.
|
||||
fn is_following_tool_response(messages: &[ChatMessage], message_index: usize) -> bool {
|
||||
matches!(messages[message_index], ChatMessage::ToolResponse { .. })
|
||||
&& message_index > 0
|
||||
&& matches!(
|
||||
messages[message_index - 1],
|
||||
ChatMessage::ToolResponse { .. }
|
||||
)
|
||||
}
|
||||
|
||||
/// Return whether one rendered entry should be treated as user-like.
|
||||
fn is_user_like_entry(message: &ChatMessage) -> bool {
|
||||
matches!(
|
||||
message,
|
||||
ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. }
|
||||
)
|
||||
}
|
||||
|
||||
/// Return whether the next rendered entry is assistant, or there is no next
|
||||
/// entry.
|
||||
fn next_rendered_entry_is_assistant_or_end(messages: &[ChatMessage], message_index: usize) -> bool {
|
||||
let mut next_index = message_index + 1;
|
||||
if matches!(messages[message_index], ChatMessage::ToolResponse { .. }) {
|
||||
while next_index < messages.len()
|
||||
&& matches!(messages[next_index], ChatMessage::ToolResponse { .. })
|
||||
{
|
||||
next_index += 1;
|
||||
}
|
||||
}
|
||||
|
||||
messages
|
||||
.get(next_index)
|
||||
.map(|message| matches!(message, ChatMessage::Assistant { .. }))
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Render the tool preamble shown to the model, V4 flavor.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
|
||||
out.push_str(
|
||||
r#"## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
"#,
|
||||
);
|
||||
|
||||
for (index, tool) in tools.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_schema(out, tool)?;
|
||||
}
|
||||
|
||||
out.push_str(
|
||||
"\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n",
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialize one typed tool schema into the JSON shape embedded in the prompt.
|
||||
fn render_tool_schema(out: &mut String, tool: &ChatTool) -> Result<()> {
|
||||
out.push_str(&json_dumps(&RenderedToolSchema {
|
||||
name: &tool.name,
|
||||
description: tool.description.as_deref(),
|
||||
parameters: &tool.parameters,
|
||||
strict: tool.strict,
|
||||
})?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render a system turn, optionally followed by the V4 tool preamble.
|
||||
fn render_system_message(
|
||||
out: &mut String,
|
||||
content: Option<&ChatContent>,
|
||||
tools: &[ChatTool],
|
||||
) -> Result<()> {
|
||||
if let Some(content) = content {
|
||||
write_chat_content(out, content)?;
|
||||
}
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Developer messages are rendered as user-like turns with optional tools.
|
||||
fn render_developer_message(
|
||||
out: &mut String,
|
||||
content: &ChatContent,
|
||||
tools: &[ChatTool],
|
||||
) -> Result<()> {
|
||||
if content.is_empty() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V4 developer message: empty content".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str(USER_SP_TOKEN);
|
||||
write_chat_content(out, content)?;
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render one plain user turn.
|
||||
fn render_user_message(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
out.push_str(USER_SP_TOKEN);
|
||||
write_chat_content(out, content)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render a contiguous tool-response run as one synthetic user turn.
|
||||
fn render_tool_response_block(
|
||||
out: &mut String,
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
) -> Result<()> {
|
||||
let (block_start, block_end) = tool_response_block_bounds(messages, message_index);
|
||||
let sorted_indices = sorted_tool_response_indices(messages, block_start, block_end);
|
||||
|
||||
out.push_str(USER_SP_TOKEN);
|
||||
for (offset, message_index) in sorted_indices.iter().enumerate() {
|
||||
if offset > 0 {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
let ChatMessage::ToolResponse { content, .. } = &messages[*message_index] else {
|
||||
unreachable!("tool response block should only contain tool messages");
|
||||
};
|
||||
write_tool_result(out, content)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the contiguous tool-response block containing `actual_index`.
|
||||
fn tool_response_block_bounds(messages: &[ChatMessage], actual_index: usize) -> (usize, usize) {
|
||||
let mut block_start = actual_index;
|
||||
while block_start > 0 && matches!(messages[block_start - 1], ChatMessage::ToolResponse { .. }) {
|
||||
block_start -= 1;
|
||||
}
|
||||
|
||||
let mut block_end = actual_index + 1;
|
||||
while block_end < messages.len()
|
||||
&& matches!(messages[block_end], ChatMessage::ToolResponse { .. })
|
||||
{
|
||||
block_end += 1;
|
||||
}
|
||||
|
||||
(block_start, block_end)
|
||||
}
|
||||
|
||||
fn sorted_tool_response_indices(
|
||||
messages: &[ChatMessage],
|
||||
block_start: usize,
|
||||
block_end: usize,
|
||||
) -> Vec<usize> {
|
||||
let Some(tool_call_order) = last_tool_call_order_before(messages, block_start) else {
|
||||
return (block_start..block_end).collect();
|
||||
};
|
||||
|
||||
let mut indices = (block_start..block_end).collect::<Vec<_>>();
|
||||
indices.sort_by_key(|index| {
|
||||
let ChatMessage::ToolResponse { tool_call_id, .. } = &messages[*index] else {
|
||||
unreachable!("tool response block should only contain tool messages");
|
||||
};
|
||||
tool_call_order.get(tool_call_id.as_str()).copied().unwrap_or(0)
|
||||
});
|
||||
indices
|
||||
}
|
||||
|
||||
fn last_tool_call_order_before(
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
) -> Option<HashMap<&str, usize>> {
|
||||
let mut tool_call_order = None;
|
||||
for message in &messages[..message_index] {
|
||||
if let ChatMessage::Assistant { content } = message {
|
||||
let order = content
|
||||
.tool_calls()
|
||||
.enumerate()
|
||||
.map(|(index, tool_call)| (tool_call.id.as_str(), index))
|
||||
.collect::<HashMap<_, _>>();
|
||||
if !order.is_empty() {
|
||||
tool_call_order = Some(order);
|
||||
}
|
||||
}
|
||||
}
|
||||
tool_call_order
|
||||
}
|
||||
|
||||
/// Render one tool response payload inside a V4 `<tool_result>` block.
|
||||
fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
out.push_str("<tool_result>");
|
||||
write_chat_content(out, content)?;
|
||||
out.push_str("</tool_result>");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append the assistant transition token after a user-like turn.
|
||||
fn write_assistant_transition(
|
||||
out: &mut String,
|
||||
thinking_mode: ThinkingMode,
|
||||
drop_thinking: bool,
|
||||
opens_thinking: bool,
|
||||
) {
|
||||
out.push_str(ASSISTANT_SP_TOKEN);
|
||||
if thinking_mode == ThinkingMode::Thinking && (!drop_thinking || opens_thinking) {
|
||||
out.push_str(THINKING_START_TOKEN);
|
||||
} else {
|
||||
out.push_str(THINKING_END_TOKEN);
|
||||
}
|
||||
}
|
||||
|
||||
/// Render one assistant turn, including optional reasoning, DSML tool calls,
|
||||
/// and the trailing EOS marker.
|
||||
fn render_assistant_message(
|
||||
out: &mut String,
|
||||
emit_thinking_block: bool,
|
||||
append_eos: bool,
|
||||
content: &[AssistantContentBlock],
|
||||
) -> Result<()> {
|
||||
let has_tool_calls = content.has_tool_calls();
|
||||
|
||||
if emit_thinking_block {
|
||||
if content.has_reasoning() {
|
||||
write_assistant_reasoning(out, content);
|
||||
}
|
||||
out.push_str(THINKING_END_TOKEN);
|
||||
}
|
||||
|
||||
write_assistant_text(out, content);
|
||||
|
||||
if has_tool_calls {
|
||||
out.push_str("\n\n<|DSML|tool_calls>\n");
|
||||
for (index, tool_call) in content.tool_calls().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_call(out, tool_call)?;
|
||||
}
|
||||
out.push_str("\n</|DSML|tool_calls>");
|
||||
}
|
||||
|
||||
if append_eos {
|
||||
out.push_str(EOS_TOKEN);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render one assistant tool call in DSML XML-like format.
|
||||
fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call)?;
|
||||
write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convert one assistant tool-call arguments object into DSML parameter form.
|
||||
///
|
||||
/// String values are emitted raw with `string="true"`, while all other JSON
|
||||
/// values are rendered with JSON syntax and `string="false"`.
|
||||
fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek V4: {error}"
|
||||
))
|
||||
})?;
|
||||
let Some(arguments) = arguments.as_object() else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let mut wrote_parameter = false;
|
||||
for (key, value) in arguments {
|
||||
if wrote_parameter {
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
let is_string = matches!(value, Value::String(_));
|
||||
write!(
|
||||
out,
|
||||
"<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
|
||||
if is_string { "true" } else { "false" }
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
|
||||
match value {
|
||||
Value::String(value) => out.push_str(value),
|
||||
value => out.push_str(&json_dumps(value)?),
|
||||
}
|
||||
|
||||
write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
|
||||
wrote_parameter = true;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write chat content directly into the destination buffer without flattening
|
||||
/// it into an intermediate `String`.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => out.push_str(text),
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
out.push_str(part.as_text()?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write all reasoning blocks in encounter order.
|
||||
fn write_assistant_reasoning(out: &mut String, content: &[AssistantContentBlock]) {
|
||||
for block in content {
|
||||
if let AssistantContentBlock::Reasoning { text } = block {
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write all visible assistant text blocks in encounter order.
|
||||
fn write_assistant_text(out: &mut String, content: &[AssistantContentBlock]) {
|
||||
for block in content {
|
||||
if let AssistantContentBlock::Text { text } = block {
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compact JSON serialization used by this renderer for exact prompt text.
|
||||
fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
|
||||
JsonFormat::new()
|
||||
.comma(", ")
|
||||
.expect("literal comma separator is valid JSON")
|
||||
.colon(": ")
|
||||
.expect("literal colon separator is valid JSON")
|
||||
.ascii(false)
|
||||
.format_to_string(value)
|
||||
.map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"failed to serialize DeepSeek V4 JSON payload: {error}"
|
||||
))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a specific location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city name"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "Temperature unit"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Search the web for information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query"
|
||||
},
|
||||
"num_results": {
|
||||
"type": "integer",
|
||||
"description": "Number of results to return"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in Beijing?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "The user wants to know the weather in Beijing. I should use the get_weather tool.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_001",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_001",
|
||||
"content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "Got the weather data. Let me format a nice response.",
|
||||
"content": "The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "The user said hello, I should greet back.",
|
||||
"content": "Hi there! How can I help you?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the capital of France?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "The user asks about the capital of France. It is Paris.",
|
||||
"content": "The capital of France is Paris."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,36 @@
|
||||
<|begin▁of▁sentence|>You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="get_weather">
|
||||
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
|
||||
<|DSML|parameter name="unit" string="true">celsius</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1 @@
|
||||
<|begin▁of▁sentence|>You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,30 @@
|
||||
mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Dedicated DeepSeek V4 renderer.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct DeepSeekV4ChatRenderer;
|
||||
|
||||
impl DeepSeekV4ChatRenderer {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for DeepSeekV4ChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,369 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use expect_test::{ExpectFile, expect, expect_file};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::DeepSeekV4ChatRenderer;
|
||||
use crate::event::{AssistantContentBlock, AssistantToolCall};
|
||||
use crate::request::{
|
||||
ChatMessage, ChatRequest, ChatTool, ChatToolChoice, GenerationPromptMode, ReasoningEffort,
|
||||
};
|
||||
use crate::{ChatRenderer, ChatRole};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum FixtureFile {
|
||||
WithTools(FixtureRequest),
|
||||
MessagesOnly(Vec<FixtureMessage>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureRequest {
|
||||
#[serde(default)]
|
||||
tools: Vec<FixtureTool>,
|
||||
messages: Vec<FixtureMessage>,
|
||||
}
|
||||
|
||||
impl FixtureFile {
|
||||
fn into_parts(self) -> (Vec<FixtureTool>, Vec<FixtureMessage>) {
|
||||
match self {
|
||||
Self::WithTools(req) => (req.tools, req.messages),
|
||||
Self::MessagesOnly(messages) => (Vec::new(), messages),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureTool {
|
||||
function: FixtureToolFunction,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolFunction {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
parameters: Value,
|
||||
#[serde(default)]
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "role", rename_all = "snake_case")]
|
||||
enum FixtureMessage {
|
||||
System {
|
||||
content: String,
|
||||
},
|
||||
Developer {
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
tools: Vec<FixtureTool>,
|
||||
},
|
||||
User {
|
||||
content: String,
|
||||
},
|
||||
Assistant {
|
||||
#[serde(default)]
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
reasoning_content: String,
|
||||
#[serde(default)]
|
||||
tool_calls: Vec<FixtureToolCall>,
|
||||
},
|
||||
Tool {
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
tool_call_id: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolCall {
|
||||
#[serde(default)]
|
||||
id: Option<String>,
|
||||
function: FixtureToolCallFunction,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureToolCallFunction {
|
||||
name: String,
|
||||
arguments: String,
|
||||
}
|
||||
|
||||
fn render_request(request: &ChatRequest) -> String {
|
||||
DeepSeekV4ChatRenderer::new()
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_text()
|
||||
.expect("deepseek v4 renderer should return text prompt")
|
||||
}
|
||||
|
||||
fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
let fixture = fs::read_to_string(fixture_path(input_name)).unwrap();
|
||||
let fixture: FixtureFile = serde_json::from_str(&fixture).unwrap();
|
||||
let (fixture_tools, fixture_messages) = fixture.into_parts();
|
||||
let mut request = ChatRequest {
|
||||
request_id: "deepseek-v4-fixture".to_string(),
|
||||
messages: fixture_messages
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, message)| match message {
|
||||
FixtureMessage::System { content } => ChatMessage::system(content),
|
||||
FixtureMessage::Developer { content, tools } => ChatMessage::developer(
|
||||
content,
|
||||
(!tools.is_empty()).then(|| to_chat_tools(&tools)),
|
||||
),
|
||||
FixtureMessage::User { content } => ChatMessage::user(content),
|
||||
FixtureMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
tool_calls,
|
||||
} => {
|
||||
let mut blocks = Vec::new();
|
||||
if !reasoning_content.is_empty() {
|
||||
blocks.push(AssistantContentBlock::Reasoning {
|
||||
text: reasoning_content,
|
||||
});
|
||||
}
|
||||
if !content.is_empty() {
|
||||
blocks.push(AssistantContentBlock::Text { text: content });
|
||||
}
|
||||
blocks.extend(tool_calls.into_iter().enumerate().map(
|
||||
|(tool_index, tool_call)| {
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: tool_call.id.unwrap_or_else(|| {
|
||||
format!("fixture-tool-call-{index}-{tool_index}")
|
||||
}),
|
||||
name: tool_call.function.name,
|
||||
arguments: tool_call.function.arguments,
|
||||
})
|
||||
},
|
||||
));
|
||||
ChatMessage::assistant_blocks(blocks)
|
||||
}
|
||||
FixtureMessage::Tool {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => ChatMessage::tool_response(
|
||||
content,
|
||||
tool_call_id.unwrap_or_else(|| format!("fixture-tool-response-{index}")),
|
||||
),
|
||||
})
|
||||
.collect(),
|
||||
tools: to_chat_tools(&fixture_tools),
|
||||
tool_choice: if fixture_tools.is_empty() {
|
||||
ChatToolChoice::None
|
||||
} else {
|
||||
ChatToolChoice::Auto
|
||||
},
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
if matches!(
|
||||
request.messages.last().map(ChatMessage::role),
|
||||
Some(ChatRole::Assistant)
|
||||
) {
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
}
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request
|
||||
}
|
||||
|
||||
fn to_chat_tools(tools: &[FixtureTool]) -> Vec<ChatTool> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| ChatTool {
|
||||
name: tool.function.name.clone(),
|
||||
description: tool.function.description.clone(),
|
||||
parameters: tool.function.parameters.clone(),
|
||||
strict: tool.function.strict,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("src/renderer/deepseek_v4")
|
||||
.join("fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn assert_fixture(input_name: &str, expected: ExpectFile) {
|
||||
let request = fixture_request(input_name);
|
||||
let rendered = render_request(&request);
|
||||
expected.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_v4_fixture_1_tool_call_round_trip() {
|
||||
assert_fixture(
|
||||
"test_input_1.json",
|
||||
expect_file!["fixtures/test_output_1.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_v4_fixture_2_multi_turn_drop_thinking() {
|
||||
assert_fixture(
|
||||
"test_input_2.json",
|
||||
expect_file!["fixtures/test_output_2.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![ChatMessage::user("solve it")],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect![[r#"
|
||||
<|begin▁of▁sentence|>Reasoning Effort: Absolute maximum with no shortcuts permitted.
|
||||
You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.
|
||||
Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.
|
||||
|
||||
<|User|>solve it<|Assistant|><think>"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_effort_none_disables_thinking() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![ChatMessage::user("answer directly")],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect!["<|begin▁of▁sentence|><|User|>answer directly<|Assistant|></think>"]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_effort_template_kwarg_is_ignored() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![ChatMessage::user("solve it")],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request.chat_options.template_kwargs.insert(
|
||||
"reasoning_effort".to_string(),
|
||||
Value::String("max".to_string()),
|
||||
);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect!["<|begin▁of▁sentence|><|User|>solve it<|Assistant|><think>"].assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_results_are_sorted_by_previous_assistant_tool_call_order() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "second".to_string(),
|
||||
name: "second_tool".to_string(),
|
||||
arguments: "{}".to_string(),
|
||||
}),
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "first".to_string(),
|
||||
name: "first_tool".to_string(),
|
||||
arguments: "{}".to_string(),
|
||||
}),
|
||||
]),
|
||||
ChatMessage::tool_response("first result", "first"),
|
||||
ChatMessage::tool_response("second result", "second"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect![[r#"
|
||||
<|begin▁of▁sentence|>
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="second_tool">
|
||||
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="first_tool">
|
||||
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>second result</tool_result>
|
||||
|
||||
<tool_result>first result</tool_result><|Assistant|></think>"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_thinking_false_keeps_prior_assistant_reasoning() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "old reasoning".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "old answer".to_string(),
|
||||
},
|
||||
]),
|
||||
ChatMessage::user("next"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), Value::Bool(true));
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("drop_thinking".to_string(), Value::Bool(false));
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect!(
|
||||
"<|begin▁of▁sentence|>old reasoning</think>old answer<|end▁of▁sentence|><|User|>next<|Assistant|><think>"
|
||||
)
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continue_final_assistant_omits_final_eos() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::user("write"),
|
||||
ChatMessage::assistant_text("partial answer"),
|
||||
],
|
||||
chat_options: crate::request::ChatOptions {
|
||||
generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant,
|
||||
..Default::default()
|
||||
},
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
expect!["<|begin▁of▁sentence|><|User|>write<|Assistant|></think>partial answer"]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
use thiserror::Error as ThisError;
|
||||
|
||||
#[derive(Debug, ThisError)]
|
||||
pub(crate) enum TemplateError {
|
||||
#[error("failed to render jinja template")]
|
||||
Jinja(#[from] minijinja::Error),
|
||||
#[error("failed to read chat template file")]
|
||||
ReadTemplateFile(#[source] std::io::Error),
|
||||
#[error("chat template looks like a file path but does not exist")]
|
||||
MissingTemplatePath,
|
||||
#[error("failed to parse chat_template.json")]
|
||||
ParseTemplateJson(#[source] serde_json::Error),
|
||||
#[error("chat_template.json does not contain a valid template")]
|
||||
InvalidTemplateJson,
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt};
|
||||
use minijinja::machinery::{WhitespaceConfig, parse};
|
||||
use minijinja::syntax::SyntaxConfig;
|
||||
use serde_with::DeserializeFromStr;
|
||||
|
||||
/// Chat template content format.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum ChatTemplateContentFormat {
|
||||
/// Content is a simple string.
|
||||
#[default]
|
||||
String,
|
||||
/// Content is a list of structured parts (OpenAI format).
|
||||
OpenAi,
|
||||
}
|
||||
|
||||
/// Configurable chat-template content format selection.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
pub enum ChatTemplateContentFormatOption {
|
||||
/// Detect the format from the template source.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Always flatten content into plain strings before rendering.
|
||||
String,
|
||||
/// Always pass content through in OpenAI-compatible structured form.
|
||||
OpenAi,
|
||||
}
|
||||
|
||||
impl ChatTemplateContentFormatOption {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const OPENAI_LITERAL: &str = "openai";
|
||||
pub const STRING_LITERAL: &str = "string";
|
||||
}
|
||||
|
||||
impl FromStr for ChatTemplateContentFormatOption {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case(Self::STRING_LITERAL) {
|
||||
Ok(Self::String)
|
||||
} else if value.eq_ignore_ascii_case(Self::OPENAI_LITERAL) {
|
||||
Ok(Self::OpenAi)
|
||||
} else {
|
||||
Err(format!(
|
||||
"invalid content format `{value}`; expected one of: {}, {}, {}",
|
||||
Self::AUTO_LITERAL,
|
||||
Self::STRING_LITERAL,
|
||||
Self::OPENAI_LITERAL
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ChatTemplateContentFormatOption {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Auto => f.write_str(Self::AUTO_LITERAL),
|
||||
Self::String => f.write_str(Self::STRING_LITERAL),
|
||||
Self::OpenAi => f.write_str(Self::OPENAI_LITERAL),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_var_access(expr: &Expr, varname: &str) -> bool {
|
||||
matches!(expr, Expr::Var(v) if v.id == varname)
|
||||
}
|
||||
|
||||
fn is_const_str(expr: &Expr, value: &str) -> bool {
|
||||
matches!(expr, Expr::Const(c) if c.value.as_str() == Some(value))
|
||||
}
|
||||
|
||||
fn is_attr_access(expr: &Expr, varname: &str, key: &str) -> bool {
|
||||
match expr {
|
||||
Expr::GetItem(g) => is_var_access(&g.expr, varname) && is_const_str(&g.subscript_expr, key),
|
||||
Expr::GetAttr(g) => is_var_access(&g.expr, varname) && g.name == key,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_var_or_elems_access(expr: &Expr, varname: &str, key: Option<&str>) -> bool {
|
||||
match expr {
|
||||
Expr::Filter(f) => {
|
||||
f.expr.as_ref().is_some_and(|inner| is_var_or_elems_access(inner, varname, key))
|
||||
}
|
||||
Expr::Test(t) => is_var_or_elems_access(&t.expr, varname, key),
|
||||
Expr::Slice(s) => is_var_or_elems_access(&s.expr, varname, key),
|
||||
_ => key.map_or_else(
|
||||
|| is_var_access(expr, varname),
|
||||
|key| is_attr_access(expr, varname, key),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_stmt<'a>(
|
||||
stmt: &'a Stmt<'a>,
|
||||
assignments: &mut Vec<&'a Set<'a>>,
|
||||
loops: &mut Vec<&'a ForLoop<'a>>,
|
||||
) {
|
||||
match stmt {
|
||||
Stmt::Template(t) => {
|
||||
for child in &t.children {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::ForLoop(fl) => {
|
||||
loops.push(fl);
|
||||
for child in &fl.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
for child in &fl.else_body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::IfCond(ic) => {
|
||||
for child in &ic.true_body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
for child in &ic.false_body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::WithBlock(wb) => {
|
||||
for child in &wb.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::Set(set_stmt) => assignments.push(set_stmt),
|
||||
Stmt::SetBlock(sb) => {
|
||||
for child in &sb.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::AutoEscape(ae) => {
|
||||
for child in &ae.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::FilterBlock(fb) => {
|
||||
for child in &fb.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::Block(b) => {
|
||||
for child in &b.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::Macro(m) => {
|
||||
for child in &m.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
Stmt::CallBlock(cb) => {
|
||||
for child in &cb.macro_decl.body {
|
||||
visit_stmt(child, assignments, loops);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_assignments_and_loops<'a>(
|
||||
root: &'a Stmt<'a>,
|
||||
) -> (Vec<&'a Set<'a>>, Vec<&'a ForLoop<'a>>) {
|
||||
let mut assignments = Vec::new();
|
||||
let mut loops = Vec::new();
|
||||
visit_stmt(root, &mut assignments, &mut loops);
|
||||
(assignments, loops)
|
||||
}
|
||||
|
||||
fn iter_nodes_assign_var_or_elems(root: &Stmt<'_>, varname: &str) -> Vec<String> {
|
||||
let (assignments, _) = collect_assignments_and_loops(root);
|
||||
|
||||
let mut discovered = vec![varname.to_string()];
|
||||
let mut seen = HashSet::from([varname.to_string()]);
|
||||
let mut related = VecDeque::from([varname.to_string()]);
|
||||
|
||||
while let Some(related_varname) = related.pop_front() {
|
||||
for assign in &assignments {
|
||||
let Expr::Var(lhs) = &assign.target else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if is_var_or_elems_access(&assign.expr, &related_varname, None) {
|
||||
let lhs_name = lhs.id.to_string();
|
||||
if seen.insert(lhs_name.clone()) {
|
||||
discovered.push(lhs_name.clone());
|
||||
if lhs_name != related_varname {
|
||||
related.push_back(lhs_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
discovered
|
||||
}
|
||||
|
||||
fn iter_nodes_assign_messages_item(root: &Stmt<'_>) -> Vec<String> {
|
||||
let message_varnames = iter_nodes_assign_var_or_elems(root, "messages");
|
||||
let (_, loops) = collect_assignments_and_loops(root);
|
||||
|
||||
let mut discovered = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
|
||||
for loop_ast in loops {
|
||||
let Expr::Var(target) = &loop_ast.target else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if message_varnames
|
||||
.iter()
|
||||
.any(|varname| is_var_or_elems_access(&loop_ast.iter, varname, None))
|
||||
{
|
||||
let target_name = target.id.to_string();
|
||||
if seen.insert(target_name.clone()) {
|
||||
discovered.push(target_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
discovered
|
||||
}
|
||||
|
||||
fn has_content_item_loop(root: &Stmt<'_>) -> bool {
|
||||
let message_varnames = iter_nodes_assign_messages_item(root);
|
||||
let (_, loops) = collect_assignments_and_loops(root);
|
||||
|
||||
loops.into_iter().any(|loop_ast| {
|
||||
matches!(loop_ast.target, Expr::Var(_))
|
||||
&& message_varnames
|
||||
.iter()
|
||||
.any(|varname| is_var_or_elems_access(&loop_ast.iter, varname, Some("content")))
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect the content format expected by a Jinja2 chat template based on AST
|
||||
/// analysis.
|
||||
pub fn detect_chat_template_content_format(template: &str) -> ChatTemplateContentFormat {
|
||||
let ast = match parse(
|
||||
template,
|
||||
"template",
|
||||
SyntaxConfig {},
|
||||
WhitespaceConfig::default(),
|
||||
) {
|
||||
Ok(ast) => ast,
|
||||
Err(_) => return ChatTemplateContentFormat::String,
|
||||
};
|
||||
|
||||
if has_content_item_loop(&ast) {
|
||||
ChatTemplateContentFormat::OpenAi
|
||||
} else {
|
||||
ChatTemplateContentFormat::String
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use expect_test::expect;
|
||||
|
||||
use super::{ChatTemplateContentFormat, detect_chat_template_content_format};
|
||||
|
||||
fn detect(template: &str) -> ChatTemplateContentFormat {
|
||||
detect_chat_template_content_format(template)
|
||||
}
|
||||
|
||||
fn vllm_examples_dir() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/templates/vllm_examples")
|
||||
.canonicalize()
|
||||
.expect("vLLM example template directory should exist locally")
|
||||
}
|
||||
|
||||
fn read_vllm_example(relative_path: &str) -> String {
|
||||
fs::read_to_string(vllm_examples_dir().join(relative_path))
|
||||
.unwrap_or_else(|_| panic!("failed to read vLLM example template: {relative_path}"))
|
||||
}
|
||||
|
||||
fn iter_vllm_example_template_paths() -> impl Iterator<Item = PathBuf> {
|
||||
let mut paths = fs::read_dir(vllm_examples_dir())
|
||||
.expect("failed to read vLLM example template directory")
|
||||
.map(|entry| entry.expect("failed to read vLLM example template dir entry").path())
|
||||
.filter(|path| path.extension().is_some_and(|ext| ext == "jinja"))
|
||||
.collect::<Vec<_>>();
|
||||
paths.sort();
|
||||
paths.into_iter()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_string_template_without_content_loop() {
|
||||
assert_eq!(
|
||||
detect("{% for message in messages %}{{ message.content }}{% endfor %}"),
|
||||
ChatTemplateContentFormat::String
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_openai_template_with_direct_content_loop() {
|
||||
assert_eq!(
|
||||
detect(
|
||||
"{% for message in messages %}{% for content in message['content'] %}{{ content }}{% endfor %}{% endfor %}"
|
||||
),
|
||||
ChatTemplateContentFormat::OpenAi
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_openai_template_with_messages_alias() {
|
||||
assert_eq!(
|
||||
detect(
|
||||
"{% set msgs = messages %}{% for message in msgs %}{% for content in message.content %}{{ content }}{% endfor %}{% endfor %}"
|
||||
),
|
||||
ChatTemplateContentFormat::OpenAi
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_detect_content_alias_loop_as_openai() {
|
||||
assert_eq!(
|
||||
detect(
|
||||
"{% for message in messages %}{% set parts = message.content %}{% for item in parts %}{{ item }}{% endfor %}{% endfor %}"
|
||||
),
|
||||
ChatTemplateContentFormat::String
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_treat_length_or_index_access_as_openai() {
|
||||
assert_eq!(
|
||||
detect("{% for message in messages %}{{ message.content|length }}{% endfor %}"),
|
||||
ChatTemplateContentFormat::String
|
||||
);
|
||||
assert_eq!(
|
||||
detect("{% for message in messages %}{{ message.content[0] }}{% endfor %}"),
|
||||
ChatTemplateContentFormat::String
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_vllm_example_template_formats() {
|
||||
let snapshot = iter_vllm_example_template_paths()
|
||||
.map(|path| {
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.expect("template file name should be valid UTF-8");
|
||||
let template = read_vllm_example(file_name);
|
||||
let format = detect(&template);
|
||||
format!("{file_name:50} => {format:?}")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
expect![[r#"
|
||||
template_alpaca.jinja => String
|
||||
template_baichuan.jinja => String
|
||||
template_chatglm.jinja => String
|
||||
template_chatglm2.jinja => String
|
||||
template_chatml.jinja => String
|
||||
template_falcon.jinja => String
|
||||
template_falcon_180b.jinja => String
|
||||
template_inkbot.jinja => String
|
||||
template_teleflm.jinja => String
|
||||
tool_chat_template_deepseekr1.jinja => String
|
||||
tool_chat_template_deepseekv3.jinja => String
|
||||
tool_chat_template_deepseekv31.jinja => String
|
||||
tool_chat_template_functiongemma.jinja => String
|
||||
tool_chat_template_gemma3_pythonic.jinja => OpenAi
|
||||
tool_chat_template_gemma4.jinja => OpenAi
|
||||
tool_chat_template_glm4.jinja => String
|
||||
tool_chat_template_granite.jinja => String
|
||||
tool_chat_template_granite_20b_fc.jinja => String
|
||||
tool_chat_template_hermes.jinja => String
|
||||
tool_chat_template_hunyuan_a13b.jinja => String
|
||||
tool_chat_template_internlm2_tool.jinja => String
|
||||
tool_chat_template_llama3.1_json.jinja => OpenAi
|
||||
tool_chat_template_llama3.2_json.jinja => OpenAi
|
||||
tool_chat_template_llama3.2_pythonic.jinja => String
|
||||
tool_chat_template_llama4_json.jinja => OpenAi
|
||||
tool_chat_template_llama4_pythonic.jinja => OpenAi
|
||||
tool_chat_template_minimax_m1.jinja => OpenAi
|
||||
tool_chat_template_mistral.jinja => String
|
||||
tool_chat_template_mistral3.jinja => OpenAi
|
||||
tool_chat_template_mistral_parallel.jinja => String
|
||||
tool_chat_template_phi4_mini.jinja => String
|
||||
tool_chat_template_qwen3coder.jinja => String
|
||||
tool_chat_template_toolace.jinja => String
|
||||
tool_chat_template_xlam_llama.jinja => String
|
||||
tool_chat_template_xlam_qwen.jinja => String"#]]
|
||||
.assert_eq(&snapshot);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,970 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{info, trace, warn};
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::backend::hf::{
|
||||
HfSpecialTokens, HfTokenizerConfig, ResolvedModelFiles, load_tokenizer_config,
|
||||
};
|
||||
|
||||
use self::format::{
|
||||
ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption,
|
||||
};
|
||||
use self::template::{CompiledChatTemplate, TemplateContext};
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
use crate::{
|
||||
AssistantContentBlock, AssistantMessageExt, ChatTool, Error, LoadModelBackendsOptions,
|
||||
};
|
||||
|
||||
mod error;
|
||||
mod format;
|
||||
mod template;
|
||||
mod tojson;
|
||||
|
||||
pub use template::{load_chat_template, resolve_chat_template};
|
||||
|
||||
pub use self::format::ChatTemplateContentFormatOption;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MultimodalRenderInfo {
|
||||
pub placeholder_token: String,
|
||||
}
|
||||
|
||||
/// Hugging Face chat-template renderer backed by the local Jinja chat-template
|
||||
/// state.
|
||||
pub struct HfChatRenderer {
|
||||
default_template: Option<CompiledChatTemplate>,
|
||||
default_template_kwargs: HashMap<String, Value>,
|
||||
content_format: ContentFormatOption,
|
||||
special_tokens: Option<HfSpecialTokens>,
|
||||
multimodal: Option<MultimodalRenderInfo>,
|
||||
}
|
||||
|
||||
impl HfChatRenderer {
|
||||
/// Create a renderer from the given template string.
|
||||
pub fn new(
|
||||
template: Option<String>,
|
||||
default_template_kwargs: HashMap<String, Value>,
|
||||
content_format: ContentFormatOption,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
default_template: template
|
||||
.map(|template| {
|
||||
CompiledChatTemplate::new(template, content_format)
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))
|
||||
})
|
||||
.transpose()?,
|
||||
default_template_kwargs,
|
||||
content_format,
|
||||
special_tokens: None,
|
||||
multimodal: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_special_tokens(mut self, special_tokens: Option<HfSpecialTokens>) -> Self {
|
||||
self.special_tokens = special_tokens;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_multimodal(mut self, multimodal: Option<MultimodalRenderInfo>) -> Self {
|
||||
self.multimodal = multimodal;
|
||||
self
|
||||
}
|
||||
|
||||
/// Create a renderer from the given model files and loading options.
|
||||
pub fn load(
|
||||
files: &ResolvedModelFiles,
|
||||
options: LoadModelBackendsOptions,
|
||||
multimodal: Option<MultimodalRenderInfo>,
|
||||
) -> Result<Self> {
|
||||
let HfTokenizerConfig {
|
||||
special_tokens,
|
||||
chat_template,
|
||||
..
|
||||
} = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
|
||||
let mut template = chat_template;
|
||||
let special_tokens = (!special_tokens.is_empty()).then_some(special_tokens);
|
||||
|
||||
if let Some(configured_template) = options.chat_template.as_deref() {
|
||||
template = Some(
|
||||
resolve_chat_template(configured_template)
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?,
|
||||
);
|
||||
info!("using configured chat template override");
|
||||
} else if let Some(chat_template_path) = files.chat_template_path.as_deref() {
|
||||
// If independent chat template file(s) exist and contain non-empty content,
|
||||
// they take priority over template entries in the tokenizer config
|
||||
let file_template = load_chat_template(chat_template_path)
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
|
||||
|
||||
if file_template.as_ref().is_some_and(|t| !t.trim().is_empty()) {
|
||||
info!(
|
||||
path = %chat_template_path.display(),
|
||||
"loaded dedicated chat template file, overriding tokenizer_config chat_template"
|
||||
);
|
||||
template = file_template;
|
||||
} else {
|
||||
warn!(
|
||||
path = %chat_template_path.display(),
|
||||
"ignoring empty dedicated chat template file and falling back to tokenizer_config chat_template"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self::new(
|
||||
template,
|
||||
options.default_chat_template_kwargs,
|
||||
options.chat_template_content_format,
|
||||
)?
|
||||
.with_special_tokens(special_tokens)
|
||||
.with_multimodal(multimodal))
|
||||
}
|
||||
|
||||
/// Apply the chat template to one chat request, rendering the prompt string
|
||||
/// to be tokenized and submitted to the model.
|
||||
///
|
||||
/// If the request carries a per-request `chat_template` override, a
|
||||
/// temporary template is compiled from that string and used instead of
|
||||
/// the model's default.
|
||||
fn apply_chat_template(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
let override_template = request
|
||||
.chat_options
|
||||
.chat_template
|
||||
.as_ref()
|
||||
.map(|template| {
|
||||
CompiledChatTemplate::new(template.clone(), self.content_format)
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
let template = override_template
|
||||
.as_ref()
|
||||
.or(self.default_template.as_ref())
|
||||
.ok_or(Error::MissingChatTemplate)?;
|
||||
|
||||
self.apply_chat_template_inner(template, request)
|
||||
}
|
||||
|
||||
fn apply_chat_template_inner(
|
||||
&self,
|
||||
effective_template: &CompiledChatTemplate,
|
||||
request: &ChatRequest,
|
||||
) -> Result<RenderedPrompt> {
|
||||
let messages = to_template_messages(
|
||||
&request.messages,
|
||||
effective_template.content_format(),
|
||||
self.multimodal.as_ref(),
|
||||
)?;
|
||||
let tools = request.tool_parsing_enabled().then(|| to_template_tools(&request.tools));
|
||||
trace!(
|
||||
message_count = messages.len(),
|
||||
content_format = ?effective_template.content_format(),
|
||||
?messages,
|
||||
?tools,
|
||||
"applying chat template"
|
||||
);
|
||||
|
||||
let mut merged_template_kwargs = self.default_template_kwargs.clone();
|
||||
merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());
|
||||
let prompt = effective_template
|
||||
.apply(TemplateContext {
|
||||
messages: &messages,
|
||||
add_generation_prompt: request.chat_options.add_generation_prompt(),
|
||||
continue_final_message: request.chat_options.continue_final_message(),
|
||||
tools: tools.as_deref(),
|
||||
documents: request.documents.as_deref(),
|
||||
template_kwargs: Some(&merged_template_kwargs),
|
||||
special_tokens: self.special_tokens.as_ref(),
|
||||
reasoning_effort: request.chat_options.reasoning_effort,
|
||||
})
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
|
||||
|
||||
trace!(
|
||||
prompt_len = prompt.len(),
|
||||
prompt, "rendered chat template prompt"
|
||||
);
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for HfChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
self.apply_chat_template(request)
|
||||
}
|
||||
}
|
||||
|
||||
/// Chat message in the JSON shape expected by Jinja chat templates.
|
||||
// TODO: borrow more fields directly from the original `ChatMessage`.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateMessage {
|
||||
role: &'static str,
|
||||
content: TemplateContent,
|
||||
// Developer-role messages may provide message-local tools in the same shape
|
||||
// as top-level request tools.
|
||||
tools: Option<Vec<TemplateTool>>,
|
||||
// Reasoning-capable HF templates are inconsistent on the exact field name,
|
||||
// so expose both variants for compatibility.
|
||||
reasoning: Option<String>,
|
||||
reasoning_content: Option<String>,
|
||||
// Function-call-capable templates commonly expect assistant tool calls
|
||||
// under this OpenAI-compatible field name.
|
||||
tool_calls: Option<Vec<TemplateToolCall>>,
|
||||
// Tool-role messages refer back to the assistant call they are answering.
|
||||
tool_call_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Chat content in the two shapes HF templates commonly expect.
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum TemplateContent {
|
||||
String(String),
|
||||
OpenAi(Vec<TemplateContentPart>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum TemplateContentPart {
|
||||
Text { text: String },
|
||||
Image,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateToolCall {
|
||||
id: String,
|
||||
r#type: &'static str, // always "function"
|
||||
function: TemplateToolFunction,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateToolFunction {
|
||||
name: String,
|
||||
arguments: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct TemplateTool {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: &'static str,
|
||||
function: TemplateToolDefinition,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateToolDefinition {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
parameters: Value,
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
/// Convert chat messages into the JSON shape expected by Jinja chat templates.
|
||||
fn to_template_messages(
|
||||
messages: &[ChatMessage],
|
||||
content_format: ChatTemplateContentFormat,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<Vec<TemplateMessage>> {
|
||||
messages
|
||||
.iter()
|
||||
.map(|message| to_template_message(message, content_format, multimodal))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_template_message(
|
||||
message: &ChatMessage,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<TemplateMessage> {
|
||||
Ok(match message {
|
||||
ChatMessage::System { content } => TemplateMessage {
|
||||
role: "system",
|
||||
content: to_template_content(content, content_format, multimodal)?,
|
||||
tools: None,
|
||||
reasoning: None,
|
||||
reasoning_content: None,
|
||||
tool_calls: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
ChatMessage::Developer { content, tools } => TemplateMessage {
|
||||
role: "developer",
|
||||
content: to_template_content(content, content_format, multimodal)?,
|
||||
tools: tools.as_deref().map(to_template_tools),
|
||||
reasoning: None,
|
||||
reasoning_content: None,
|
||||
tool_calls: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
ChatMessage::User { content } => TemplateMessage {
|
||||
role: "user",
|
||||
content: to_template_content(content, content_format, multimodal)?,
|
||||
tools: None,
|
||||
reasoning: None,
|
||||
reasoning_content: None,
|
||||
tool_calls: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
ChatMessage::Assistant { content } => {
|
||||
let text = content.text();
|
||||
let reasoning = content.reasoning();
|
||||
let tool_calls = to_template_tool_calls(content)?;
|
||||
let content =
|
||||
to_template_content(&ChatContent::Text(text), content_format, multimodal)?;
|
||||
TemplateMessage {
|
||||
role: "assistant",
|
||||
content,
|
||||
tools: None,
|
||||
reasoning: reasoning.clone(),
|
||||
reasoning_content: reasoning,
|
||||
tool_calls,
|
||||
tool_call_id: None,
|
||||
}
|
||||
}
|
||||
ChatMessage::ToolResponse {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => TemplateMessage {
|
||||
role: "tool",
|
||||
content: to_template_content(content, content_format, multimodal)?,
|
||||
tools: None,
|
||||
reasoning: None,
|
||||
reasoning_content: None,
|
||||
tool_calls: None,
|
||||
tool_call_id: Some(tool_call_id.clone()),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn to_template_tool_calls(
|
||||
content: &[AssistantContentBlock],
|
||||
) -> Result<Option<Vec<TemplateToolCall>>> {
|
||||
let mut tool_calls = Vec::new();
|
||||
|
||||
for tool_call in content.tool_calls() {
|
||||
let arguments = serde_json::from_str::<Value>(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call `{}` has invalid JSON arguments: {}",
|
||||
tool_call.id,
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
|
||||
tool_calls.push(TemplateToolCall {
|
||||
id: tool_call.id.clone(),
|
||||
r#type: "function",
|
||||
function: TemplateToolFunction {
|
||||
name: tool_call.name.clone(),
|
||||
arguments,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Ok((!tool_calls.is_empty()).then_some(tool_calls))
|
||||
}
|
||||
|
||||
fn to_template_content(
|
||||
content: &ChatContent,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<TemplateContent> {
|
||||
Ok(match content_format {
|
||||
ChatTemplateContentFormat::String => {
|
||||
TemplateContent::String(to_template_string_content(content, multimodal)?)
|
||||
}
|
||||
ChatTemplateContentFormat::OpenAi => {
|
||||
TemplateContent::OpenAi(to_template_openai_content(content, multimodal)?)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn to_template_openai_content(
|
||||
content: &ChatContent,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<Vec<TemplateContentPart>> {
|
||||
match content {
|
||||
ChatContent::Text(text) => Ok(vec![TemplateContentPart::Text { text: text.clone() }]),
|
||||
ChatContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.map(|part| match part {
|
||||
ChatContentPart::Text { text } => {
|
||||
Ok(TemplateContentPart::Text { text: text.clone() })
|
||||
}
|
||||
// All multimodal contents are normalized to `{ "type": <modality> }`.
|
||||
ChatContentPart::ImageUrl { .. } => {
|
||||
multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
|
||||
Ok(TemplateContentPart::Image)
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_template_string_content(
|
||||
content: &ChatContent,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<String> {
|
||||
match content {
|
||||
ChatContent::Text(text) => Ok(text.clone()),
|
||||
ChatContent::Parts(parts) => {
|
||||
let mut out = String::new();
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => out.push_str(text),
|
||||
ChatContentPart::ImageUrl { .. } => {
|
||||
let multimodal =
|
||||
multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
|
||||
out.push_str(&multimodal.placeholder_token);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_template_tools(tools: &[ChatTool]) -> Vec<TemplateTool> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| TemplateTool {
|
||||
tool_type: "function",
|
||||
function: TemplateToolDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: tool.parameters.clone(),
|
||||
strict: tool.strict,
|
||||
},
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use expect_test::expect;
|
||||
use serde_json::Value;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
|
||||
|
||||
use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::request::{
|
||||
ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
|
||||
GenerationPromptMode, ReasoningEffort,
|
||||
};
|
||||
use crate::{AssistantContentBlock, ChatRenderer, Error, Result};
|
||||
|
||||
const QWEN3_0_6B_TEMPLATE: &str = include_str!("../../../tests/templates/qwen3.jinja");
|
||||
const QWEN3_5_0_8B_TEMPLATE: &str = include_str!("../../../tests/templates/qwen35.jinja");
|
||||
|
||||
fn sample_request(messages: Vec<ChatMessage>) -> ChatRequest {
|
||||
ChatRequest {
|
||||
messages,
|
||||
request_id: "render-test".to_string(),
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
fn render(template: Option<&str>, request: &ChatRequest) -> Result<String> {
|
||||
HfChatRenderer::new(
|
||||
template.map(str::to_owned),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)?
|
||||
.render(request)?
|
||||
.prompt
|
||||
.into_text()
|
||||
.map_err(|_| unreachable!("HF renderer should return text prompt"))
|
||||
}
|
||||
|
||||
fn render_mm(
|
||||
template: &str,
|
||||
request: &ChatRequest,
|
||||
content_format: ChatTemplateContentFormatOption,
|
||||
) -> Result<crate::RenderedPrompt> {
|
||||
HfChatRenderer::new(Some(template.to_string()), HashMap::new(), content_format)?
|
||||
.with_multimodal(Some(MultimodalRenderInfo {
|
||||
placeholder_token: "<image>".to_string(),
|
||||
}))
|
||||
.render(request)
|
||||
}
|
||||
|
||||
fn image_request() -> ChatRequest {
|
||||
sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("a"),
|
||||
ChatContentPart::image_url("data:image/png;base64,test"),
|
||||
ChatContentPart::text("b"),
|
||||
])])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_content_format_replaces_image_with_placeholder_text() {
|
||||
let rendered = render_mm(
|
||||
"{{ messages[0].content }}",
|
||||
&image_request(),
|
||||
ChatTemplateContentFormatOption::String,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered.prompt, Prompt::Text("a<image>b".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_content_format_normalizes_image_url_for_template() {
|
||||
let rendered = render_mm(
|
||||
"{% for item in messages[0].content %}{% if item.type == 'image' %}<|image_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
|
||||
&image_request(),
|
||||
ChatTemplateContentFormatOption::OpenAi,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered.prompt, Prompt::Text("a<|image_pad|>b".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_supports_pycompat_templates() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "<think>hello")]);
|
||||
|
||||
let rendered = render(
|
||||
Some(
|
||||
"{% for message in messages %}{% if message.content.startswith('<think>') %}think{% else %}plain{% endif %}{% endfor %}",
|
||||
),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "think");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_passes_continue_final_message_to_template() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(
|
||||
ChatRole::Assistant,
|
||||
"The capital of",
|
||||
)]);
|
||||
|
||||
assert_eq!(
|
||||
render(
|
||||
Some("{% if continue_final_message %}continue{% else %}new{% endif %}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap(),
|
||||
"new"
|
||||
);
|
||||
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::ContinueFinalAssistant;
|
||||
|
||||
assert_eq!(
|
||||
render(
|
||||
Some("{% if continue_final_message %}continue{% else %}new{% endif %}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap(),
|
||||
"continue"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_flattens_text_parts_for_string_templates() {
|
||||
let request = sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("hello"),
|
||||
ChatContentPart::text(" world"),
|
||||
])]);
|
||||
|
||||
let rendered = render(Some("{{ messages[0].content }}"), &request).unwrap();
|
||||
|
||||
assert_eq!(rendered, "hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_exposes_developer_tools() {
|
||||
let request = sample_request(vec![ChatMessage::developer(
|
||||
"policy",
|
||||
Some(vec![ChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}),
|
||||
strict: Some(true),
|
||||
}]),
|
||||
)]);
|
||||
|
||||
let rendered = render(
|
||||
Some("{{ messages[0].role }}|{{ messages[0].content }}|{{ messages[0].tools[0].function.name }}|{{ messages[0].tools[0].function.parameters.required[0] }}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "developer|policy|get_weather|city");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_keeps_string_text_for_openai_detected_templates() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
|
||||
let rendered = render(
|
||||
Some(
|
||||
"{%- for message in messages %}{%- if message.content is string %}{%- set content = message.content %}{{ content }}{%- endif %}{%- endfor %}",
|
||||
),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_emits_openai_text_blocks_for_structured_templates() {
|
||||
let request = sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("hello"),
|
||||
ChatContentPart::text("world"),
|
||||
])]);
|
||||
|
||||
let rendered = render(
|
||||
Some(
|
||||
"{%- for message in messages %}{%- for item in message.content %}{{ item.text }}|{%- endfor %}{%- endfor %}",
|
||||
),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "hello|world|");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_per_request_override() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
|
||||
// Default template renders one way.
|
||||
let default_rendered = render(Some("{{ messages[0].content }}"), &request).unwrap();
|
||||
assert_eq!(default_rendered, "hello");
|
||||
|
||||
// Per-request override replaces the default template entirely.
|
||||
request.chat_options.chat_template = Some("override:{{ messages[0].content }}".to_string());
|
||||
let overridden = render(Some("{{ messages[0].content }}"), &request).unwrap();
|
||||
assert_eq!(overridden, "override:hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_per_request_override_without_default_template() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.chat_options.chat_template = Some("override:{{ messages[0].content }}".to_string());
|
||||
|
||||
let rendered = render(None, &request).unwrap();
|
||||
|
||||
assert_eq!(rendered, "override:hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_requires_a_template() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
let error = render(None, &request).unwrap_err();
|
||||
|
||||
assert!(matches!(error, Error::MissingChatTemplate));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_injects_special_tokens_into_context() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
let special_tokens = HfSpecialTokens {
|
||||
bos_token: Some(NamedSpecialToken::Text("<bos>".to_string())),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let rendered = HfChatRenderer::new(
|
||||
Some("{{ bos_token }}|{{ bos_token is defined }}".to_string()),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap()
|
||||
.with_special_tokens(Some(special_tokens))
|
||||
.apply_chat_template(&request)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered.prompt, Prompt::Text("<bos>|true".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_exposes_assistant_reasoning_separately() {
|
||||
let request = sample_request(vec![ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "inner".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "outer".to_string(),
|
||||
},
|
||||
])]);
|
||||
|
||||
let rendered = render(
|
||||
Some("{{ messages[0].reasoning_content }}|{{ messages[0].content }}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "inner|outer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_forces_string_content_format_when_configured() {
|
||||
let request = sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("hello"),
|
||||
ChatContentPart::text(" world"),
|
||||
])]);
|
||||
|
||||
let rendered = HfChatRenderer::new(
|
||||
Some(
|
||||
"{%- if messages[0].content is string -%}{{ messages[0].content }}{%- else -%}{%- for item in messages[0].content %}{{ item.text }}|{%- endfor -%}{%- endif -%}".to_string(),
|
||||
),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::String,
|
||||
)
|
||||
.unwrap()
|
||||
.render(&request)
|
||||
.unwrap()
|
||||
.prompt;
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("hello world".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_forces_openai_content_format_when_configured() {
|
||||
let request = sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("hello"),
|
||||
ChatContentPart::text(" world"),
|
||||
])]);
|
||||
|
||||
let rendered = HfChatRenderer::new(
|
||||
Some("{{ messages[0].content[0].text }}{{ messages[0].content[1].text }}".to_string()),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::OpenAi,
|
||||
)
|
||||
.unwrap()
|
||||
.render(&request)
|
||||
.unwrap()
|
||||
.prompt;
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("hello world".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_merges_default_template_kwargs_before_request_kwargs() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let renderer = HfChatRenderer::new(
|
||||
Some("{{ enable_thinking }}|{{ default_only }}".to_string()),
|
||||
HashMap::from([
|
||||
("enable_thinking".to_string(), Value::Bool(false)),
|
||||
("default_only".to_string(), Value::String("x".to_string())),
|
||||
]),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt;
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("true|x".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_reasoning_effort_overrides_template_kwargs() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
|
||||
request.chat_options.template_kwargs.insert(
|
||||
"reasoning_effort".to_string(),
|
||||
Value::String("low".to_string()),
|
||||
);
|
||||
|
||||
let renderer = HfChatRenderer::new(
|
||||
Some("{{ reasoning_effort }}".to_string()),
|
||||
HashMap::from([(
|
||||
"reasoning_effort".to_string(),
|
||||
Value::String("medium".to_string()),
|
||||
)]),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt;
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("max".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_template_omits_reasoning_for_historical_assistant_messages() {
|
||||
let request = sample_request(vec![
|
||||
ChatMessage::text(
|
||||
ChatRole::User,
|
||||
"Hi. Tell me about the capital of France in short",
|
||||
),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "\nOkay, the user is asking... I think that's all.\n".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "Paris is the capital of France.".to_string(),
|
||||
},
|
||||
]),
|
||||
ChatMessage::text(ChatRole::User, "Tell me about Paris more."),
|
||||
]);
|
||||
|
||||
let rendered = render(Some(QWEN3_0_6B_TEMPLATE), &request).unwrap();
|
||||
|
||||
expect![[r#"
|
||||
<|im_start|>user
|
||||
Hi. Tell me about the capital of France in short<|im_end|>
|
||||
<|im_start|>assistant
|
||||
Paris is the capital of France.<|im_end|>
|
||||
<|im_start|>user
|
||||
Tell me about Paris more.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_template_keeps_reasoning_after_the_last_user_query() {
|
||||
let mut request = sample_request(vec![
|
||||
ChatMessage::text(ChatRole::User, "What is 1 + 1?"),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "need simple arithmetic".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "2".to_string(),
|
||||
},
|
||||
]),
|
||||
]);
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
|
||||
let rendered = render(Some(QWEN3_0_6B_TEMPLATE), &request).unwrap();
|
||||
|
||||
expect![[r#"
|
||||
<|im_start|>user
|
||||
What is 1 + 1?<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
need simple arithmetic
|
||||
</think>
|
||||
|
||||
2<|im_end|>
|
||||
"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_exposes_tools_to_templates_when_auto_enabled() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.tools = vec![ChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}),
|
||||
strict: None,
|
||||
}];
|
||||
request.tool_choice = ChatToolChoice::Auto;
|
||||
|
||||
let rendered = render(
|
||||
Some("{{ tools[0].function.name }}|{{ tools[0].function.parameters.required[0] }}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "get_weather|city");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_exposes_assistant_tool_calls_and_tool_messages() {
|
||||
let request = sample_request(vec![
|
||||
ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(
|
||||
crate::AssistantToolCall {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
arguments: r#"{"city":"Paris"}"#.to_string(),
|
||||
},
|
||||
)]),
|
||||
ChatMessage::tool_response("Sunny", "call_1"),
|
||||
]);
|
||||
|
||||
let rendered = render(
|
||||
Some(
|
||||
"{{ messages[0].tool_calls[0].function.name }}|{{ messages[0].tool_calls[0].function.arguments.city }}|{{ messages[1].tool_call_id }}|{{ messages[1].content }}",
|
||||
),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "get_weather|Paris|call_1|Sunny");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen35_template_renders_prefilled_reasoning_start_when_thinking_enabled() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let rendered = render(Some(QWEN3_5_0_8B_TEMPLATE), &request).unwrap();
|
||||
|
||||
expect![[r#"
|
||||
<|im_start|>user
|
||||
hello<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen35_template_renders_closed_empty_reasoning_span_when_thinking_disabled() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(false));
|
||||
|
||||
let rendered = render(Some(QWEN3_5_0_8B_TEMPLATE), &request).unwrap();
|
||||
|
||||
expect![[r#"
|
||||
<|im_start|>user
|
||||
hello<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen35_template_omits_assistant_reasoning_prefill_without_generation_prompt() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let rendered = render(Some(QWEN3_5_0_8B_TEMPLATE), &request).unwrap();
|
||||
|
||||
expect![[r#"
|
||||
<|im_start|>user
|
||||
hello<|im_end|>
|
||||
"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
//! Chat template support for tokenizers using Jinja2 templates.
|
||||
//!
|
||||
//! This module is inlined from SMG's tokenizer crate with local adaptations:
|
||||
//! - thinking-related detection/state is removed
|
||||
//! - special tokens are wired to `vllm_text::backends::hf::HfSpecialTokens`
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use minijinja::Environment;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{self};
|
||||
use vllm_text::backend::hf::HfSpecialTokens;
|
||||
|
||||
use super::error::TemplateError;
|
||||
use super::format::{
|
||||
ChatTemplateContentFormat, ChatTemplateContentFormatOption, detect_chat_template_content_format,
|
||||
};
|
||||
use super::tojson::hf_tojson_filter;
|
||||
use crate::renderer::hf::{TemplateMessage, TemplateTool};
|
||||
use crate::request::ReasoningEffort;
|
||||
|
||||
type Result<T> = std::result::Result<T, TemplateError>;
|
||||
|
||||
/// Build a pre-configured environment with the given template string.
|
||||
fn build_environment(template: String) -> Result<Environment<'static>> {
|
||||
let mut env = Environment::new();
|
||||
|
||||
env.set_trim_blocks(true);
|
||||
env.set_lstrip_blocks(true);
|
||||
|
||||
env.add_template_owned("chat".to_owned(), template)?;
|
||||
|
||||
env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
|
||||
env.add_filter("tojson", hf_tojson_filter);
|
||||
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Default, Serialize)]
|
||||
pub(super) struct TemplateContext<'a> {
|
||||
pub(super) messages: &'a [TemplateMessage],
|
||||
pub(super) add_generation_prompt: bool,
|
||||
pub(super) continue_final_message: bool,
|
||||
pub(super) tools: Option<&'a [TemplateTool]>,
|
||||
pub(super) documents: Option<&'a [serde_json::Value]>,
|
||||
#[serde(flatten)]
|
||||
pub(super) special_tokens: Option<&'a HfSpecialTokens>,
|
||||
#[serde(flatten)]
|
||||
pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
|
||||
// By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any
|
||||
// `reasoning_effort` value that might be present there.
|
||||
pub(super) reasoning_effort: Option<ReasoningEffort>,
|
||||
}
|
||||
|
||||
/// Load chat template from a file (`.jinja` or `.json` containing Jinja).
|
||||
pub fn load_chat_template(template_path: &Path) -> Result<Option<String>> {
|
||||
let content = fs::read_to_string(template_path).map_err(TemplateError::ReadTemplateFile)?;
|
||||
|
||||
if template_path.extension().is_some_and(|ext| ext == "json") {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum ChatTemplateFile {
|
||||
String(String),
|
||||
Object { chat_template: String },
|
||||
}
|
||||
|
||||
let json_value =
|
||||
serde_json::from_str(&content).map_err(TemplateError::ParseTemplateJson)?;
|
||||
let json_template =
|
||||
serde_json::from_value(json_value).map_err(|_| TemplateError::InvalidTemplateJson)?;
|
||||
|
||||
return Ok(Some(match json_template {
|
||||
ChatTemplateFile::String(template) => template,
|
||||
ChatTemplateFile::Object { chat_template } => chat_template,
|
||||
}));
|
||||
}
|
||||
|
||||
let template = content.trim().replace("\\n", "\n");
|
||||
Ok(Some(template))
|
||||
}
|
||||
|
||||
/// Resolve a configured chat template value into a template string.
|
||||
pub fn resolve_chat_template(chat_template: &str) -> Result<String> {
|
||||
let path = Path::new(chat_template);
|
||||
if path.exists() {
|
||||
return load_chat_template(path).map(|template| template.unwrap_or_default());
|
||||
}
|
||||
|
||||
const JINJA_CHARS: [char; 3] = ['{', '}', '\n'];
|
||||
if chat_template.chars().any(|c| JINJA_CHARS.contains(&c)) {
|
||||
return Ok(chat_template.to_string());
|
||||
}
|
||||
|
||||
Err(TemplateError::MissingTemplatePath)
|
||||
}
|
||||
|
||||
/// One compiled chat template with its Jinja environment and detected content
|
||||
/// format.
|
||||
pub(super) struct CompiledChatTemplate {
|
||||
/// Cached, fully-configured environment for one compiled template.
|
||||
env: Environment<'static>,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
}
|
||||
|
||||
impl CompiledChatTemplate {
|
||||
/// Compile the given chat template string into a [`CompiledChatTemplate`].
|
||||
pub fn new(template: String, content_format: ChatTemplateContentFormatOption) -> Result<Self> {
|
||||
let content_format = match content_format {
|
||||
ChatTemplateContentFormatOption::Auto => detect_chat_template_content_format(&template),
|
||||
ChatTemplateContentFormatOption::String => ChatTemplateContentFormat::String,
|
||||
ChatTemplateContentFormatOption::OpenAi => ChatTemplateContentFormat::OpenAi,
|
||||
};
|
||||
let env = build_environment(template)?;
|
||||
Ok(Self {
|
||||
env,
|
||||
content_format,
|
||||
})
|
||||
}
|
||||
|
||||
/// Apply the compiled template to the given context and return the rendered
|
||||
/// prompt.
|
||||
pub fn apply(&self, ctx: TemplateContext<'_>) -> Result<String> {
|
||||
let tmpl = self.env.get_template("chat")?;
|
||||
tmpl.render(ctx).map_err(TemplateError::from)
|
||||
}
|
||||
|
||||
pub fn content_format(&self) -> ChatTemplateContentFormat {
|
||||
self.content_format
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_chat_template_state_valid_template() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{{ messages }}".to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(template.content_format(), ChatTemplateContentFormat::String);
|
||||
let result = template.apply(TemplateContext::default()).unwrap();
|
||||
assert_eq!(result, "[]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_template_state_invalid_template() {
|
||||
let result = CompiledChatTemplate::new(
|
||||
"{% invalid".to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
let err = result.err().unwrap().to_string();
|
||||
assert!(
|
||||
err.contains("failed to render jinja template"),
|
||||
"Error should explain parse failure, got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_special_tokens_injected_into_context() {
|
||||
let template = "{{ bos_token }}hello{{ eos_token }}";
|
||||
let template =
|
||||
CompiledChatTemplate::new(template.to_string(), ChatTemplateContentFormatOption::Auto)
|
||||
.unwrap();
|
||||
|
||||
let special_tokens = HfSpecialTokens {
|
||||
bos_token: Some(NamedSpecialToken::Text("<s>".to_string())),
|
||||
eos_token: Some(NamedSpecialToken::Text("</s>".to_string())),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = template
|
||||
.apply(TemplateContext {
|
||||
special_tokens: Some(&special_tokens),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, "<s>hello</s>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_special_tokens_undefined_when_not_provided() {
|
||||
let template = "{% if bos_token is defined %}{{ bos_token }}{% endif %}hello";
|
||||
let template =
|
||||
CompiledChatTemplate::new(template.to_string(), ChatTemplateContentFormatOption::Auto)
|
||||
.unwrap();
|
||||
|
||||
let result = template.apply(TemplateContext::default()).unwrap();
|
||||
assert_eq!(result, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_special_tokens_partial() {
|
||||
let template =
|
||||
"{{ bos_token }}hello{% if eos_token is defined %}{{ eos_token }}{% endif %}";
|
||||
let template =
|
||||
CompiledChatTemplate::new(template.to_string(), ChatTemplateContentFormatOption::Auto)
|
||||
.unwrap();
|
||||
|
||||
let special_tokens = HfSpecialTokens {
|
||||
bos_token: Some(NamedSpecialToken::Text("<s>".to_string())),
|
||||
eos_token: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = template
|
||||
.apply(TemplateContext {
|
||||
special_tokens: Some(&special_tokens),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, "<s>hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tojson_filter_supports_indent_and_sort_keys() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{{ payload | tojson(indent=2, sort_keys=true) }}".to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
let mut kwargs = HashMap::new();
|
||||
kwargs.insert("payload".to_string(), serde_json::json!({"b": 1, "a": 2}));
|
||||
|
||||
let result = template
|
||||
.apply(TemplateContext {
|
||||
template_kwargs: Some(&kwargs),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, "{\n \"a\": 2,\n \"b\": 1\n}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_chat_template_from_file_jinja() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("chat_template.jinja");
|
||||
fs::write(&path, "{{ messages }}").unwrap();
|
||||
|
||||
let template = load_chat_template(&path).unwrap();
|
||||
|
||||
assert_eq!(template.as_deref(), Some("{{ messages }}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_chat_template_from_inline_literal() {
|
||||
let template = resolve_chat_template("{{ messages }}").unwrap();
|
||||
|
||||
assert_eq!(template, "{{ messages }}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_chat_template_from_existing_file() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("chat_template.jinja");
|
||||
fs::write(&path, "{{ messages }}").unwrap();
|
||||
|
||||
let template = resolve_chat_template(path.to_str().unwrap()).unwrap();
|
||||
|
||||
assert_eq!(template, "{{ messages }}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_chat_template_rejects_missing_path_like_value() {
|
||||
let error = resolve_chat_template("missing_template.jinja").unwrap_err();
|
||||
|
||||
assert!(matches!(error, TemplateError::MissingTemplatePath));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_template_state_respects_explicit_content_format_override() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{% for item in messages[0].content %}{{ item.text }}{% endfor %}".to_string(),
|
||||
ChatTemplateContentFormatOption::String,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(template.content_format(), ChatTemplateContentFormat::String);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_chat_template_from_file_json_string() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("chat_template.json");
|
||||
fs::write(&path, "\"{{ messages }}\"").unwrap();
|
||||
|
||||
let template = load_chat_template(&path).unwrap();
|
||||
|
||||
assert_eq!(template.as_deref(), Some("{{ messages }}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_chat_template_from_file_json_object() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("chat_template.json");
|
||||
fs::write(&path, r#"{"chat_template":"{{ messages }}"}"#).unwrap();
|
||||
|
||||
let template = load_chat_template(&path).unwrap();
|
||||
|
||||
assert_eq!(template.as_deref(), Some("{{ messages }}"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
use minijinja::value::{Kwargs, ViaDeserialize};
|
||||
use minijinja::{Error as MinijinjaError, ErrorKind, Value};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{self, Value as JsonValue};
|
||||
use serde_json_fmt::{JsonFormat, JsonSyntaxError};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
/// Hugging Face-compatible `tojson` filter for chat templates.
|
||||
///
|
||||
/// We cannot use MiniJinja's built-in filter directly because HF relies on
|
||||
/// Python `json.dumps` semantics:
|
||||
/// - no HTML escaping
|
||||
/// - extra kwargs such as `ensure_ascii`, `separators`, and `sort_keys`
|
||||
/// - Python-style `indent` handling
|
||||
pub(super) fn hf_tojson_filter(
|
||||
value: Value,
|
||||
kwargs: Kwargs,
|
||||
) -> std::result::Result<Value, MinijinjaError> {
|
||||
let ensure_ascii = kwargs.get::<Option<bool>>("ensure_ascii")?.unwrap_or(false);
|
||||
let indent = parse_indent(
|
||||
kwargs.get::<Option<ViaDeserialize<IndentArg>>>("indent")?.map(|value| value.0),
|
||||
);
|
||||
let separators = parse_separators(
|
||||
kwargs
|
||||
.get::<Option<ViaDeserialize<SeparatorsArg>>>("separators")?
|
||||
.map(|value| value.0),
|
||||
indent.is_some(),
|
||||
);
|
||||
let sort_keys = kwargs.get::<Option<bool>>("sort_keys")?.unwrap_or(false);
|
||||
|
||||
kwargs.assert_all_used()?;
|
||||
|
||||
let json_value: serde_json::Value = serde_json::to_value(&value).map_err(|e| {
|
||||
MinijinjaError::new(
|
||||
ErrorKind::InvalidOperation,
|
||||
format!("Failed to convert to JSON value: {e}"),
|
||||
)
|
||||
})?;
|
||||
|
||||
let json_str = {
|
||||
let value_to_serialize = if sort_keys {
|
||||
&sort_json_keys(&json_value)
|
||||
} else {
|
||||
&json_value
|
||||
};
|
||||
|
||||
build_json_format(indent, separators.0, separators.1, ensure_ascii)?
|
||||
.format_to_string(value_to_serialize)
|
||||
.map_err(|e| {
|
||||
MinijinjaError::new(
|
||||
ErrorKind::InvalidOperation,
|
||||
format!("Failed to serialize JSON: {}", e.as_report()),
|
||||
)
|
||||
})?
|
||||
};
|
||||
|
||||
Ok(Value::from_safe_string(json_str))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum IndentArg {
|
||||
// Python `json.dumps` accepts bool, int, and string indentation styles.
|
||||
Bool(bool),
|
||||
Integer(i64),
|
||||
String(String),
|
||||
}
|
||||
|
||||
fn parse_indent(value: Option<IndentArg>) -> Option<String> {
|
||||
match value? {
|
||||
IndentArg::Bool(indent) => Some(if indent {
|
||||
" ".to_owned()
|
||||
} else {
|
||||
String::new()
|
||||
}),
|
||||
IndentArg::Integer(indent) => Some(if indent > 0 {
|
||||
" ".repeat(indent as usize)
|
||||
} else {
|
||||
String::new()
|
||||
}),
|
||||
IndentArg::String(indent) => Some(indent),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SeparatorsArg((String, String));
|
||||
|
||||
fn parse_separators(value: Option<SeparatorsArg>, pretty: bool) -> (String, String) {
|
||||
let Some(SeparatorsArg((item_separator, key_separator))) = value else {
|
||||
let default_item_separator = if pretty { "," } else { ", " };
|
||||
let default_key_separator = ": ";
|
||||
|
||||
return (
|
||||
default_item_separator.to_owned(),
|
||||
default_key_separator.to_owned(),
|
||||
);
|
||||
};
|
||||
|
||||
(item_separator, key_separator)
|
||||
}
|
||||
|
||||
fn build_json_format(
|
||||
indent: Option<String>,
|
||||
item_separator: String,
|
||||
key_separator: String,
|
||||
ensure_ascii: bool,
|
||||
) -> std::result::Result<JsonFormat, MinijinjaError> {
|
||||
JsonFormat::new()
|
||||
.indent(indent)
|
||||
.map_err(map_json_syntax_error("indent"))?
|
||||
.comma(item_separator)
|
||||
.map_err(map_json_syntax_error("separators (item)"))?
|
||||
.colon(key_separator)
|
||||
.map_err(map_json_syntax_error("separators (key)"))
|
||||
.map(|format| format.ascii(ensure_ascii))
|
||||
}
|
||||
|
||||
fn map_json_syntax_error(
|
||||
field: &'static str,
|
||||
) -> impl FnOnce(JsonSyntaxError) -> MinijinjaError + Copy {
|
||||
move |error| {
|
||||
MinijinjaError::new(
|
||||
ErrorKind::InvalidOperation,
|
||||
format!("invalid {field} value for tojson: {error}"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursively sort all object keys in a JSON value.
|
||||
fn sort_json_keys(value: &JsonValue) -> JsonValue {
|
||||
match value {
|
||||
JsonValue::Object(map) => {
|
||||
let mut sorted: serde_json::Map<String, JsonValue> = serde_json::Map::new();
|
||||
let mut keys: Vec<_> = map.keys().collect();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
sorted.insert(key.clone(), sort_json_keys(&map[key]));
|
||||
}
|
||||
JsonValue::Object(sorted)
|
||||
}
|
||||
JsonValue::Array(arr) => JsonValue::Array(arr.iter().map(sort_json_keys).collect()),
|
||||
_ => value.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use expect_test::expect;
|
||||
use minijinja::Environment;
|
||||
use serde_json::json;
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::hf_tojson_filter;
|
||||
|
||||
fn render(template: &str, payload: serde_json::Value) -> String {
|
||||
let mut env = Environment::new();
|
||||
env.add_filter("tojson", hf_tojson_filter);
|
||||
env.render_str(template, json!({ "payload": payload })).unwrap()
|
||||
}
|
||||
|
||||
fn render_error(template: &str, payload: serde_json::Value) -> minijinja::Error {
|
||||
let mut env = Environment::new();
|
||||
env.add_filter("tojson", hf_tojson_filter);
|
||||
env.render_str(template, json!({ "payload": payload })).unwrap_err()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_does_not_html_escape_like_minijinja_builtin() {
|
||||
let rendered = render("{{ payload|tojson }}", json!("<tag>&'"));
|
||||
assert_eq!(rendered, "\"<tag>&'\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_sort_keys_recursively() {
|
||||
let rendered = render(
|
||||
"{{ payload|tojson(sort_keys=true) }}",
|
||||
json!({
|
||||
"z": {"b": 1, "a": 2},
|
||||
"a": 0
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(rendered, "{\"a\": 0, \"z\": {\"a\": 2, \"b\": 1}}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_indent() {
|
||||
let rendered = render("{{ payload|tojson(indent=2) }}", json!([1, 2]));
|
||||
|
||||
assert_eq!(rendered, "[\n 1,\n 2\n]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_ensure_ascii_false() {
|
||||
let rendered = render("{{ payload|tojson(ensure_ascii=false) }}", json!("中文"));
|
||||
assert_eq!(rendered, "\"中文\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_ensure_ascii_true() {
|
||||
let rendered = render("{{ payload|tojson(ensure_ascii=true) }}", json!("中文"));
|
||||
assert_eq!(rendered, "\"\\u4e2d\\u6587\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_separators() {
|
||||
let rendered = render(
|
||||
"{{ payload|tojson(separators=[',', ':']) }}",
|
||||
json!({
|
||||
"x": [1, 2]
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(rendered, "{\"x\":[1,2]}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_negative_indent_as_newline_only() {
|
||||
let rendered = render("{{ payload|tojson(indent=-1) }}", json!([1, 2]));
|
||||
assert_eq!(rendered, "[\n1,\n2\n]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_string_indent() {
|
||||
let rendered = render("{{ payload|tojson(indent=' ') }}", json!([1, 2]));
|
||||
assert_eq!(rendered, "[\n 1,\n 2\n]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_supports_boolean_indent() {
|
||||
let rendered_true = render("{{ payload|tojson(indent=true) }}", json!([1, 2]));
|
||||
assert_eq!(rendered_true, "[\n 1,\n 2\n]");
|
||||
|
||||
let rendered_false = render("{{ payload|tojson(indent=false) }}", json!([1, 2]));
|
||||
assert_eq!(rendered_false, "[\n1,\n2\n]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_combines_indent_sort_keys_separators_and_ensure_ascii() {
|
||||
let rendered = render(
|
||||
"{{ payload|tojson(ensure_ascii=true, sort_keys=true, separators=[',', ':'], indent=' ') }}",
|
||||
json!({
|
||||
"b": "<中>",
|
||||
"a": [1, 2]
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
rendered,
|
||||
"{\n \"a\":[\n 1,\n 2\n ],\n \"b\":\"<\\u4e2d>\"\n}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_rejects_invalid_indent() {
|
||||
let error = render_error("{{ payload|tojson(indent='-->') }}", json!({"a": 1}));
|
||||
expect!["invalid operation: invalid indent value for tojson: string contains unexpected character '-' (in <string>:1)"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_rejects_invalid_separator_shape() {
|
||||
let error = render_error("{{ payload|tojson(separators=':,') }}", json!({"a": 1}));
|
||||
expect!["cannot deserialize: invalid type: string \":,\", expected a tuple of size 2 (in <string>:1)"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_rejects_invalid_key_separator() {
|
||||
let error = render_error(
|
||||
"{{ payload|tojson(separators=[',', '=>']) }}",
|
||||
json!({"a": 1}),
|
||||
);
|
||||
expect!["invalid operation: invalid separators (key) value for tojson: string contains unexpected character '=' (in <string>:1)"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
pub mod hf;
|
||||
mod selection;
|
||||
|
||||
pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RenderedPrompt {
|
||||
pub prompt: Prompt,
|
||||
}
|
||||
|
||||
/// Minimal chat-prompt renderer used by `vllm-chat`.
|
||||
pub trait ChatRenderer: Send + Sync {
|
||||
/// Render one chat request into the text prompt submitted to the text
|
||||
/// backend.
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt>;
|
||||
}
|
||||
|
||||
/// Shared trait-object form of [`ChatRenderer`].
|
||||
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
|
||||
@@ -0,0 +1,109 @@
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde_with::DeserializeFromStr;
|
||||
|
||||
/// Specify which chat renderer implementation to use.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
pub enum RendererSelection {
|
||||
/// Use model-based auto-detection.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Force the generic Hugging Face chat-template renderer.
|
||||
Hf,
|
||||
/// Force the DeepSeek V3.2 renderer.
|
||||
DeepSeekV32,
|
||||
/// Force the DeepSeek V4 renderer.
|
||||
DeepSeekV4,
|
||||
}
|
||||
|
||||
impl RendererSelection {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
|
||||
/// Resolve the renderer selection using the given model type string, if
|
||||
/// it's `Auto`.
|
||||
pub fn resolve(self, model_type: &str) -> Self {
|
||||
match self {
|
||||
Self::Auto => match model_type {
|
||||
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
_ => Self::Hf,
|
||||
},
|
||||
selection => selection,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for RendererSelection {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case(Self::HF_LITERAL) {
|
||||
Ok(Self::Hf)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V32_LITERAL) {
|
||||
Ok(Self::DeepSeekV32)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
|
||||
Ok(Self::DeepSeekV4)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown renderer `{value}` (expected one of: auto, hf, deepseek_v32, deepseek_v4)"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for RendererSelection {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Auto => f.write_str(Self::AUTO_LITERAL),
|
||||
Self::Hf => f.write_str(Self::HF_LITERAL),
|
||||
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RendererSelection;
|
||||
|
||||
#[test]
|
||||
fn renderer_selection_parses_known_values() {
|
||||
assert_eq!(
|
||||
"auto".parse::<RendererSelection>().unwrap(),
|
||||
RendererSelection::Auto
|
||||
);
|
||||
assert_eq!(
|
||||
"hf".parse::<RendererSelection>().unwrap(),
|
||||
RendererSelection::Hf
|
||||
);
|
||||
assert_eq!(
|
||||
"deepseek_v32".parse::<RendererSelection>().unwrap(),
|
||||
RendererSelection::DeepSeekV32
|
||||
);
|
||||
assert_eq!(
|
||||
"deepseek_v4".parse::<RendererSelection>().unwrap(),
|
||||
RendererSelection::DeepSeekV4
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renderer_selection_display_round_trips() {
|
||||
for selection in [
|
||||
RendererSelection::Auto,
|
||||
RendererSelection::Hf,
|
||||
RendererSelection::DeepSeekV32,
|
||||
RendererSelection::DeepSeekV4,
|
||||
] {
|
||||
assert_eq!(
|
||||
selection.to_string().parse::<RendererSelection>().unwrap(),
|
||||
selection
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
pub use vllm_text::SamplingParams;
|
||||
use vllm_text::TextDecodeOptions;
|
||||
pub use vllm_tool_parser::Tool as ChatTool;
|
||||
|
||||
use crate::AssistantMessageExt;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::{AssistantContentBlock, AssistantMessage};
|
||||
|
||||
/// Role label for one text-only chat message.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ChatRole {
|
||||
System,
|
||||
Developer,
|
||||
User,
|
||||
Assistant,
|
||||
ToolResponse,
|
||||
}
|
||||
|
||||
/// One text-only chat content part in OpenAI-style block format.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ChatContentPart {
|
||||
/// One plain-text content block.
|
||||
Text { text: String },
|
||||
/// One image URL/data URL content block.
|
||||
ImageUrl {
|
||||
image_url: String,
|
||||
detail: Option<ImageDetail>,
|
||||
uuid: Option<String>,
|
||||
},
|
||||
// ImageData...
|
||||
// ImageEmbeds...
|
||||
}
|
||||
|
||||
impl ChatContentPart {
|
||||
/// Construct one text content part with plain string content.
|
||||
pub fn text(text: impl Into<String>) -> Self {
|
||||
Self::Text { text: text.into() }
|
||||
}
|
||||
|
||||
/// Construct one image URL content part with the given URL string.
|
||||
pub fn image_url(image_url: impl Into<String>) -> Self {
|
||||
Self::ImageUrl {
|
||||
image_url: image_url.into(),
|
||||
detail: None,
|
||||
uuid: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the text content of this part when it's a text block, or an
|
||||
/// "unsupported multimodal content" error otherwise.
|
||||
pub(crate) fn as_text(&self) -> Result<&str> {
|
||||
match self {
|
||||
Self::Text { text } => Ok(text),
|
||||
Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this part is a text block with empty content.
|
||||
pub(crate) fn is_empty_text(&self) -> bool {
|
||||
matches!(self, Self::Text { text } if text.is_empty())
|
||||
}
|
||||
|
||||
/// Return whether this part contains any multimodal content.
|
||||
pub(crate) fn is_multimodal(&self) -> bool {
|
||||
match self {
|
||||
Self::Text { .. } => false,
|
||||
Self::ImageUrl { .. } => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Text-only chat content.
|
||||
///
|
||||
/// This supports either a simple string or an OpenAI-style list of text blocks.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatContent {
|
||||
/// Simple text content.
|
||||
Text(String),
|
||||
/// OpenAI-style blocks.
|
||||
Parts(Vec<ChatContentPart>),
|
||||
}
|
||||
|
||||
impl ChatContent {
|
||||
/// Flatten the text content into one plain string without adding
|
||||
/// separators.
|
||||
// TODO: this method will be truly fallible once we add non-text content parts.
|
||||
pub fn try_flatten_to_text(&self) -> Result<String> {
|
||||
Ok(match self {
|
||||
Self::Text(text) => text.clone(),
|
||||
Self::Parts(parts) => {
|
||||
parts.iter().map(ChatContentPart::as_text).collect::<Result<Vec<_>>>()?.concat()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Return whether there's no text content or only empty text blocks.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Text(text) => text.is_empty(),
|
||||
Self::Parts(parts) => parts.iter().all(ChatContentPart::is_empty_text),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this content contains any multimodal parts.
|
||||
pub fn has_multimodal(&self) -> bool {
|
||||
match self {
|
||||
Self::Text(_) => false,
|
||||
Self::Parts(parts) => parts.iter().any(ChatContentPart::is_multimodal),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for ChatContent {
|
||||
fn from(value: String) -> Self {
|
||||
Self::Text(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for ChatContent {
|
||||
fn from(value: &str) -> Self {
|
||||
Self::Text(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<ChatContentPart>> for ChatContent {
|
||||
fn from(value: Vec<ChatContentPart>) -> Self {
|
||||
Self::Parts(value)
|
||||
}
|
||||
}
|
||||
|
||||
/// One chat message.
|
||||
///
|
||||
/// Original Python API reference:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/chat_utils.py#L309-L333>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "role", rename_all = "snake_case")]
|
||||
pub enum ChatMessage {
|
||||
/// System message content.
|
||||
System { content: ChatContent },
|
||||
/// Developer message content plus optional message-local tools.
|
||||
Developer {
|
||||
content: ChatContent,
|
||||
tools: Option<Vec<ChatTool>>,
|
||||
},
|
||||
/// User message content.
|
||||
User { content: ChatContent },
|
||||
/// Assistant history content assembled from structured assistant blocks.
|
||||
Assistant { content: Vec<AssistantContentBlock> },
|
||||
/// Tool response content associated with one prior assistant tool call.
|
||||
ToolResponse {
|
||||
content: ChatContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ChatMessage {
|
||||
/// Construct one chat message with plain string content.
|
||||
pub fn text(role: ChatRole, text: impl Into<String>) -> Self {
|
||||
let content: String = text.into();
|
||||
|
||||
match role {
|
||||
ChatRole::System => Self::system(content),
|
||||
ChatRole::Developer => Self::developer(content, None),
|
||||
ChatRole::User => Self::user(content),
|
||||
ChatRole::Assistant => Self::assistant_text(content),
|
||||
ChatRole::ToolResponse => {
|
||||
panic!(
|
||||
"tool response messages require a tool_call_id; \
|
||||
use ChatMessage::tool_response() instead"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with system role.
|
||||
pub fn system(content: impl Into<ChatContent>) -> Self {
|
||||
Self::System {
|
||||
content: content.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with developer role.
|
||||
pub fn developer(content: impl Into<ChatContent>, tools: Option<Vec<ChatTool>>) -> Self {
|
||||
Self::Developer {
|
||||
content: content.into(),
|
||||
tools,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with user role.
|
||||
pub fn user(content: impl Into<ChatContent>) -> Self {
|
||||
Self::User {
|
||||
content: content.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with assistant role and plain string content.
|
||||
pub fn assistant_text(text: impl Into<String>) -> Self {
|
||||
Self::Assistant {
|
||||
content: vec![AssistantContentBlock::Text { text: text.into() }],
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with assistant role and structured content
|
||||
/// blocks.
|
||||
pub fn assistant_blocks(content: Vec<AssistantContentBlock>) -> Self {
|
||||
Self::Assistant { content }
|
||||
}
|
||||
|
||||
/// Construct one tool-role message.
|
||||
pub fn tool_response(content: impl Into<ChatContent>, tool_call_id: impl Into<String>) -> Self {
|
||||
Self::ToolResponse {
|
||||
content: content.into(),
|
||||
tool_call_id: tool_call_id.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the chat role of this message.
|
||||
pub fn role(&self) -> ChatRole {
|
||||
match self {
|
||||
Self::System { .. } => ChatRole::System,
|
||||
Self::Developer { .. } => ChatRole::Developer,
|
||||
Self::User { .. } => ChatRole::User,
|
||||
Self::Assistant { .. } => ChatRole::Assistant,
|
||||
Self::ToolResponse { .. } => ChatRole::ToolResponse,
|
||||
}
|
||||
}
|
||||
|
||||
/// Concatenate the visible text carried by this message.
|
||||
pub fn text_content(&self) -> Result<String> {
|
||||
match self {
|
||||
Self::System { content }
|
||||
| Self::Developer { content, .. }
|
||||
| Self::User { content }
|
||||
| Self::ToolResponse { content, .. } => content.try_flatten_to_text(),
|
||||
Self::Assistant { content } => Ok(content.text()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Concatenate assistant reasoning text when present.
|
||||
pub fn reasoning_content(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Assistant { content } => content.reasoning(),
|
||||
Self::System { .. }
|
||||
| Self::Developer { .. }
|
||||
| Self::User { .. }
|
||||
| Self::ToolResponse { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this message contains any multimodal content.
|
||||
pub fn has_multimodal(&self) -> bool {
|
||||
match self {
|
||||
Self::System { content }
|
||||
| Self::Developer { content, .. }
|
||||
| Self::User { content }
|
||||
| Self::ToolResponse { content, .. } => content.has_multimodal(),
|
||||
Self::Assistant { .. } => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AssistantMessage> for ChatMessage {
|
||||
fn from(value: AssistantMessage) -> Self {
|
||||
Self::Assistant {
|
||||
content: value.content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Controls how prompt rendering should end after the existing chat history.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum GenerationPromptMode {
|
||||
/// Append a generation prompt for a new assistant turn.
|
||||
///
|
||||
/// Equivalent to `add_generation_prompt = true` and `continue_final_message
|
||||
/// = false`.
|
||||
#[default]
|
||||
StartNewAssistant,
|
||||
/// Leave the final assistant message open so generation continues it.
|
||||
///
|
||||
/// Equivalent to `add_generation_prompt = false` and
|
||||
/// `continue_final_message = true`.
|
||||
ContinueFinalAssistant,
|
||||
/// Render the existing chat history without adding any trailing generation
|
||||
/// prompt.
|
||||
///
|
||||
/// Equivalent to `add_generation_prompt = false` and
|
||||
/// `continue_final_message = false`.
|
||||
NoGenerationPrompt,
|
||||
}
|
||||
|
||||
/// Effort level for reasoning models.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ReasoningEffort {
|
||||
None,
|
||||
Minimal,
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
XHigh,
|
||||
Max,
|
||||
}
|
||||
|
||||
impl ReasoningEffort {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::None => "none",
|
||||
Self::Minimal => "minimal",
|
||||
Self::Low => "low",
|
||||
Self::Medium => "medium",
|
||||
Self::High => "high",
|
||||
Self::XHigh => "xhigh",
|
||||
Self::Max => "max",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Chat-template-related request options.
|
||||
///
|
||||
/// These are the small subset of chat controls that currently affect prompt
|
||||
/// rendering in `vllm-chat`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ChatOptions {
|
||||
/// Controls whether rendering starts a new assistant turn, continues the
|
||||
/// final assistant message, or emits no trailing generation prompt at
|
||||
/// all.
|
||||
pub generation_prompt_mode: GenerationPromptMode,
|
||||
|
||||
/// Per-request Jinja chat template override. When set, this template is
|
||||
/// used instead of the model's default chat template.
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Effort level exposed to chat templates for reasoning models.
|
||||
pub reasoning_effort: Option<ReasoningEffort>,
|
||||
|
||||
/// Additional keyword arguments exposed to the chat template.
|
||||
pub template_kwargs: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl Default for ChatOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
generation_prompt_mode: GenerationPromptMode::StartNewAssistant,
|
||||
chat_template: None,
|
||||
reasoning_effort: None,
|
||||
template_kwargs: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatOptions {
|
||||
/// Whether to add a generation prompt for a new assistant turn after the
|
||||
/// existing chat history.
|
||||
pub fn add_generation_prompt(&self) -> bool {
|
||||
matches!(
|
||||
self.generation_prompt_mode,
|
||||
GenerationPromptMode::StartNewAssistant
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether to leave the final assistant message open so generation
|
||||
/// continues it.
|
||||
pub fn continue_final_message(&self) -> bool {
|
||||
matches!(
|
||||
self.generation_prompt_mode,
|
||||
GenerationPromptMode::ContinueFinalAssistant
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool-choice semantics supported by `vllm-chat`.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ChatToolChoice {
|
||||
Auto,
|
||||
#[default]
|
||||
None,
|
||||
}
|
||||
|
||||
/// One chat request ready to be rendered into a prompt and lowered into a
|
||||
/// generate request.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ChatRequest {
|
||||
/// Stable caller-supplied request ID.
|
||||
pub request_id: String,
|
||||
/// Ordered chat history to render.
|
||||
pub messages: Vec<ChatMessage>,
|
||||
/// User-facing sampling parameters accepted by `vllm-chat`.
|
||||
pub sampling_params: SamplingParams,
|
||||
/// Chat-specific rendering options.
|
||||
pub chat_options: ChatOptions,
|
||||
/// Function tools made available to the model for this request.
|
||||
pub tools: Vec<ChatTool>,
|
||||
/// Tool-choice behavior for this request.
|
||||
pub tool_choice: ChatToolChoice,
|
||||
/// Text decode options for incremental detokenization.
|
||||
pub decode_options: TextDecodeOptions,
|
||||
/// Whether to emit intermediate northbound content deltas before the
|
||||
/// terminal result.
|
||||
///
|
||||
/// If `false`, callers only observe the terminal accumulated assistant
|
||||
/// output. If `true`, callers may receive zero or more incremental
|
||||
/// content events before the final terminal one.
|
||||
pub intermediate: bool,
|
||||
/// Request scheduling priority (lower means earlier handling; default 0).
|
||||
pub priority: i32,
|
||||
/// Documents for RAG (retrieval-augmented generation), passed to the chat
|
||||
/// template.
|
||||
pub documents: Option<Vec<Value>>,
|
||||
/// Salt for prefix cache isolation in multi-user environments.
|
||||
pub cache_salt: Option<String>,
|
||||
/// Whether to add special tokens (e.g. BOS) during prompt tokenization.
|
||||
pub add_special_tokens: bool,
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
}
|
||||
|
||||
impl ChatRequest {
|
||||
/// Return one minimal valid request fixture for tests.
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
request_id: "test-request".to_string(),
|
||||
messages: vec![ChatMessage::text(ChatRole::User, "test")],
|
||||
sampling_params: SamplingParams::default(),
|
||||
chat_options: ChatOptions::default(),
|
||||
tools: Vec::new(),
|
||||
tool_choice: ChatToolChoice::None,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: true,
|
||||
priority: 0,
|
||||
documents: None,
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate basic request invariants before rendering.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.messages.is_empty() {
|
||||
return Err(Error::EmptyMessages);
|
||||
}
|
||||
match (
|
||||
self.chat_options.generation_prompt_mode,
|
||||
self.messages.last().map(ChatMessage::role),
|
||||
) {
|
||||
(GenerationPromptMode::ContinueFinalAssistant, Some(ChatRole::Assistant)) => {}
|
||||
(GenerationPromptMode::ContinueFinalAssistant, _) => {
|
||||
return Err(Error::ContinueFinalAssistantWithoutFinalAssistant);
|
||||
}
|
||||
(GenerationPromptMode::NoGenerationPrompt, _)
|
||||
| (GenerationPromptMode::StartNewAssistant, _) => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return true if this request contains any multimodal content in its
|
||||
/// messages.
|
||||
pub fn has_multimodal(&self) -> bool {
|
||||
self.messages.iter().any(ChatMessage::has_multimodal)
|
||||
}
|
||||
|
||||
/// Return true if this request should enable tool parsing based on the tool
|
||||
/// choice and tool list.
|
||||
pub(crate) fn tool_parsing_enabled(&self) -> bool {
|
||||
matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty()
|
||||
}
|
||||
|
||||
/// Return the request-level thinking toggle when explicitly requested.
|
||||
///
|
||||
/// We currently accept the two request kwargs `thinking` and
|
||||
/// `enable_thinking`. Both must be booleans when present. If both are
|
||||
/// present, they must have the same value. If neither key is provided,
|
||||
/// return `None`.
|
||||
pub(crate) fn enable_thinking(&self) -> Result<Option<bool>> {
|
||||
let thinking = self.parse_template_bool("thinking")?;
|
||||
let enable_thinking = self.parse_template_bool("enable_thinking")?;
|
||||
|
||||
match (thinking, enable_thinking) {
|
||||
(None, None) => Ok(None),
|
||||
(Some(thinking), Some(enable_thinking)) if thinking != enable_thinking => {
|
||||
Err(Error::ChatTemplate(
|
||||
"template kwargs `thinking` and `enable_thinking` must match when both are set"
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
(Some(thinking), _) => Ok(Some(thinking)),
|
||||
(None, Some(enable_thinking)) => Ok(Some(enable_thinking)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_template_bool(&self, key: &str) -> Result<Option<bool>> {
|
||||
match self.chat_options.template_kwargs.get(key) {
|
||||
None => Ok(None),
|
||||
Some(Value::Bool(value)) => Ok(Some(*value)),
|
||||
Some(other) => Err(Error::ChatTemplate(format!(
|
||||
"template kwarg `{key}` must be a boolean, got {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRole {
|
||||
/// Return the chat-template role string used by the current text-only chat
|
||||
/// backend.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::System => "system",
|
||||
Self::Developer => "developer",
|
||||
Self::User => "user",
|
||||
Self::Assistant => "assistant",
|
||||
Self::ToolResponse => "tool_response",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::{json, to_value};
|
||||
|
||||
use super::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool};
|
||||
use crate::Error;
|
||||
use crate::event::AssistantContentBlock;
|
||||
|
||||
#[test]
|
||||
fn chat_content_deserializes_from_raw_string() {
|
||||
let content: ChatContent = serde_json::from_value(json!("hello")).unwrap();
|
||||
assert_eq!(content, ChatContent::Text("hello".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_content_deserializes_from_openai_text_blocks() {
|
||||
let content: ChatContent =
|
||||
serde_json::from_value(json!([{ "type": "text", "text": "hello" }])).unwrap();
|
||||
assert_eq!(
|
||||
content,
|
||||
ChatContent::Parts(vec![ChatContentPart::text("hello")])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_content_from_string_like_values_builds_text() {
|
||||
assert_eq!(
|
||||
ChatContent::from("hello"),
|
||||
ChatContent::Text("hello".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
ChatContent::from("hello".to_string()),
|
||||
ChatContent::Text("hello".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_content_try_flattens_text_parts_without_separators() {
|
||||
let content = ChatContent::Parts(vec![
|
||||
ChatContentPart::text("hello"),
|
||||
ChatContentPart::text(" world"),
|
||||
]);
|
||||
assert_eq!(content.try_flatten_to_text().unwrap(), "hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_message_collects_visible_and_reasoning_text() {
|
||||
let message = ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "inner".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "outer".to_string(),
|
||||
},
|
||||
]);
|
||||
|
||||
assert_eq!(message.role(), ChatRole::Assistant);
|
||||
assert_eq!(message.text_content().unwrap(), "outer");
|
||||
assert_eq!(message.reasoning_content().as_deref(), Some("inner"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn developer_message_round_trips_through_serde() {
|
||||
let message = ChatMessage::developer(
|
||||
"hello",
|
||||
Some(vec![ChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: Some(true),
|
||||
}]),
|
||||
);
|
||||
|
||||
let value = to_value(&message).unwrap();
|
||||
let decoded: ChatMessage = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(decoded, message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enable_thinking_is_none_when_no_kwargs_are_present() {
|
||||
let request = ChatRequest::for_test();
|
||||
assert_eq!(request.enable_thinking().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enable_thinking_accepts_matching_duplicate_kwargs() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request.chat_options.template_kwargs.insert("thinking".to_string(), json!(true));
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), json!(true));
|
||||
|
||||
assert_eq!(request.enable_thinking().unwrap(), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enable_thinking_rejects_non_boolean_kwargs() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), json!("yes"));
|
||||
|
||||
assert!(matches!(
|
||||
request.enable_thinking(),
|
||||
Err(Error::ChatTemplate(message))
|
||||
if message.contains("`thinking` must be a boolean")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enable_thinking_rejects_conflicting_duplicate_kwargs() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), json!(false));
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), json!(true));
|
||||
|
||||
assert!(matches!(
|
||||
request.enable_thinking(),
|
||||
Err(Error::ChatTemplate(message))
|
||||
if message.contains("`thinking` and `enable_thinking` must match")
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::Stream;
|
||||
use trait_set::trait_set;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs};
|
||||
|
||||
use crate::FinishReason;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent};
|
||||
|
||||
/// Final structured assistant message plus terminal stream metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CollectedAssistantMessage {
|
||||
pub message: AssistantMessage,
|
||||
pub prompt_token_count: usize,
|
||||
pub prompt_token_ids: Arc<[u32]>,
|
||||
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub output_token_count: usize,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Per-request stream of chat events.
|
||||
pub struct ChatEventStream {
|
||||
request_id: String,
|
||||
inner: Pin<Box<dyn Stream<Item = Result<ChatEvent>> + Send>>,
|
||||
}
|
||||
|
||||
impl ChatEventStream {
|
||||
pub(crate) fn new(request_id: String, inner: impl crate::output::ChatEventStream) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
inner: Box::pin(inner),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the request ID associated with this stream.
|
||||
pub fn request_id(&self) -> &str {
|
||||
&self.request_id
|
||||
}
|
||||
|
||||
/// Collect the stream to completion and return the final assembled
|
||||
/// assistant message.
|
||||
pub async fn collect_message(mut self) -> Result<CollectedAssistantMessage> {
|
||||
use futures::StreamExt as _;
|
||||
|
||||
let mut message = AssistantMessage::default();
|
||||
let mut prompt_logprobs = None;
|
||||
let mut prompt_token_ids: Arc<[u32]> = Arc::from([]);
|
||||
let mut logprob_positions: Vec<DecodedPositionLogprobs> = Vec::new();
|
||||
let mut token_ids: Vec<u32> = Vec::new();
|
||||
while let Some(event) = self.next().await.transpose()? {
|
||||
match event {
|
||||
ChatEvent::Start {
|
||||
prompt_logprobs: start_prompt_logprobs,
|
||||
prompt_token_ids: start_prompt_token_ids,
|
||||
} => {
|
||||
prompt_logprobs = start_prompt_logprobs;
|
||||
prompt_token_ids = start_prompt_token_ids;
|
||||
}
|
||||
ChatEvent::BlockEnd { block, .. } => message.push_block(block),
|
||||
ChatEvent::LogprobsDelta {
|
||||
logprobs,
|
||||
token_ids: delta_ids,
|
||||
} => {
|
||||
if let Some(logprobs) = logprobs {
|
||||
logprob_positions.extend(logprobs.positions);
|
||||
}
|
||||
token_ids.extend(delta_ids);
|
||||
}
|
||||
ChatEvent::Done {
|
||||
message: done,
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
return Ok(CollectedAssistantMessage {
|
||||
message: done,
|
||||
prompt_token_count,
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
logprobs: (!logprob_positions.is_empty()).then_some(DecodedLogprobs {
|
||||
positions: logprob_positions,
|
||||
}),
|
||||
token_ids,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
});
|
||||
}
|
||||
ChatEvent::ToolCallEnd { call, .. } => {
|
||||
message.push_block(AssistantContentBlock::ToolCall(call));
|
||||
}
|
||||
ChatEvent::BlockStart { .. }
|
||||
| ChatEvent::BlockDelta { .. }
|
||||
| ChatEvent::ToolCallStart { .. }
|
||||
| ChatEvent::ToolCallArgumentsDelta { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Note: this is actually unreachable, as the underlying stream always emit an
|
||||
// error on unexpected close.
|
||||
Err(Error::StreamClosedBeforeTerminalOutput {
|
||||
request_id: self.request_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for ChatEventStream {
|
||||
type Item = Result<ChatEvent>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
Pin::new(&mut self.inner).poll_next(cx)
|
||||
}
|
||||
}
|
||||
|
||||
trait_set! {
|
||||
pub trait ChatEventStreamTrait = Stream<Item = Result<ChatEvent>> + Send + 'static;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use futures::stream;
|
||||
use vllm_llm::FinishReason;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
|
||||
};
|
||||
|
||||
use super::{ChatEventStream, CollectedAssistantMessage};
|
||||
use crate::error::Error;
|
||||
use crate::event::ChatEvent;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_message_requires_terminal_done_event() {
|
||||
let stream = ChatEventStream::new(
|
||||
"chat-missing-done".to_string(),
|
||||
stream::iter([Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
})]),
|
||||
);
|
||||
|
||||
let error = stream.collect_message().await.expect_err("missing done");
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::StreamClosedBeforeTerminalOutput { request_id }
|
||||
if request_id == "chat-missing-done"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_message_retains_prompt_and_sample_logprobs() {
|
||||
let stream = ChatEventStream::new(
|
||||
"chat-logprobs".to_string(),
|
||||
stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![10, 11].into(),
|
||||
prompt_logprobs: Some(DecodedPromptLogprobs {
|
||||
first_token_id: 0,
|
||||
first_token: "o".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "p".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
let collected = stream.collect_message().await.unwrap();
|
||||
assert_eq!(
|
||||
collected,
|
||||
CollectedAssistantMessage {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 2,
|
||||
prompt_token_ids: vec![10, 11].into(),
|
||||
prompt_logprobs: Some(DecodedPromptLogprobs {
|
||||
first_token_id: 0,
|
||||
first_token: "o".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "p".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
output_token_count: 1,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+89
@@ -0,0 +1,89 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- messages[0].content + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- for message in messages %}
|
||||
{%- if message.content is string %}
|
||||
{%- set content = message.content %}
|
||||
{%- else %}
|
||||
{%- set content = '' %}
|
||||
{%- endif %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- if '</think>' in content %}
|
||||
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if loop.index0 > ns.last_query_index %}
|
||||
{%- if loop.last or (not loop.last and reasoning_content) %}
|
||||
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
||||
{%- if content is string %}
|
||||
{{- content }}
|
||||
{%- elif content is iterable and content is not mapping %}
|
||||
{%- for item in content %}
|
||||
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain images.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
||||
{%- elif 'video' in item or item.type == 'video' %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain videos.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
||||
{%- elif 'text' in item %}
|
||||
{{- item.text }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected item type in content.') }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif content is none or content is undefined %}
|
||||
{{- '' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected content type.') }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
{%- if not messages %}
|
||||
{{- raise_exception('No messages provided.') }}
|
||||
{%- endif %}
|
||||
{%- if tools and tools is iterable and tools is not mapping %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
||||
{%- if content %}
|
||||
{{- '\n\n' + content }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
||||
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" %}
|
||||
{%- set content = render_content(message.content, false)|trim %}
|
||||
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if ns.multi_step_tool %}
|
||||
{{- raise_exception('No user query found in messages.') }}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- set content = render_content(message.content, true)|trim %}
|
||||
{%- if message.role == "system" %}
|
||||
{%- if not loop.first %}
|
||||
{{- raise_exception('System message must be at the beginning.') }}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- if '</think>' in content %}
|
||||
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = reasoning_content|trim %}
|
||||
{%- if loop.index0 > ns.last_query_index %}
|
||||
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{%- if loop.first %}
|
||||
{%- if content|trim %}
|
||||
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- else %}
|
||||
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value }}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected message role.') }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if enable_thinking is defined and enable_thinking is true %}
|
||||
{{- '<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,6 @@
|
||||
# vLLM Example Templates
|
||||
|
||||
These fixtures are copied from `vllm/examples/`.
|
||||
|
||||
They are currently used by `src/chat/src/renderers/hf/format.rs` tests to keep
|
||||
our chat-template content format detection aligned with Python vLLM behavior.
|
||||
@@ -0,0 +1,29 @@
|
||||
{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}
|
||||
|
||||
{% for message in messages %}
|
||||
{% if message['role'] == 'user' %}
|
||||
### Instruction:
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
|
||||
{% endif %}
|
||||
{% elif message['role'] == 'assistant' %}
|
||||
### Response:
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
|
||||
{% endif %}
|
||||
{% elif message['role'] == 'user_context' %}
|
||||
### Input:
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}
|
||||
### Response:
|
||||
{% endif %}
|
||||
@@ -0,0 +1,13 @@
|
||||
{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}
|
||||
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{- '<reserved_106>' + message['content'] -}}
|
||||
{%- elif message['role'] == 'assistant' -%}
|
||||
{{- '<reserved_107>' + message['content'] -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- '<reserved_107>' -}}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,18 @@
|
||||
{%- set counter = namespace(index=0) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{- '[Round ' + counter.index|string + ']\n问:' + message['content'] -}}
|
||||
{%- set counter.index = counter.index + 1 -%}
|
||||
{%- endif -%}
|
||||
{%- if message['role'] == 'assistant' -%}
|
||||
{{- '\n答:' + message['content'] -}}
|
||||
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
|
||||
{{- '\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- '\n答:' -}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,18 @@
|
||||
{%- set counter = namespace(index=1) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{- '[Round ' + counter.index|string + ']\n\n问:' + message['content'] -}}
|
||||
{%- set counter.index = counter.index + 1 -%}
|
||||
{%- endif -%}
|
||||
{%- if message['role'] == 'assistant' -%}
|
||||
{{- '\n\n答:' + message['content'] -}}
|
||||
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
|
||||
{{- '\n\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- '\n\n答:' -}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,2 @@
|
||||
{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %}
|
||||
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\n' }}{% endif %}
|
||||
@@ -0,0 +1,15 @@
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{- 'User: ' + message['content'] -}}
|
||||
{%- elif message['role'] == 'assistant' -%}
|
||||
{{- 'Assistant: ' + message['content'] -}}
|
||||
{%- endif -%}
|
||||
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
|
||||
{{- '\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- 'Assistant:' -}}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,17 @@
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'system' -%}
|
||||
{{- 'System: ' + message['content'] -}}
|
||||
{%- elif message['role'] == 'user' -%}
|
||||
{{- 'User: ' + message['content'] -}}
|
||||
{%- elif message['role'] == 'assistant' -%}
|
||||
{{- 'Falcon: ' + message['content'] -}}
|
||||
{%- endif -%}
|
||||
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
|
||||
{{- '\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- 'Falcon:' -}}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,30 @@
|
||||
<#meta#>
|
||||
- Date: {{ (messages|selectattr('role', 'equalto', 'meta-current_date')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'meta-current_date')|list) else '' }}
|
||||
- Task: {{ (messages|selectattr('role', 'equalto', 'meta-task_name')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'meta-task_name')|list) else '' }}
|
||||
<#system#>
|
||||
{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}
|
||||
<#chat#>
|
||||
{% for message in messages %}
|
||||
{% if message['role'] == 'user' %}
|
||||
<#user#>
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
{% endif %}
|
||||
{% elif message['role'] == 'assistant' %}
|
||||
<#bot#>
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
{% endif %}
|
||||
{% elif message['role'] == 'user_context' %}
|
||||
<#user_context#>
|
||||
{{ message['content']|trim -}}
|
||||
{% if not loop.last %}
|
||||
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}
|
||||
<#bot#>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,12 @@
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'user' %}
|
||||
{{- '<_user>' + message['content']|trim }}
|
||||
{%- elif message['role'] == 'system' %}
|
||||
{{- '<_system>' + message['content']|trim }}
|
||||
{%- elif message['role'] == 'assistant' %}
|
||||
{{- '<_bot>' + message['content'] }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<_bot>' }}
|
||||
{%- endif %}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
{% if not add_generation_prompt is defined %}
|
||||
{% set add_generation_prompt = false %}
|
||||
{% endif %}
|
||||
{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' %}
|
||||
{%- if ns.is_first_sp %}
|
||||
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
|
||||
{% set ns.is_first_sp = false %}
|
||||
{%- else %}
|
||||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor -%}
|
||||
|
||||
{#- Adapted from https://github.com/sgl-project/sglang/blob/main/examples/chat_template/tool_chat_template_deepseekr1.jinja #}
|
||||
{% if tools is defined and tools is not none %}
|
||||
{% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. '
|
||||
'When a tool call is needed, you MUST use the following format to issue the call:\n'
|
||||
'<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>FUNCTION_NAME\n'
|
||||
'```json\n{"param1": "value1", "param2": "value2"}\n```<|tool▁call▁end|><|tool▁calls▁end|>\n\n'
|
||||
'Make sure the JSON is valid.'
|
||||
'## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %}
|
||||
{% for tool in tools %}
|
||||
{% set tool_ns.text = tool_ns.text + '\n```json\n' + (tool | tojson) + '\n```\n' %}
|
||||
{% endfor %}
|
||||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
|
||||
{% endif %}
|
||||
|
||||
{{- bos_token }}
|
||||
{{- ns.system_prompt }}
|
||||
{%- for message in messages %}
|
||||
{% set content = message['content'] %}
|
||||
{%- if message['role'] == 'user' %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- set ns.is_first = false -%}
|
||||
{%- set ns.is_last_user = true -%}
|
||||
{{'<|User|>' + content + '<|Assistant|>'}}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'assistant' %}
|
||||
{% if '</think>' in content %}
|
||||
{% set content = content.split('</think>')[-1] %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- if ns.is_tool %}
|
||||
{{- '<|tool▁outputs▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_first = false %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- set ns.is_output_first = true %}
|
||||
{%- for tool in message['tool_calls'] %}
|
||||
{%- if not ns.is_first %}
|
||||
{%- if content is none %}
|
||||
{{- '<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- else %}
|
||||
{{- content + '<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_first = true -%}
|
||||
{%- else %}
|
||||
{{- '\n' + '<|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- '<|tool▁calls▁end|><|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- if ns.is_tool %}
|
||||
{{- '<|tool▁outputs▁end|>' + content + '<|end▁of▁sentence|>'}}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- else %}
|
||||
{{- content + '<|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'tool' %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- set ns.is_tool = true -%}
|
||||
{%- if ns.is_output_first %}
|
||||
{{- '<|tool▁outputs▁begin|><|tool▁output▁begin|>' + content + '<|tool▁output▁end|>'}}
|
||||
{%- set ns.is_output_first = false %}
|
||||
{%- else %}
|
||||
{{- '\n<|tool▁output▁begin|>' + content + '<|tool▁output▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor -%}
|
||||
{% if ns.is_tool %}
|
||||
{{- '<|tool▁outputs▁end|>'}}
|
||||
{%- endif %}
|
||||
{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
|
||||
{{- '<|Assistant|>'}}
|
||||
{%- endif %}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
{% if not add_generation_prompt is defined %}
|
||||
{% set add_generation_prompt = false %}
|
||||
{% endif %}
|
||||
|
||||
{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' %}
|
||||
{%- if ns.is_first_sp %}
|
||||
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
|
||||
{% set ns.is_first_sp = false %}
|
||||
{%- else %}
|
||||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{{ bos_token }}
|
||||
{{ ns.system_prompt }}
|
||||
{%- if tools %}
|
||||
{{"\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{"\n</tools>\n\n"}}
|
||||
|
||||
{{"For function call returns, you should first print <|tool▁calls▁begin|>"}}
|
||||
|
||||
{{"For each function call, you should return object like:\n" }}
|
||||
{{"<|tool▁call▁begin|>function<|tool▁sep|><function_name>\n```json\n<function_arguments_in_json_format>\n```<|tool▁call▁end|>"}}
|
||||
|
||||
{{"At the end of function call returns, you should print <|tool▁calls▁end|><|end▁of▁sentence|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'user' %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- set ns.is_first = false -%}
|
||||
{%- set ns.is_last_user = true -%}
|
||||
{{'<|User|>' + message['content'] + '<|Assistant|>'}}
|
||||
{%- endif %}
|
||||
|
||||
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- if ns.is_tool %}
|
||||
{{'<|tool▁outputs▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_first = false %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- set ns.is_output_first = true %}
|
||||
|
||||
{%- for tool in message['tool_calls'] %}
|
||||
{%- if not ns.is_first %}
|
||||
{%- if message['content'] is none %}
|
||||
{{'<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- else %}
|
||||
{{message['content'] + '<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_first = true -%}
|
||||
{%- else %}
|
||||
{{'\n' + '<|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{'<|tool▁calls▁end|><|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- if ns.is_tool %}
|
||||
{{'<|tool▁outputs▁end|>' + message['content'] + '<|end▁of▁sentence|>'}}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- else %}
|
||||
{% set content = message['content'] %}
|
||||
{{content + '<|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if message['role'] == 'tool' %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- set ns.is_tool = true -%}
|
||||
{%- if ns.is_output_first %}
|
||||
{{'<|tool▁outputs▁begin|><|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}
|
||||
{%- set ns.is_output_first = false %}
|
||||
{%- else %}
|
||||
{{'\n<|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor -%}
|
||||
|
||||
{% if ns.is_tool %}
|
||||
{{'<|tool▁outputs▁end|>'}}
|
||||
{% endif %}
|
||||
|
||||
{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
|
||||
{{'<|Assistant|>'}}
|
||||
{% endif %}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
{% if not add_generation_prompt is defined %}
|
||||
{% set add_generation_prompt = false %}
|
||||
{% endif %}
|
||||
{% if not thinking is defined %}
|
||||
{% set thinking = false %}
|
||||
{% endif %}
|
||||
{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' %}
|
||||
{%- if ns.is_first_sp %}
|
||||
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
|
||||
{% set ns.is_first_sp = false %}
|
||||
{%- else %}
|
||||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{% if tools is defined and tools is not none %}
|
||||
{% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %}
|
||||
{% for tool in tools %}
|
||||
{% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %}
|
||||
{% endfor %}
|
||||
{% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<|tool▁calls▁begin|><|tool▁call▁begin|>tool_call_name<|tool▁sep|>tool_call_arguments<|tool▁call▁end|>{{additional_tool_calls}}<|tool▁calls▁end|>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %}
|
||||
{% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
|
||||
{% endif %}
|
||||
|
||||
{{ bos_token }}{{ ns.system_prompt }}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'user' %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- set ns.is_first = false -%}
|
||||
{%- set ns.is_last_user = true -%}
|
||||
{{'<|User|>' + message['content']}}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
|
||||
{%- if ns.is_last_user %}
|
||||
{{'<|Assistant|></think>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- set ns.is_first = false %}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- for tool in message['tool_calls'] %}
|
||||
{%- if not ns.is_first %}
|
||||
{%- if message['content'] is none %}
|
||||
{{'<|tool▁calls▁begin|><|tool▁call▁begin|>'+ tool['function']['name'] + '<|tool▁sep|>' + tool['function']['arguments']|tojson + '<|tool▁call▁end|>'}}
|
||||
{%- else %}
|
||||
{{message['content'] + '<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['function']['name'] + '<|tool▁sep|>' + tool['function']['arguments']|tojson + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- set ns.is_first = true -%}
|
||||
{%- else %}
|
||||
{{'<|tool▁call▁begin|>'+ tool['function']['name'] + '<|tool▁sep|>' + tool['function']['arguments']|tojson + '<|tool▁call▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{'<|tool▁calls▁end|><|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %}
|
||||
{%- if ns.is_last_user %}
|
||||
{{'<|Assistant|>'}}
|
||||
{%- if message['prefix'] is defined and message['prefix'] and thinking %}
|
||||
{{'<think>'}}
|
||||
{%- else %}
|
||||
{{'</think>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- if ns.is_tool %}
|
||||
{{message['content'] + '<|end▁of▁sentence|>'}}
|
||||
{%- set ns.is_tool = false -%}
|
||||
{%- else %}
|
||||
{%- set content = message['content'] -%}
|
||||
{%- if '</think>' in content %}
|
||||
{%- set content = content.split('</think>', 1)[1] -%}
|
||||
{%- endif %}
|
||||
{{content + '<|end▁of▁sentence|>'}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if message['role'] == 'tool' %}
|
||||
{%- set ns.is_last_user = false -%}
|
||||
{%- set ns.is_tool = true -%}
|
||||
{{'<|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}
|
||||
{%- endif %}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %}
|
||||
{{'<|Assistant|>'}}
|
||||
{%- if not thinking %}
|
||||
{{'</think>'}}
|
||||
{%- else %}
|
||||
{{'<think>'}}
|
||||
{%- endif %}
|
||||
{% endif %}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
{%- set ns = namespace(developer_content='', has_tools=false) -%}
|
||||
|
||||
{%- if tools is defined and tools | length > 0 -%}
|
||||
{%- set ns.has_tools = true -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- for message in messages -%}
|
||||
{%- if message.role == 'developer' or message.role == 'system' -%}
|
||||
<start_of_turn>user
|
||||
{{ message.content }}
|
||||
{%- if ns.has_tools %}
|
||||
|
||||
Available functions:
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.type == 'function' %}
|
||||
|
||||
Function: {{ tool.function.name }}
|
||||
Description: {{ tool.function.description | default('No description provided') }}
|
||||
Parameters: {{ tool.function.parameters | tojson }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
<end_of_turn>
|
||||
{%- elif message.role == 'user' -%}
|
||||
<start_of_turn>user
|
||||
{{ message.content }}<end_of_turn>
|
||||
{%- elif message.role == 'assistant' -%}
|
||||
{%- if message.tool_calls is defined and message.tool_calls | length > 0 -%}
|
||||
<start_of_turn>model
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
<start_function_call>call:{{ tool_call.function.name }}{
|
||||
{%- set args = tool_call.function.arguments -%}
|
||||
{%- if args is string -%}
|
||||
{%- set args = args | fromjson -%}
|
||||
{%- endif -%}
|
||||
{%- for key, value in args.items() -%}
|
||||
{{ key }}:<escape>{{ value }}<escape>{% if not loop.last %},{% endif %}
|
||||
{%- endfor -%}
|
||||
}<end_function_call>
|
||||
{%- endfor %}
|
||||
<end_of_turn>
|
||||
{%- else -%}
|
||||
<start_of_turn>model
|
||||
{{ message.content }}<end_of_turn>
|
||||
{%- endif -%}
|
||||
{%- elif message.role == 'tool' -%}
|
||||
<start_of_turn>user
|
||||
Function result for {{ message.name | default('function') }}: {{ message.content }}<end_of_turn>
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if add_generation_prompt -%}
|
||||
<start_of_turn>model
|
||||
{%- endif -%}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
{#- Begin-of-sequence token to start the model prompt -#}
|
||||
{{ bos_token }}
|
||||
{#- Extracts the system message. Gemma does not support system messages so it will be prepended to first user message. -#}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{#- Set tools to none if not defined for this ChatCompletion request (helps avoid errors later) -#}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
{#- Validate alternating user/assistant messages (excluding 'tool' messages and ones with tool_calls) -#}
|
||||
{%- for message in loop_messages | rejectattr("role", "equalto", "tool") | selectattr("tool_calls", "undefined") -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{#- Main loop over all messages in the conversation history -#}
|
||||
{%- for message in loop_messages -%}
|
||||
{#- Normalize roles for model prompt formatting -#}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- elif (message['role'] == 'tool') -%}
|
||||
{%- set role = "user" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{#- Mark the start of a message block with the appropriate role -#}
|
||||
{{ '<start_of_turn>' + role + '\n' -}}
|
||||
|
||||
{#- Insert system message content (if present) at the beginning of the first message. -#}
|
||||
{%- if loop.first -%}
|
||||
{{ first_user_prefix }}
|
||||
{#- Append system message with tool information if using tools in message request. -#}
|
||||
{%- if tools is not none -%}
|
||||
{{- "Tools (functions) are available. If you decide to invoke one or more of the tools, you must respond with a python list of the function calls.\n" -}}
|
||||
{{- "Example Format: [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] \n" -}}
|
||||
{{- "Do not use variables. DO NOT USE MARKDOWN SYNTAX. You SHOULD NOT include any other text in the response if you call a function. If none of the functions can be used, point it out. If you lack the parameters required by the function, also point it out.\n" -}}
|
||||
{{- "Here is a list of functions in JSON format that you can invoke.\n" -}}
|
||||
{{- tools | tojson(indent=4) -}}
|
||||
{{- "\n\n" -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
|
||||
{#- Format model tool calls (turns where model indicates they want to call a tool) -#}
|
||||
{%- if 'tool_calls' in message -%}
|
||||
{#- Opening bracket for tool call list. -#}
|
||||
{{- '[' -}}
|
||||
{#- For each tool call -#}
|
||||
{%- for tool_call in message.tool_calls -%}
|
||||
{#- Get tool call function. -#}
|
||||
{%- if tool_call.function is defined -%}
|
||||
{%- set tool_call = tool_call.function -%}
|
||||
{%- endif -%}
|
||||
{#- Function name & opening parenthesis. -#}
|
||||
{{- tool_call.name + '(' -}}
|
||||
|
||||
{#-- Handle arguments as list (positional) or dict (named) --#}
|
||||
{#-- Named arguments (dict) --#}
|
||||
{%- if tool_call.arguments is iterable and tool_call.arguments is mapping -%}
|
||||
{%- set first = true -%}
|
||||
{%- for key, val in tool_call.arguments.items() -%}
|
||||
{%- if not first %}, {% endif -%}
|
||||
{{ key }}={{ val | tojson }}
|
||||
{%- set first = false -%}
|
||||
{%- endfor -%}
|
||||
{#-- Positional arguments (list) --#}
|
||||
{%- elif tool_call.arguments is iterable -%}
|
||||
{{- tool_call.arguments | map('tojson') | join(', ') -}}
|
||||
{#-- Fallback: single positional value --#}
|
||||
{%- else -%}
|
||||
{{- tool_call.arguments | tojson -}}
|
||||
{#-- Closing parenthesis. --#}
|
||||
{%- endif -%}
|
||||
{{- ')' -}}
|
||||
{#-- If more than one tool call, place comma and move to formatting next tool call --#}
|
||||
{%- if not loop.last -%}, {% endif -%}
|
||||
{%- endfor -%}
|
||||
{#- Closing bracket for tool call list. -#}
|
||||
{{- ']' -}}
|
||||
{%- endif -%}
|
||||
|
||||
{#- Tool response start tag (for messages from a tool) -#}
|
||||
{%- if (message['role'] == 'tool') -%}
|
||||
{{ '<tool_response>\n' -}}
|
||||
{%- endif -%}
|
||||
|
||||
{#- Render the message content: handle plain string or multimodal content like image/text -#}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
|
||||
{#- Tool response end tag -#}
|
||||
{%- if (message['role'] == 'tool') -%}
|
||||
{{ '</tool_response>' -}}
|
||||
{%- endif -%}
|
||||
|
||||
{#- Mark end of a single turn -#}
|
||||
{{ '<end_of_turn>\n' }}
|
||||
{%- endfor -%}
|
||||
|
||||
{#- If generation is to be triggered, add model prompt prefix -#}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model\n'}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,331 @@
|
||||
{%- macro format_parameters(properties, required) -%}
|
||||
{%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
|
||||
{%- set ns = namespace(found_first=false) -%}
|
||||
{%- for key, value in properties | dictsort -%}
|
||||
{%- set add_comma = false -%}
|
||||
{%- if key not in standard_keys -%}
|
||||
{%- if ns.found_first %},{% endif -%}
|
||||
{%- set ns.found_first = true -%}
|
||||
{{ key }}:{
|
||||
{%- if value['description'] -%}
|
||||
description:<|"|>{{ value['description'] }}<|"|>
|
||||
{%- set add_comma = true -%}
|
||||
{%- endif -%}
|
||||
{%- if value['nullable'] %}
|
||||
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
|
||||
nullable:true
|
||||
{%- endif -%}
|
||||
{%- if value['type'] | upper == 'STRING' -%}
|
||||
{%- if value['enum'] -%}
|
||||
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
|
||||
enum:{{ format_argument(value['enum']) }}
|
||||
{%- endif -%}
|
||||
{%- elif value['type'] | upper == 'OBJECT' -%}
|
||||
,properties:{
|
||||
{%- if value['properties'] is defined and value['properties'] is mapping -%}
|
||||
{{- format_parameters(value['properties'], value['required'] | default([])) -}}
|
||||
{%- elif value is mapping -%}
|
||||
{{- format_parameters(value, value['required'] | default([])) -}}
|
||||
{%- endif -%}
|
||||
}
|
||||
{%- if value['required'] -%}
|
||||
,required:[
|
||||
{%- for item in value['required'] | default([]) -%}
|
||||
<|"|>{{- item -}}<|"|>
|
||||
{%- if not loop.last %},{% endif -%}
|
||||
{%- endfor -%}
|
||||
]
|
||||
{%- endif -%}
|
||||
{%- elif value['type'] | upper == 'ARRAY' -%}
|
||||
{%- if value['items'] is mapping and value['items'] -%}
|
||||
,items:{
|
||||
{%- set ns_items = namespace(found_first=false) -%}
|
||||
{%- for item_key, item_value in value['items'] | dictsort -%}
|
||||
{%- if item_value is not none -%}
|
||||
{%- if ns_items.found_first %},{% endif -%}
|
||||
{%- set ns_items.found_first = true -%}
|
||||
{%- if item_key == 'properties' -%}
|
||||
properties:{
|
||||
{%- if item_value is mapping -%}
|
||||
{{- format_parameters(item_value, value['items']['required'] | default([])) -}}
|
||||
{%- endif -%}
|
||||
}
|
||||
{%- elif item_key == 'required' -%}
|
||||
required:[
|
||||
{%- for req_item in item_value -%}
|
||||
<|"|>{{- req_item -}}<|"|>
|
||||
{%- if not loop.last %},{% endif -%}
|
||||
{%- endfor -%}
|
||||
]
|
||||
{%- elif item_key == 'type' -%}
|
||||
{%- if item_value is string -%}
|
||||
type:{{ format_argument(item_value | upper) }}
|
||||
{%- else -%}
|
||||
type:{{ format_argument(item_value | map('upper') | list) }}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{{ item_key }}:{{ format_argument(item_value) }}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
|
||||
type:<|"|>{{ value['type'] | upper }}<|"|>}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endmacro -%}
|
||||
{%- macro format_function_declaration(tool_data) -%}
|
||||
declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
|
||||
{%- set params = tool_data['function']['parameters'] -%}
|
||||
{%- if params -%}
|
||||
,parameters:{
|
||||
{%- if params['properties'] -%}
|
||||
properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
|
||||
{%- endif -%}
|
||||
{%- if params['required'] -%}
|
||||
required:[
|
||||
{%- for item in params['required'] -%}
|
||||
<|"|>{{- item -}}<|"|>
|
||||
{{- ',' if not loop.last -}}
|
||||
{%- endfor -%}
|
||||
],
|
||||
{%- endif -%}
|
||||
{%- if params['type'] -%}
|
||||
type:<|"|>{{- params['type'] | upper -}}<|"|>}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- if 'response' in tool_data['function'] -%}
|
||||
{%- set response_declaration = tool_data['function']['response'] -%}
|
||||
,response:{
|
||||
{%- if response_declaration['description'] -%}
|
||||
description:<|"|>{{- response_declaration['description'] -}}<|"|>,
|
||||
{%- endif -%}
|
||||
{%- if response_declaration['type'] | upper == 'OBJECT' -%}
|
||||
type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
}
|
||||
{%- endmacro -%}
|
||||
{%- macro format_argument(argument, escape_keys=True) -%}
|
||||
{%- if argument is string -%}
|
||||
{{- '<|"|>' + argument + '<|"|>' -}}
|
||||
{%- elif argument is boolean -%}
|
||||
{{- 'true' if argument else 'false' -}}
|
||||
{%- elif argument is mapping -%}
|
||||
{{- '{' -}}
|
||||
{%- set ns = namespace(found_first=false) -%}
|
||||
{%- for key, value in argument | dictsort -%}
|
||||
{%- if ns.found_first %},{% endif -%}
|
||||
{%- set ns.found_first = true -%}
|
||||
{%- if escape_keys -%}
|
||||
{{- '<|"|>' + key + '<|"|>' -}}
|
||||
{%- else -%}
|
||||
{{- key -}}
|
||||
{%- endif -%}
|
||||
:{{- format_argument(value, escape_keys=escape_keys) -}}
|
||||
{%- endfor -%}
|
||||
{{- '}' -}}
|
||||
{%- elif argument is sequence -%}
|
||||
{{- '[' -}}
|
||||
{%- for item in argument -%}
|
||||
{{- format_argument(item, escape_keys=escape_keys) -}}
|
||||
{%- if not loop.last %},{% endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ']' -}}
|
||||
{%- else -%}
|
||||
{{- argument -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
{%- macro strip_thinking(text) -%}
|
||||
{%- set ns = namespace(result='') -%}
|
||||
{%- for part in text.split('<channel|>') -%}
|
||||
{%- if '<|channel>' in part -%}
|
||||
{%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
|
||||
{%- else -%}
|
||||
{%- set ns.result = ns.result + part -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.result | trim -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro format_tool_response_block(tool_name, response) -%}
|
||||
{{- '<|tool_response>' -}}
|
||||
{%- if response is mapping -%}
|
||||
{{- 'response:' + tool_name + '{' -}}
|
||||
{%- for key, value in response | dictsort -%}
|
||||
{{- key -}}:{{- format_argument(value, escape_keys=False) -}}
|
||||
{%- if not loop.last %},{% endif -%}
|
||||
{%- endfor -%}
|
||||
{{- '}' -}}
|
||||
{%- else -%}
|
||||
{{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}
|
||||
{%- endif -%}
|
||||
{{- '<tool_response|>' -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- set ns = namespace(prev_message_type=None) -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{{ bos_token }}
|
||||
{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
|
||||
{{- '<|turn>system\n' -}}
|
||||
|
||||
{%- if enable_thinking is defined and enable_thinking -%}
|
||||
{{- '<|think|>' -}}
|
||||
{%- set ns.prev_message_type = 'think' -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if messages[0]['role'] in ['system', 'developer'] -%}
|
||||
{{- messages[0]['content'] | trim -}}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if tools -%}
|
||||
{%- for tool in tools %}
|
||||
{{- '<|tool>' -}}
|
||||
{{- format_function_declaration(tool) | trim -}}
|
||||
{{- '<tool|>' -}}
|
||||
{%- endfor %}
|
||||
{%- set ns.prev_message_type = 'tool' -%}
|
||||
{%- endif -%}
|
||||
|
||||
{{- '<turn|>\n' -}}
|
||||
{%- endif %}
|
||||
|
||||
{%- set ns_turn = namespace(last_user_idx=-1) -%}
|
||||
{%- for i in range(loop_messages | length) -%}
|
||||
{%- if loop_messages[i]['role'] == 'user' -%}
|
||||
{%- set ns_turn.last_user_idx = i -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if message['role'] != 'tool' -%}
|
||||
{%- set ns.prev_message_type = None -%}
|
||||
{%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
|
||||
{#- OpenAI may emit multiple assistant messages in one tool loop (user → asst → tool → asst → tool).
|
||||
Only the first of those should open <|turn>model; later ones continue the same model turn. -#}
|
||||
{%- set prev_nt = namespace(role=None, found=false) -%}
|
||||
{%- if loop.index0 > 0 -%}
|
||||
{%- for j in range(loop.index0 - 1, -1, -1) -%}
|
||||
{%- if not prev_nt.found -%}
|
||||
{%- if loop_messages[j]['role'] != 'tool' -%}
|
||||
{%- set prev_nt.role = loop_messages[j]['role'] -%}
|
||||
{%- set prev_nt.found = true -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}
|
||||
{%- if not continue_same_model_turn -%}
|
||||
{{- '<|turn>' + role + '\n' }}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.get('reasoning') and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}
|
||||
{{- '<|channel>thought\n' + message['reasoning'] + '\n<channel|>'}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message['tool_calls'] -%}
|
||||
{%- for tool_call in message['tool_calls'] -%}
|
||||
{%- set function = tool_call['function'] -%}
|
||||
{{- '<|tool_call>call:' + function['name'] + '{' -}}
|
||||
{%- if function['arguments'] is mapping -%}
|
||||
{%- set ns_args = namespace(found_first=false) -%}
|
||||
{%- for key, value in function['arguments'] | dictsort -%}
|
||||
{%- if ns_args.found_first %},{% endif -%}
|
||||
{%- set ns_args.found_first = true -%}
|
||||
{{- key -}}:{{- format_argument(value, escape_keys=False) -}}
|
||||
{%- endfor -%}
|
||||
{%- elif function['arguments'] is string -%}
|
||||
{{- function['arguments'] -}}
|
||||
{%- endif -%}
|
||||
{{- '}<tool_call|>' -}}
|
||||
{%- endfor -%}
|
||||
{%- set ns.prev_message_type = 'tool_call' -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set ns_tr_out = namespace(flag=false) -%}
|
||||
{%- if message.get('tool_responses') -%}
|
||||
{#- Legacy: tool_responses embedded on the assistant message -#}
|
||||
{%- for tool_response in message['tool_responses'] -%}
|
||||
{{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}
|
||||
{%- set ns_tr_out.flag = true -%}
|
||||
{%- set ns.prev_message_type = 'tool_response' -%}
|
||||
{%- endfor -%}
|
||||
{%- elif message.get('tool_calls') -%}
|
||||
{#- OpenAI Chat Completions: consecutive following messages with role "tool" (no break/continue; range scan) -#}
|
||||
{%- set ns_tool_scan = namespace(stopped=false) -%}
|
||||
{%- for k in range(loop.index0 + 1, loop_messages | length) -%}
|
||||
{%- if ns_tool_scan.stopped -%}
|
||||
{%- elif loop_messages[k]['role'] != 'tool' -%}
|
||||
{%- set ns_tool_scan.stopped = true -%}
|
||||
{%- else -%}
|
||||
{%- set follow = loop_messages[k] -%}
|
||||
{%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}
|
||||
{%- for tc in message['tool_calls'] -%}
|
||||
{%- if tc.get('id') == follow.get('tool_call_id') -%}
|
||||
{%- set ns_tname.name = tc['function']['name'] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set tool_body = follow.get('content') -%}
|
||||
{%- if tool_body is string -%}
|
||||
{{- format_tool_response_block(ns_tname.name, tool_body) -}}
|
||||
{%- elif tool_body is sequence and tool_body is not string -%}
|
||||
{%- set ns_txt = namespace(s='') -%}
|
||||
{%- for part in tool_body -%}
|
||||
{%- if part.get('type') == 'text' -%}
|
||||
{%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}
|
||||
{%- else -%}
|
||||
{{- format_tool_response_block(ns_tname.name, tool_body) -}}
|
||||
{%- endif -%}
|
||||
{%- set ns_tr_out.flag = true -%}
|
||||
{%- set ns.prev_message_type = 'tool_response' -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message['content'] is string -%}
|
||||
{%- if role == 'model' -%}
|
||||
{{- strip_thinking(message['content']) -}}
|
||||
{%- else -%}
|
||||
{{- message['content'] | trim -}}
|
||||
{%- endif -%}
|
||||
{%- elif message['content'] is sequence -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'text' -%}
|
||||
{%- if role == 'model' -%}
|
||||
{{- strip_thinking(item['text']) -}}
|
||||
{%- else -%}
|
||||
{{- item['text'] | trim -}}
|
||||
{%- endif -%}
|
||||
{%- elif item['type'] == 'image' -%}
|
||||
{{- '\n\n<|image|>\n\n' -}}
|
||||
{%- set ns.prev_message_type = 'image' -%}
|
||||
{%- elif item['type'] == 'audio' -%}
|
||||
{{- '<|audio|>' -}}
|
||||
{%- set ns.prev_message_type = 'audio' -%}
|
||||
{%- elif item['type'] == 'video' -%}
|
||||
{{- '\n\n<|video|>\n\n' -}}
|
||||
{%- set ns.prev_message_type = 'video' -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if not (ns_tr_out.flag and not message.get('content')) -%}
|
||||
{{- '<turn|>\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if add_generation_prompt -%}
|
||||
{%- if ns.prev_message_type != 'tool_response' -%}
|
||||
{{- '<|turn>model\n' -}}
|
||||
{%- endif -%}
|
||||
{%- if not enable_thinking | default(false) -%}
|
||||
{{- '<|channel>thought\n<channel|>' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,54 @@
|
||||
{%- set counter = namespace(index=0) -%}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if messages and messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant." %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if tools is not none %}
|
||||
{%- set tool_instruction %}
|
||||
You have access to the following tools. When you need to call a tool, you MUST use the following format:
|
||||
|
||||
<tool_call>function_name
|
||||
<arg_key>parameter_name</arg_key>
|
||||
<arg_value>parameter_value</arg_value>
|
||||
</tool_call>
|
||||
|
||||
Important rules:
|
||||
- Always wrap tool calls with <tool_call>...</tool_call> tags
|
||||
- Put the function name on the first line after <tool_call>
|
||||
- Use <arg_key> and <arg_value> tags for each parameter
|
||||
- If a parameter value is a string, keep it as-is. If it's a number or boolean, convert it appropriately
|
||||
- You can make multiple tool calls if needed
|
||||
- If no tool is suitable, respond with regular text
|
||||
|
||||
Available tools:
|
||||
{% endset %}
|
||||
{{- tool_instruction + "\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{- '[Round ' + counter.index|string + ']\n问:' + message['content'] -}}
|
||||
{%- set counter.index = counter.index + 1 -%}
|
||||
{%- endif -%}
|
||||
{%- if message['role'] == 'assistant' -%}
|
||||
{{- '\n答:' + message['content'] -}}
|
||||
{%- if (loop.last and add_generation_prompt) or not loop.last -%}
|
||||
{{- '\n' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
|
||||
{{- '\n答:' -}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,36 @@
|
||||
{%- if tools %}
|
||||
{{- '<|start_of_role|>available_tools<|end_of_role|>
|
||||
' }}
|
||||
{%- for tool in tools %}
|
||||
{{- tool | tojson(indent=4) }}
|
||||
{%- if not loop.last %}
|
||||
{{- '
|
||||
|
||||
' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- '<|end_of_text|>
|
||||
' }}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' %}
|
||||
{{- '<|start_of_role|>system<|end_of_role|>' + message['content'] + '<|end_of_text|>
|
||||
' }}
|
||||
{%- elif message['role'] == 'user' %}
|
||||
{{- '<|start_of_role|>user<|end_of_role|>' + message['content'] + '<|end_of_text|>
|
||||
' }}
|
||||
{%- elif message['role'] == 'assistant_tool_call' or (message['role'] == 'assistant' and message.tool_calls is defined) %}
|
||||
{{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message.tool_calls|map(attribute='function')|list|tojson(indent=4) + '<|end_of_text|>
|
||||
' }}
|
||||
{%- elif message['role'] == 'assistant' %}
|
||||
{{- '<|start_of_role|>assistant<|end_of_role|>' + message['content'] + '<|end_of_text|>
|
||||
' }}
|
||||
{%- elif message['role'] == 'tool_response' or message['role'] == 'tool' %}
|
||||
{{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>
|
||||
' }}
|
||||
{%- endif %}
|
||||
{%- if loop.last and add_generation_prompt %}
|
||||
{{- '<|start_of_role|>assistant<|end_of_role|>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
{%- macro json_to_python_type(json_spec) %}
|
||||
{%- set basic_type_map = {
|
||||
"string": "str",
|
||||
"number": "float",
|
||||
"integer": "int",
|
||||
"boolean": "bool"
|
||||
} %}
|
||||
|
||||
{%- if basic_type_map[json_spec.type] is defined %}
|
||||
{{- basic_type_map[json_spec.type] }}
|
||||
{%- elif json_spec.type == "array" %}
|
||||
{{- "list[" + json_to_python_type(json_spec|items) + "]" }}
|
||||
{%- elif json_spec.type == "object" %}
|
||||
{%- if json_spec.additionalProperties is defined %}
|
||||
{{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']' }}
|
||||
{%- else %}
|
||||
{{- "dict" }}
|
||||
{%- endif %}
|
||||
{%- elif json_spec.type is iterable %}
|
||||
{{- "Union[" }}
|
||||
{%- for t in json_spec.type %}
|
||||
{{- json_to_python_type({"type": t}) }}
|
||||
{%- if not loop.last %}
|
||||
{{- "," }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "]" }}
|
||||
{%- else %}
|
||||
{{- "Any" }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- if not full_function_description is defined %}
|
||||
{%- set full_function_description = false %}
|
||||
{%- endif %}
|
||||
|
||||
{%- macro full_description(tool) %}
|
||||
{{- tool.name + '(' }}
|
||||
{%- if tool.parameters is defined %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- param_name + ": " + json_to_python_type(param_fields) }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- ")" }}
|
||||
{%- if tool.return is defined %}
|
||||
{{- " -> " + json_to_python_type(tool.return) }}
|
||||
{%- endif %}
|
||||
{{- " - " + tool.description + "\n\n" }}
|
||||
{%- if tool.parameters is defined %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{%- if loop.first %}
|
||||
{{- " Args:\n" }}
|
||||
{%- endif %}
|
||||
{{- " " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- if tool.return is defined and tool.return.description is defined %}
|
||||
{{- "\n Returns:\n " + tool.return.description }}
|
||||
{%- endif %}
|
||||
{{- '"' }}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- macro simple_description(tool) %}
|
||||
{{- tool.description }}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- macro function_description(tool) %}
|
||||
{%- if full_function_description %}
|
||||
{{- full_description(tool) }}
|
||||
{%- else %}
|
||||
{{- simple_description(tool) }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set sys_prompt = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set loop_messages = messages %}
|
||||
{% set sys_prompt = 'You are a helpful assistant with access to the following function calls. Your task is to understand the given conversation with function calls and responses and generate natural language response as the ASSISTANT to continue the conversation. You may use the following function calls to understand how to respond to the user query.' %}
|
||||
{%- endif %}
|
||||
|
||||
{{ 'SYSTEM: ' + sys_prompt }}
|
||||
{% if tools is iterable and tools | length > 0 %}
|
||||
<|function_call_library|>
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- '{"name": "' + tool.name + '", ' }}
|
||||
{{- '"description": "' + function_description(tool) }}
|
||||
{{- ', "parameters": ' }}
|
||||
{%- if not tool.parameters is defined or tool.parameters.properties | length == 0 %}
|
||||
{{- "{}" }}
|
||||
{%- else %}
|
||||
{{- tool.parameters|tojson }}
|
||||
{%- endif %}
|
||||
{{- "}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- "\n" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
If none of the functions are relevant or the given question lacks the parameters required by the function, please output \"<function_call> {\"name\": \"no_function\", \"arguments\": {}}\".
|
||||
{%- endif %}
|
||||
|
||||
|
||||
|
||||
{% for message in messages %}
|
||||
{% if message['role'] == 'user' %}
|
||||
{{- '\nUSER: ' + message['content'] }}
|
||||
{% elif message['role'] == 'assistant' and message.tool_calls is defined %}
|
||||
{{- '\nASSISTANT:' }}
|
||||
{% for tc in message.tool_calls %}
|
||||
{{- '<function_call> ' + {'name': tc.function.name, 'arguments': tc.function.arguments}|tojson }}
|
||||
{% endfor %}
|
||||
{{- '<|endoftext|>' }}
|
||||
{% elif message['role'] == 'assistant' %}
|
||||
{{- '\nASSISTANT: ' + message['content'] + ' <|endoftext|>' }}
|
||||
{% elif message['role'] == 'tool' %}
|
||||
{{- '<function_response> ' + message['content'] }}
|
||||
{%- else %}
|
||||
{{- raise_exception("Unexpected combination of role and message content") }}
|
||||
{% endif %}
|
||||
{% if loop.last and add_generation_prompt %}
|
||||
{{- '\nASSISTANT: ' }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,130 @@
|
||||
{%- macro json_to_python_type(json_spec) %}
|
||||
{%- set basic_type_map = {
|
||||
"string": "str",
|
||||
"number": "float",
|
||||
"integer": "int",
|
||||
"boolean": "bool"
|
||||
} %}
|
||||
|
||||
{%- if basic_type_map[json_spec.type] is defined %}
|
||||
{{- basic_type_map[json_spec.type] }}
|
||||
{%- elif json_spec.type == "array" %}
|
||||
{{- "list[" + json_to_python_type(json_spec|items) + "]" }}
|
||||
{%- elif json_spec.type == "object" %}
|
||||
{%- if json_spec.additionalProperties is defined %}
|
||||
{{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']' }}
|
||||
{%- else %}
|
||||
{{- "dict" }}
|
||||
{%- endif %}
|
||||
{%- elif json_spec.type is iterable %}
|
||||
{{- "Union[" }}
|
||||
{%- for t in json_spec.type %}
|
||||
{{- json_to_python_type({"type": t}) }}
|
||||
{%- if not loop.last %}
|
||||
{{- "," }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "]" }}
|
||||
{%- else %}
|
||||
{{- "Any" }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
|
||||
{{- bos_token }}
|
||||
{{- "<|im_start|>system\nYou are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- '{"type": "function", "function": ' }}
|
||||
{{- '{"name": "' + tool.name + '", ' }}
|
||||
{{- '"description": "' + tool.name + '(' }}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- param_name + ": " + json_to_python_type(param_fields) }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- ")" }}
|
||||
{%- if tool.return is defined %}
|
||||
{{- " -> " + json_to_python_type(tool.return) }}
|
||||
{%- endif %}
|
||||
{{- " - " + tool.description + "\n\n" }}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{%- if loop.first %}
|
||||
{{- " Args:\n" }}
|
||||
{%- endif %}
|
||||
{{- " " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
|
||||
{%- endfor %}
|
||||
{%- if tool.return is defined and tool.return.description is defined %}
|
||||
{{- "\n Returns:\n " + tool.return.description }}
|
||||
{%- endif %}
|
||||
{{- '"' }}
|
||||
{{- ', "parameters": ' }}
|
||||
{%- if tool.parameters.properties | length == 0 %}
|
||||
{{- "{}" }}
|
||||
{%- else %}
|
||||
{{- tool.parameters|tojson }}
|
||||
{%- endif %}
|
||||
{{- "}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- "\n" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- " </tools>" }}
|
||||
{{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
|
||||
' }}
|
||||
{{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
|
||||
" }}
|
||||
{{- "<tool_call>
|
||||
" }}
|
||||
{{- '{"name": <function-name>, "arguments": <args-dict>}
|
||||
' }}
|
||||
{{- '</tool_call><|im_end|>' }}
|
||||
{%- for message in messages %}
|
||||
{%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" and message.tool_calls is defined %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{{- '\n<tool_call>\n' }}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '{' }}
|
||||
{{- '"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '"' }}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{{- ', ' }}
|
||||
{{- '"arguments": ' }}
|
||||
{{- tool_call.arguments|tojson }}
|
||||
{%- endif %}
|
||||
{{- '}' }}
|
||||
{{- '\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>tool\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{%- if not loop.last %}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- else %}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- endif %}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
{% set loop_messages = messages %}
|
||||
{% if tools %}
|
||||
{% set weekday_map = {'Monday': '星期一', 'Tuesday': '星期二', 'Wednesday': '星期三', 'Thursday': '星期四', 'Friday': '星期五', 'Saturday': '星期六', 'Sunday': '星期日'} %}
|
||||
{% set weekday_cn = weekday_map[strftime_now('%A')] %}
|
||||
{% set datetime_str = strftime_now('%Y-%m-%d %H:%M:%S') %}
|
||||
{% set datetime_str = datetime_str + ' ' + weekday_cn %}
|
||||
{% for message in loop_messages %}
|
||||
{% if 'content' in message %}
|
||||
{% set content = message['content'] %}
|
||||
{% else %}
|
||||
{% set content = '' %}
|
||||
{% endif %}
|
||||
{% if loop.index0 == 0 %}
|
||||
{% set content_tmp = '你是一位函数组合专家。你会得到一个问题和一组可能的函数。根据问题,你需要进行一个或多个函数/工具调用以实现目的。
|
||||
如果没有一个函数可以使用,请直接使用自然语言回复用户,以助手:开头。
|
||||
如果给定的问题缺少函数所需的参数,请使用自然语言进行提问,向用户询问必要信息,以助手:开头。
|
||||
如果调用结果已经足够回答用户问题,请对历史结果进行总结,使用自然语言回复用户,以助手:开头。
|
||||
你应该只在工具调用部分返回函数调用。如果你决定调用任何函数,你必须将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>。你不应该在回复中包含任何其他文本。以下是你可以调用的函数列表,格式为JSON。
|
||||
' %}
|
||||
{% set content_tmp = content_tmp + '
|
||||
' + tools | tojson + '
|
||||
' %}
|
||||
{% if message['role'] == 'system' %}
|
||||
{% set content_tmp = content_tmp + '
|
||||
额外要求:
|
||||
' + content + '
|
||||
|
||||
如果你决定返回函数调用,请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>,不得包含其他文本。如果额外要求里有格式要求,请忽略,以此处为准。
|
||||
否则,请参考开头说的三种情况,以助手:开头进行回复。
|
||||
|
||||
如果额外要求里有时间信息,就以额外要求里的时间为准,否则,参考当前时间:' + datetime_str %}
|
||||
{% set content = '<|startoftext|>' + content_tmp + '<|extra_4|>' %}
|
||||
{% elif message['role'] == 'user' %}
|
||||
{% set content_tmp = content_tmp + '
|
||||
如果你决定返回函数调用,请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>,不得包含其他文本。
|
||||
否则,请参考开头说的三种情况,以助手:开头进行回复。
|
||||
|
||||
当前时间:' + datetime_str %}
|
||||
{% set content_tmp = '<|startoftext|>' + content_tmp + '<|extra_4|>'%}
|
||||
{% set content = content_tmp + '用户:' + content + '<|extra_0|>' %}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{% if message['role'] == 'user' %}
|
||||
{% set content = '用户:' + content + '<|extra_0|>' %}
|
||||
{% elif message['role'] == 'assistant' %}
|
||||
{% if 'tool_calls' in message %}
|
||||
{% set tool_calls = message['tool_calls'] %}
|
||||
{% set ns = namespace(tool_calls="[") %}
|
||||
{% for tool_call in tool_calls %}
|
||||
{% set function = tool_call['function'] %}
|
||||
{% set name = function['name'] %}
|
||||
{% set ns.tool_calls = ns.tool_calls + '{"name": "' + name + '", '%}
|
||||
{% set arguments = function['arguments'] %}
|
||||
{% if arguments is not string %}
|
||||
{% set arguments = arguments | tojson %}
|
||||
{% endif %}
|
||||
{% set ns.tool_calls = ns.tool_calls + '"arguments": ' + arguments + '}' %}
|
||||
{% if not loop.last %}
|
||||
{% set ns.tool_calls = ns.tool_calls + ', '%}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% set ns.tool_calls = ns.tool_calls + ']' %}
|
||||
{% set content = content + '<tool_calls>' + ns.tool_calls + '</tool_calls>' %}
|
||||
{% else %}
|
||||
{% set content = '助手:' + content %}
|
||||
{% endif %}
|
||||
{% set content = content + '<|eos|>' %}
|
||||
{% elif message['role'] == 'tool' %}
|
||||
{% if content is not string %}
|
||||
{set content = content | tojson }
|
||||
{% endif %}
|
||||
{% set content = '<tool_response>' + content + '</tool_response>' %}
|
||||
{% set content = content + '<|extra_0|>' %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{{- content -}}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% set context = {'has_head': true} %}
|
||||
{% for message in loop_messages %}
|
||||
{% if 'content' in message %}
|
||||
{% set content = message['content'] %}
|
||||
{% else %}
|
||||
{% set content = '' %}
|
||||
{% endif %}
|
||||
{% if loop.index0 == 0 %}
|
||||
{% if content == '' %}
|
||||
{% set _ = context.update({'has_head': false}) %}
|
||||
{% elif message['role'] == 'system' %}
|
||||
{% set content = '<|startoftext|>' + content + '<|extra_4|>' %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if message['role'] == 'user' %}
|
||||
{% if loop.index0 == 1 and not context.has_head %}
|
||||
{% set content = '<|startoftext|>' + content %}
|
||||
{% endif %}
|
||||
{% if loop.index0 == 1 and context.has_head %}
|
||||
{% set content = content + '<|extra_0|>' %}
|
||||
{% else %}
|
||||
{% set content = '<|startoftext|>' + content + '<|extra_0|>' %}
|
||||
{% endif %}
|
||||
{% elif message['role'] == 'assistant' %}
|
||||
{% set content = content + '<|eos|>' %}
|
||||
{% elif message['role'] == 'tool' %}
|
||||
{% set content = content + '<|extra_0|>' %}
|
||||
{% endif %}
|
||||
{{- content -}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||
{{- '<think>\n\n</think>\n' }}
|
||||
{%- endif %}
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{{- bos_token }}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message + "<|im_end|>\n" }}
|
||||
{%- endif %}
|
||||
|
||||
{%- if tools is not none %}
|
||||
{{- "<|im_start|>system name=<|plugin|>\n[" }}
|
||||
{%- for tool in tools %}
|
||||
{{- tool.function|tojson }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "<|im_end|>\n" }}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message["role"] == "user" %}
|
||||
{{- "<|im_start|>user\n" + message["content"] + "<|im_end|>\n"}}
|
||||
{%- elif message.tool_calls is defined and message.tool_calls is not none %}
|
||||
{%- set content = message["content"] if message["content"] else "" %}
|
||||
{{- "<|im_start|>assistant\n" + content }}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- set function=tool_call.function %}
|
||||
{{- "<|action_start|><|plugin|>\n" }}
|
||||
{{- '{"name": "' + function.name + '", '}}
|
||||
{{- '"arguments": ' + function.arguments|tojson + '}' }}
|
||||
{{- "<|action_end|>" }}
|
||||
{%- endfor %}
|
||||
{{- "<|im_end|>\n" }}
|
||||
{%- elif message["role"] == "assistant" %}
|
||||
{{- "<|im_start|>assistant\n" + message["content"] + "<|im_end|>\n"}}
|
||||
{%- elif message["role"] == "tool_results" or message["role"] == "tool" or message["role"] == "function" %}
|
||||
{%- if message.content is defined and message.content.content is defined %}
|
||||
{%- set content = message.content.content %}
|
||||
{%- else %}
|
||||
{%- set content = message.content %}
|
||||
{%- endif %}
|
||||
{{- "<|im_start|>environment name=<|plugin|>\n" + content|string + "<|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{{- raise_exception("Only user and assistant and tool_results and tool and function roles are supported, with the exception of an initial optional system message!") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{#- Llama 3.1 doesn't pass all tests if the tools are in the system prompt #}
|
||||
{%- set tools_in_user_message = true %}
|
||||
{%- endif %}
|
||||
{%- if not date_string is defined %}
|
||||
{%- if strftime_now is defined %}
|
||||
{%- set date_string = strftime_now("%d %b %Y") %}
|
||||
{%- else %}
|
||||
{%- set date_string = "26 Jul 2024" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set system_message = messages[0]['content'][0]['text']|trim %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- if tools is not none %}
|
||||
{%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message #}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- "Environment: ipython\n" }}
|
||||
{%- endif %}
|
||||
{{- "Cutting Knowledge Date: December 2023\n" }}
|
||||
{{- "Today Date: " + date_string + "\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set first_user_message = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
|
||||
{{- "Given the following functions, please respond with a JSON for a function call " }}
|
||||
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- first_user_message + "<|eot_id|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content'] | trim}}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{- content['text'] | trim }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{%- if not message.tool_calls|length == 1 %}
|
||||
{{- raise_exception("This model only supports single tool-calls at once!") }}
|
||||
{%- endif %}
|
||||
{%- set tool_call = message.tool_calls[0].function %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
|
||||
{{- '{"name": "' + tool_call.name + '", ' }}
|
||||
{{- '"parameters": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is string %}
|
||||
{{- { "output": message.content } | tojson }}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{- { "output": content['text'] } | tojson }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = false %}
|
||||
{%- endif %}
|
||||
{%- if not date_string is defined %}
|
||||
{%- if strftime_now is defined %}
|
||||
{%- set date_string = strftime_now("%d %b %Y") %}
|
||||
{%- else %}
|
||||
{%- set date_string = "26 Jul 2024" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- Find out if there are any images #}
|
||||
{% set image_ns = namespace(has_images=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'image' %}
|
||||
{%- set image_ns.has_images = true %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endfor %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set system_message = messages[0]['content'][0]['text']|trim %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- if tools is not none %}
|
||||
{%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message if there are no images, if the user supplied one, or if tools are used (default tool system message) #}
|
||||
{%- if system_message or not image_ns.has_images %}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- "Environment: ipython\n" }}
|
||||
{%- endif %}
|
||||
{{- "Cutting Knowledge Date: December 2023\n" }}
|
||||
{{- "Today Date: " + date_string + "\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set first_user_message = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
|
||||
{{- "Given the following functions, please respond with a JSON for a function call " }}
|
||||
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- first_user_message + "<|eot_id|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content'] | trim}}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'image' %}
|
||||
{{- '<|image|>' }}
|
||||
{%- elif content['type'] == 'text' %}
|
||||
{{- content['text'] | trim }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{%- if not message.tool_calls|length == 1 %}
|
||||
{{- raise_exception("This model only supports single tool-calls at once!") }}
|
||||
{%- endif %}
|
||||
{%- set tool_call = message.tool_calls[0].function %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
|
||||
{{- '{"name": "' + tool_call.name + '", ' }}
|
||||
{{- '"parameters": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is string %}
|
||||
{{- { "output": message.content } | tojson }}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{- { "output": content['text'] } | tojson }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = false %}
|
||||
{%- endif %}
|
||||
{%- if not date_string is defined %}
|
||||
{%- if strftime_now is defined %}
|
||||
{%- set date_string = strftime_now("%d %b %Y") %}
|
||||
{%- else %}
|
||||
{%- set date_string = "26 Jul 2024" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message #}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- "Environment: ipython\n" }}
|
||||
{%- endif %}
|
||||
{{- "Cutting Knowledge Date: December 2023\n" }}
|
||||
{{- "Today Date: " + date_string + "\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You have access to the following functions. To call functions, please respond with a python list of the calls. " }}
|
||||
{{- 'Respond in the format [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- set first_user_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
|
||||
{{- "Given the following functions, please respond with a python list for function calls " }}
|
||||
{{- "with their proper arguments to best answer the given prompt.\n\n" }}
|
||||
{{- 'Respond in the format [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] ' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- first_user_message + "<|eot_id|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n[' -}}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- tool_call.name + '(' -}}
|
||||
{%- for param in tool_call.arguments %}
|
||||
{{- param + '=' -}}
|
||||
{{- "%s" | format(tool_call.arguments[param]) -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- ')' -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- ']<|eot_id|>' -}}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is mapping %}
|
||||
{{- message.content | tojson }}
|
||||
{%- else %}
|
||||
{{- { "output": message.content } | tojson }}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
{%- macro is_array_of_type_objects(var) -%}
|
||||
{%- if var is iterable and var is not string -%}
|
||||
{%- set valid = true -%}
|
||||
{%- for item in var -%}
|
||||
{%- if 'type' not in item -%}
|
||||
{%- set valid = false -%}
|
||||
{%- break -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{ valid }}
|
||||
{%- else -%}
|
||||
{{ false }}
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- macro render_message(message) %}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content']|trim }}
|
||||
{%- elif is_array_of_type_objects(data) == 'True' %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'image' %}
|
||||
{{- '<|image|>' }}
|
||||
{%- elif content['type'] == 'text' %}
|
||||
{{- content['text']|trim }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- else %}
|
||||
{{- message['content']|tojson }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = true %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0] %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = ({ "content": "You are a helpful assistant with tool calling "
|
||||
"capabilities. Only reply with a tool call if the function exists in the "
|
||||
"library provided by the user. If it doesn't exist, just reply directly in "
|
||||
"natural language. When you receive a tool call response, use the output to "
|
||||
"format an answer to the original user question."}) %}
|
||||
{%- endif %}
|
||||
|
||||
{%- set tool_lib_preamble = 'Tools: You have access to the following tools. You might need to use one '
|
||||
'or more function/tool calls to fulfill the task. \n'
|
||||
'If none are needed, then proceed to the response.\n\n'
|
||||
'Tool Call Syntax: You can call tools using the following syntax:\n'
|
||||
'{"name": function name, "parameters": dictionary of argument name and its value}.\n'
|
||||
'Separate multiple function calls by "; ". Do not use variables.\n'
|
||||
'Do not include anything else when calling the tools with the syntax above.\n\n'
|
||||
'Here is a list of functions in JSON format that you can invoke.\n' %}
|
||||
|
||||
{{- "<|header_start|>system<|header_end|>\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- tool_lib_preamble }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- render_message(system_message) }}
|
||||
{{ "<|eot|>\n" }}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- set first_user_message = messages[0] %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|header_start|>user<|header_end|>\n\n' }}
|
||||
{{- tool_lib_preamble }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- render_message(first_user_message) + "\n<|eot|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
|
||||
{{- render_message(message) }}
|
||||
{{- "\n<|eot|>" }}
|
||||
{%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
|
||||
{{- '\n<|header_start|>assistant<|header_end|>\n\n' -}}
|
||||
{{- render_message(message) }}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{{- '{"name": "' + tool_call.function.name + '", ' }}
|
||||
{{- '"parameters": ' }}
|
||||
{{- tool_call.function.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{%- endfor %}
|
||||
{{- "\n<|eot|>" }}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "\n<|header_start|>ipython<|header_end|>\n\n" }}
|
||||
{{- render_message(message) }}
|
||||
{{- "\n<|eom|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '\n<|header_start|>assistant<|header_end|>\n\n' }}
|
||||
{%- endif %}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined and custom_tools%}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if tools is defined and tools %}
|
||||
{%- set tool_definition = tool_definition ~ (tools | tojson(indent=4)) %}
|
||||
{%- else %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set user_provided_system_message = true %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set system_message = messages[0]['content'][0]['text']|trim %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- if tools is not none %}
|
||||
{#- Since not system_message was provided by user, if tool is provided, system_message is now default tool system message #}
|
||||
{#- This system message is from llama website:https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/ #}
|
||||
{%- set system_message = "You are a helpful assistant and an expert in function composition. You can answer general questions using your internal knowledge OR invoke functions when necessary. Follow these strict guidelines:\n\n1. FUNCTION CALLS:\n- ONLY use functions that are EXPLICITLY listed in the function list below\n- If NO functions are listed (empty function list []), respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If a function is not in the list, respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If ALL required parameters are present AND the query EXACTLY matches a listed function's purpose: output ONLY the function call(s)\n- Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\nExamples:\nCORRECT: [get_weather(location=\"Vancouver\"), calculate_route(start=\"Boston\", end=\"New York\")] <- Only if get_weather and calculate_route are in function list\nINCORRECT: get_weather(location=\"New York\")\nINCORRECT: Let me check the weather: [get_weather(location=\"New York\")]\nINCORRECT: [get_events(location=\"Singapore\")] <- If function not in list\n\n2. RESPONSE RULES:\n- For pure function requests matching a listed function: ONLY output the function call(s)\n- For knowledge questions: ONLY output text\n- For missing parameters: ONLY request the specific missing parameters\n- For unavailable services (not in function list): output ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\". Do NOT execute a function call.\n- If the query asks for information beyond what a listed function provides: output ONLY with internal knowledge about your limitations\n- NEVER combine text and function calls in the same response\n- NEVER suggest alternative functions when the requested service is unavailable\n- NEVER create or invent new functions not listed below\n\n3. STRICT BOUNDARIES:\n- ONLY use functions from the list below - no exceptions\n- NEVER use a function as an alternative to unavailable information\n- NEVER call functions not present in the function list\n- NEVER add explanatory text to function calls\n- NEVER respond with empty brackets\n- Use proper Python/JSON syntax for function calls\n- Check the function list carefully before responding\n\n4. TOOL RESPONSE HANDLING:\n- When receiving tool responses: provide concise, natural language responses\n- Don't repeat tool response verbatim\n- Don't add supplementary information\n\nHere is a list of functions in JSON format that you can invoke:\n" %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{#- Now writing the system message: use the user provided system message if user_provided_system_message, else default tool system message if tools presented #}
|
||||
{%- if system_message %}
|
||||
{#- always use user provided system message to override default tool system message #}
|
||||
{{- "<|header_start|>system<|header_end|>\n\n" }}
|
||||
{{- system_message }}
|
||||
{%- if user_provided_system_message and tools %}
|
||||
{{- "\nHere is a list of functions in JSON format that you can invoke. Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\n" }}
|
||||
{{- tool_definition -}}
|
||||
{%- elif tool_definition %}
|
||||
{{- tool_definition -}}
|
||||
{%- endif %}
|
||||
{{- "<|eot|>" }}
|
||||
{%- endif %}
|
||||
|
||||
{#- Now deal with all other messages #}
|
||||
{%- for message in messages %}
|
||||
{#- Base case: messages that are not from tool role and has empty tool_call list #}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or ('tool_calls' in message and message.tool_calls|length != 0 )) %}
|
||||
{{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content'] }}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'image' %}
|
||||
{{- '<|image|>' }}
|
||||
{%- elif content['type'] == 'text' %}
|
||||
{{- content['text'] | trim }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|eot|>" }}
|
||||
{#- Tool case: messages has non-empty tool_call list, must from assistant #}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{#- assume tool_calls are always coming from assistant #}
|
||||
{%- if message.role == 'assistant' %}
|
||||
{{- '<|header_start|>assistant<|header_end|>\n\n' -}}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content'] }}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'image' %}
|
||||
{{- '<|image|>' }}
|
||||
{%- elif content['type'] == 'text' %}
|
||||
{{- content['text'] }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "[" }}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- tool_call.name + '(' -}}
|
||||
{%- for param in tool_call.arguments %}
|
||||
{{- param + '="' -}}
|
||||
{{- "%s" | format(tool_call.arguments[param]) -}}
|
||||
{{- '"' -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- ')' -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- "]<|eot|>" }}
|
||||
{%- endif %}
|
||||
{#- Tool_response case: messages are from tool_response #}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|header_start|>ipython<|header_end|>\n\n" }}
|
||||
{%- if message.content is string %}
|
||||
{{- message.content | tojson }}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{- content['text'] | tojson }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|eot|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|header_start|>assistant<|header_end|>\n\n' }}
|
||||
{%- endif %}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
{{ '<begin_of_document>' -}}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- Extract system message #}
|
||||
{% set ns = namespace(system_prompt='') -%}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set ns.system_prompt = messages[0]['content']|trim %}
|
||||
{%- else %}
|
||||
{%- set ns.system_prompt = messages[0]['content'][0]['text']|trim %}
|
||||
{%- endif %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- if tools is not none %}
|
||||
{%- set ns.system_prompt = "You are a helpful assistant created by Minimax based on MiniMax-M1 model." %}
|
||||
{%- else %}
|
||||
{%- set ns.system_prompt = "You are a helpful assistant created by Minimax based on MiniMax-M1 model." %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message #}
|
||||
{%- if ns.system_prompt != '' %}
|
||||
{{ '<beginning_of_sentence>system ai_setting=assistant\n' + ns.system_prompt + '<end_of_sentence>\n' -}}
|
||||
{%- endif %}
|
||||
|
||||
{#- Tools configuration #}
|
||||
{%- if tools is not none %}
|
||||
{{ '<beginning_of_sentence>system tool_setting=tools\nYou are provided with these tools:\n<tools>\n' -}}
|
||||
{%- for tool in tools %}
|
||||
{{ tool | tojson ~ '\n' -}}
|
||||
{%- endfor %}
|
||||
{{ '</tools>\n\nIf you need to call tools, please respond with <tool_calls></tool_calls> XML tags, and provide tool-name and json-object of arguments, following the format below:\n<tool_calls>\n{"name": <tool-name>, "arguments": <args-json-object>}\n...\n</tool_calls><end_of_sentence>\n' -}}
|
||||
{%- endif %}
|
||||
|
||||
{#- Process messages #}
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{%- if message['role'] == 'user' %}
|
||||
{{ '<beginning_of_sentence>user name=user\n' -}}
|
||||
{%- if message['content'] is string %}
|
||||
{{ message['content']|trim -}}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{ content['text']|trim -}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{ '<end_of_sentence>\n' -}}
|
||||
{%- elif message['role'] == 'assistant' %}
|
||||
{{ '<beginning_of_sentence>ai name=assistant\n' -}}
|
||||
{%- if message['content'] is string %}
|
||||
{{ message['content']|trim -}}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] | selectattr('type', 'equalto', 'text') %}
|
||||
{{ content['text']|trim -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{ '<end_of_sentence>\n' -}}
|
||||
{%- endif %}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{{ '<beginning_of_sentence>ai name=assistant\n<tool_calls>\n' -}}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{{ '{"name": "' + tool_call.function.name + '", "arguments": ' + tool_call.function.arguments | tojson + '}\n' -}}
|
||||
{%- endfor %}
|
||||
{{ '</tool_calls><end_of_sentence>\n' -}}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{ '<beginning_of_sentence>tool name=tools\n' -}}
|
||||
{%- if message.content is string %}
|
||||
{{ 'tool result: ' + message.content + '\n\n' -}}
|
||||
{%- else %}
|
||||
{%- for content in message['content'] %}
|
||||
{%- if content['type'] == 'text' %}
|
||||
{{ 'tool result: ' + content['text'] + '\n\n' -}}
|
||||
{%- elif content.get('name') %}
|
||||
{{ 'tool name: ' + content['name'] + '\ntool result: ' + content['text'] + '\n\n' -}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{ '<end_of_sentence>\n' -}}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{{ '<beginning_of_sentence>ai name=assistant\n' -}}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,86 @@
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
|
||||
|
||||
{%- for message in loop_messages | rejectattr("role", "equalto", "tool") | rejectattr("role", "equalto", "tool_results") | selectattr("tool_calls", "undefined") %}
|
||||
{%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
|
||||
{{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{{- bos_token }}
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message["role"] == "user" %}
|
||||
{%- if tools is not none and (message == user_messages[-1]) %}
|
||||
{{- "[AVAILABLE_TOOLS] [" }}
|
||||
{%- for tool in tools %}
|
||||
{%- set tool = tool.function %}
|
||||
{{- '{"type": "function", "function": {' }}
|
||||
{%- for key, val in tool.items() if key != "return" %}
|
||||
{%- if val is string %}
|
||||
{{- '"' + key + '": "' + val + '"' }}
|
||||
{%- else %}
|
||||
{{- '"' + key + '": ' + val|tojson }}
|
||||
{%- endif %}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "}}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "[/AVAILABLE_TOOLS]" }}
|
||||
{%- endif %}
|
||||
{%- if loop.last and system_message is defined %}
|
||||
{{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
|
||||
{%- else %}
|
||||
{{- "[INST] " + message["content"] + "[/INST]" }}
|
||||
{%- endif %}
|
||||
{%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
|
||||
{%- if message.tool_calls is defined %}
|
||||
{%- set tool_calls = message.tool_calls %}
|
||||
{%- else %}
|
||||
{%- set tool_calls = message.content %}
|
||||
{%- endif %}
|
||||
{{- "[TOOL_CALLS] [" }}
|
||||
{%- for tool_call in tool_calls %}
|
||||
{%- set out = tool_call.function|tojson %}
|
||||
{{- out[:-1] }}
|
||||
{%- if not tool_call.id is defined or tool_call.id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
|
||||
{%- endif %}
|
||||
{{- ', "id": "' + tool_call.id[-9:] + '"}' }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" + eos_token }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif message["role"] == "assistant" %}
|
||||
{{- " " + message["content"] + eos_token }}
|
||||
{%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
|
||||
{%- if message.content is defined and message.content.content is defined %}
|
||||
{%- set content = message.content.content %}
|
||||
{%- else %}
|
||||
{%- set content = message.content %}
|
||||
{%- endif %}
|
||||
{{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
|
||||
{%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
|
||||
{%- endif %}
|
||||
{{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
|
||||
{%- else %}
|
||||
{{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{%- set today = strftime_now("%Y-%m-%d") %}
|
||||
{%- set default_system_message = "You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\nYour knowledge base was last updated on 2023-10-01. The current date is " + today + ".\n\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")" %}
|
||||
|
||||
{{- bos_token }}
|
||||
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- if messages[0]['content'] is string %}
|
||||
{%- set system_message = messages[0]['content'] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = messages[0]['content'][0]['text'] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{%- set system_message = default_system_message %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- elif tools is not none %}
|
||||
{%- set parallel_tool_prompt = "You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\"name\": tool call name, \"arguments\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user." %}
|
||||
{%- if system_message is defined %}
|
||||
{%- set system_message = parallel_tool_prompt + "\n\n" + system_message %}
|
||||
{%- else %}
|
||||
{%- set system_message = parallel_tool_prompt %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}
|
||||
|
||||
{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
|
||||
|
||||
{%- set filtered_messages = [] %}
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message["role"] not in ["tool", "tool_results"] and not message.get("tool_calls") %}
|
||||
{%- set filtered_messages = filtered_messages + [message] %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- for message in filtered_messages %}
|
||||
{%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
|
||||
{{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message["role"] == "user" %}
|
||||
{%- if tools is not none and (message == user_messages[-1]) %}
|
||||
{{- "[AVAILABLE_TOOLS] [" }}
|
||||
{%- for tool in tools %}
|
||||
{%- set tool = tool.function %}
|
||||
{{- '{"type": "function", "function": {' }}
|
||||
{%- for key, val in tool.items() if key != "return" %}
|
||||
{%- if val is string %}
|
||||
{{- '"' + key + '": "' + val + '"' }}
|
||||
{%- else %}
|
||||
{{- '"' + key + '": ' + val|tojson }}
|
||||
{%- endif %}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "}}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "[/AVAILABLE_TOOLS]" }}
|
||||
{%- endif %}
|
||||
{%- if message['content'] is string %}
|
||||
{{- '[INST]' + message['content'] + '[/INST]' }}
|
||||
{%- else %}
|
||||
{{- '[INST]' }}
|
||||
{%- for block in message['content'] %}
|
||||
{%- if block['type'] == 'text' %}
|
||||
{{- block['text'] }}
|
||||
{%- elif block['type'] == 'image' or block['type'] == 'image_url' %}
|
||||
{{- '[IMG]' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Only text and image blocks are supported in message content!') }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- '[/INST]' }}
|
||||
{%- endif %}
|
||||
{%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
|
||||
{%- if message.tool_calls is defined %}
|
||||
{%- set tool_calls = message.tool_calls %}
|
||||
{%- else %}
|
||||
{%- set tool_calls = message.content %}
|
||||
{%- endif %}
|
||||
{{- "[TOOL_CALLS] [" }}
|
||||
{%- for tool_call in tool_calls %}
|
||||
{%- set out = tool_call.function|tojson %}
|
||||
{{- out[:-1] }}
|
||||
{%- if not tool_call.id is defined or tool_call.id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
|
||||
{%- endif %}
|
||||
{{- ', "id": "' + tool_call.id[-9:] + '"}' }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" + eos_token }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif message['role'] == 'assistant' %}
|
||||
{%- if message['content'] is string %}
|
||||
{{- message['content'] + eos_token }}
|
||||
{%- else %}
|
||||
{{- message['content'][0]['text'] + eos_token }}
|
||||
{%- endif %}
|
||||
{%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
|
||||
{%- if message.content is defined and message.content.content is defined %}
|
||||
{%- set content = message.content.content %}
|
||||
{%- else %}
|
||||
{%- set content = message.content %}
|
||||
{%- endif %}
|
||||
{{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
|
||||
{%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
|
||||
{%- endif %}
|
||||
{{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
|
||||
{%- else %}
|
||||
{{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- elif tools is not none %}
|
||||
{%- set parallel_tool_prompt = "You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\"name\": tool call name, \"arguments\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user." %}
|
||||
{%- if system_message is defined %}
|
||||
{%- set system_message = parallel_tool_prompt + "\n\n" + system_message %}
|
||||
{%- else %}
|
||||
{%- set system_message = parallel_tool_prompt %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
|
||||
|
||||
{%- for message in loop_messages | rejectattr("role", "equalto", "tool") | rejectattr("role", "equalto", "tool_results") | selectattr("tool_calls", "undefined") %}
|
||||
{%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
|
||||
{{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{{- bos_token }}
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message["role"] == "user" %}
|
||||
{%- if tools is not none and (message == user_messages[-1]) %}
|
||||
{{- "[AVAILABLE_TOOLS] [" }}
|
||||
{%- for tool in tools %}
|
||||
{%- set tool = tool.function %}
|
||||
{{- '{"type": "function", "function": {' }}
|
||||
{%- for key, val in tool.items() if key != "return" %}
|
||||
{%- if val is string %}
|
||||
{{- '"' + key + '": "' + val + '"' }}
|
||||
{%- else %}
|
||||
{{- '"' + key + '": ' + val|tojson }}
|
||||
{%- endif %}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "}}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "[/AVAILABLE_TOOLS]" }}
|
||||
{%- endif %}
|
||||
{%- if loop.last and system_message is defined %}
|
||||
{{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
|
||||
{%- else %}
|
||||
{{- "[INST] " + message["content"] + "[/INST]" }}
|
||||
{%- endif %}
|
||||
{%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
|
||||
{%- if message.tool_calls is defined %}
|
||||
{%- set tool_calls = message.tool_calls %}
|
||||
{%- else %}
|
||||
{%- set tool_calls = message.content %}
|
||||
{%- endif %}
|
||||
{{- "[TOOL_CALLS] [" }}
|
||||
{%- for tool_call in tool_calls %}
|
||||
{%- set out = tool_call.function|tojson %}
|
||||
{{- out[:-1] }}
|
||||
{%- if not tool_call.id is defined or tool_call.id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
|
||||
{%- endif %}
|
||||
{{- ', "id": "' + tool_call.id[-9:] + '"}' }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- else %}
|
||||
{{- "]" + eos_token }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif message["role"] == "assistant" %}
|
||||
{{- " " + message["content"] + eos_token }}
|
||||
{%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
|
||||
{%- if message.content is defined and message.content.content is defined %}
|
||||
{%- set content = message.content.content %}
|
||||
{%- else %}
|
||||
{%- set content = message.content %}
|
||||
{%- endif %}
|
||||
{{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
|
||||
{%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
|
||||
{{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
|
||||
{%- endif %}
|
||||
{{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
|
||||
{%- else %}
|
||||
{{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
{%- if messages and messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant." %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if messages %}
|
||||
<|system|>
|
||||
{{ system_message }}
|
||||
{%- if tools %}
|
||||
In addition to plain text responses, you can choose to call one or more of the provided functions.
|
||||
|
||||
Use the following rule to decide when to call a function:
|
||||
* if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so
|
||||
* if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls
|
||||
|
||||
If you decide to call functions:
|
||||
* prefix function calls with functools marker (no closing marker required)
|
||||
* all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...]
|
||||
* follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples
|
||||
* respect the argument type formatting. E.g., if the type is number and format is float, write value 7 as 7.0
|
||||
* make sure you pick the right functions that match the user intent
|
||||
|
||||
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}<|end|>
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if message.role != "system" %}
|
||||
<|{{ message.role }}|>
|
||||
{%- if message.content and message.role == "tools" %}
|
||||
{"result": {{ message.content }}}
|
||||
{%- elif message.content %}
|
||||
{{ message.content }}
|
||||
{%- elif message.tool_calls %}
|
||||
{%- for call in message.tool_calls %}
|
||||
{"name": "{{ call.function.name }}", "arguments": {{ call.function.arguments }}}
|
||||
{%- if not loop.last %},{% endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}<|end|>
|
||||
{%- endif %}
|
||||
{%- endfor %}<|assistant|>
|
||||
|
||||
{%- else %}
|
||||
{%- if system_message %}
|
||||
<|system|>
|
||||
|
||||
{{ system_message }}<|end|>
|
||||
{%- endif %}
|
||||
{%- if prompt %}
|
||||
<|user|>
|
||||
|
||||
{{ prompt }}<|end|>
|
||||
{%- endif %}<|assistant|>
|
||||
|
||||
{%- endif %}
|
||||
{{ response }}
|
||||
{%- if response %}<|user|>{% endif %}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
|
||||
{{- '\n' + message.content | trim + '\n' }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value }}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,65 @@
|
||||
{{- bos_token }}
|
||||
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language." %}
|
||||
{%- endif %}
|
||||
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose.\n" }}
|
||||
{{- "If none of the function can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\n" }}
|
||||
{{- "You should only return the function call in tools call sections.\n\n" }}
|
||||
{{- "If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)]\n" }}
|
||||
{{- "You SHOULD NOT include any other text in the response.\n" }}
|
||||
{{- "Here is a list of functions in JSON format that you can invoke.\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- "\n" }}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n[' -}}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- tool_call.name + '(' -}}
|
||||
{%- for param in tool_call.arguments %}
|
||||
{{- param + '=' -}}
|
||||
{{- "%s" | format(tool_call.arguments[param]) -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- ')' -}}
|
||||
{% if not loop.last %}, {% endif %}
|
||||
{%- endfor %}
|
||||
{{- ']<|eot_id|>' -}}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is mapping %}
|
||||
{{- message.content | tojson }}
|
||||
{%- else %}
|
||||
{{- { "output": message.content } | tojson }}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = true %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- Extract system message #}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content'] | trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{{- system_message + "\n" }}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant. You are developed by Salesforce xLAM team." %}
|
||||
{% set format_instruction %}You have access to a set of tools. When using tools, make calls in a single JSON array:
|
||||
|
||||
[{"name": "tool_call_name", "arguments": {"arg1": "value1", "arg2": "value2"}}, ... (additional parallel tool calls as needed)]
|
||||
|
||||
If no tool is suitable, state that explicitly. If the user's input lacks required parameters, ask for clarification. Do not interpret or respond until tool results are returned. Once they are available, process them or make additional calls if needed. For tasks that don't require tools, such as casual conversation or general advice, respond directly in plain text. The available tools are:{% endset %}
|
||||
{{- system_message + "\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- format_instruction + "\n\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if tools is not none %}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
|
||||
{%- if message['tool_calls'] %}
|
||||
{{- "[" }}
|
||||
{%- for tool_call_function in message.tool_calls %}
|
||||
{%- set tool_call = tool_call_function.function %}
|
||||
{{- '{"name": "' + tool_call.name + '", ' }}
|
||||
{{- '"arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "]" }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- elif message['content'] %}
|
||||
{{- message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- else %}
|
||||
{{- "[]\n" + '<|eot_id|>' }}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>" + "ipython" + "<|end_header_id|>\n\n" }}
|
||||
{%- set content = message["content"] %}
|
||||
{%- if content is mapping or (content is iterable and content is not string) %}
|
||||
{{- content | tojson }}
|
||||
{%- else %}
|
||||
{{- content }}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{# System message #}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content'] | trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{{- system_message + "\n" }}
|
||||
{%- else %}
|
||||
{%- set system_message = "You are a helpful assistant. You are developed by Salesforce xLAM team." %}
|
||||
{% set format_instruction %}You have access to a set of tools. When using tools, make calls in a single JSON array:
|
||||
|
||||
[{"name": "tool_call_name", "arguments": {"arg1": "value1", "arg2": "value2"}}, ... (additional parallel tool calls as needed)]
|
||||
|
||||
If no tool is suitable, state that explicitly. If the user's input lacks required parameters, ask for clarification. Do not interpret or respond until tool results are returned. Once they are available, process them or make additional calls if needed. For tasks that don't require tools, such as casual conversation or general advice, respond directly in plain text. The available tools are:{% endset %}
|
||||
{{- system_message + "\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- format_instruction + "\n\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- if tools is not none %}
|
||||
{%- for func in tools %}
|
||||
{{- func | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- "<|im_end|>\n" }}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'tool' %}
|
||||
{{- "<|im_start|>tool\n" }}
|
||||
{%- if message.content is defined and message.content.content is defined %}
|
||||
{%- set content = message.content.content %}
|
||||
{%- else %}
|
||||
{%- set content = message.content %}
|
||||
{%- endif %}
|
||||
{%- if content is mapping or content is iterable and content is not string %}
|
||||
{{- content | tojson }}
|
||||
{%- else %}
|
||||
{{- content }}
|
||||
{%- endif %}
|
||||
{{- "<|im_end|>\n" }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{{- "<|im_start|>assistant\n" }}
|
||||
{%- if message['tool_calls'] %}
|
||||
{{- "[" }}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- set out = tool_call.function | tojson %}
|
||||
{{- out }}
|
||||
{%- if not loop.last %}
|
||||
{{- ", " }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- "]"}}
|
||||
{%- elif message['content'] %}
|
||||
{{- message['content'] | trim }}
|
||||
{%- else %}
|
||||
{{- "[]\n" }}
|
||||
{%- endif %}
|
||||
{{- "<|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{{- "<|im_start|>" + message['role'] + "\n" + message['content'] | trim + "<|im_end|>\n" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{{- "<|im_start|>assistant\n" }}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "vllm-cmd"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "vllm-rs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
native-tls-vendored = ["dep:native-tls-vendored"]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
educe.workspace = true
|
||||
itertools.workspace = true
|
||||
native-tls-vendored = { workspace = true, optional = true }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["signal"] }
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-server.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
expect-test.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,44 @@
|
||||
# `vllm-rs` CLI Quick Start
|
||||
|
||||
Start Qwen3 with one managed `vllm-rs serve` command from the repo root:
|
||||
|
||||
```bash
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
cargo run --bin vllm-rs -- serve \
|
||||
Qwen/Qwen3-0.6B \
|
||||
--python ../vllm/.venv/bin/python \
|
||||
--max-model-len 512 \
|
||||
-- \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
This launches:
|
||||
|
||||
- a managed headless Python `vllm` engine
|
||||
- the Rust OpenAI-compatible frontend on `127.0.0.1:8000`
|
||||
|
||||
All Python engine arguments must be placed after `--`. Arguments before `--` are parsed by the Rust
|
||||
frontend itself.
|
||||
|
||||
You can then send OpenAI-style requests to the Rust frontend:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
If you already started headless `vllm` yourself, use `frontend` instead:
|
||||
|
||||
```bash
|
||||
cargo run --bin vllm-rs -- frontend \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
Qwen/Qwen3-0.6B
|
||||
```
|
||||
@@ -0,0 +1,434 @@
|
||||
//! CLI argument definitions for the `vllm-rs` binary.
|
||||
//!
|
||||
//! Python vLLM references:
|
||||
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
|
||||
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
|
||||
|
||||
mod unsupported;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use educe::Educe;
|
||||
use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
|
||||
RendererSelection,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
|
||||
/// Top-level parser for the `vllm-rs` binary.
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
name = "vllm-rs",
|
||||
about = "Rust frontend and managed-engine CLI for vLLM."
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[command(subcommand)]
|
||||
pub command: Command,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub fn parse() -> Self {
|
||||
Self::try_parse_from(std::env::args_os()).unwrap_or_else(|error| error.exit())
|
||||
}
|
||||
|
||||
pub fn try_parse_from<I, T>(itr: I) -> Result<Self, clap::Error>
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<OsString>,
|
||||
{
|
||||
let args: Vec<OsString> = itr.into_iter().map(Into::into).collect();
|
||||
let repartitioned_args = repartition_managed_engine_args::<Self>(&args, Some("serve"))?;
|
||||
<Self as Parser>::try_parse_from(&repartitioned_args).inspect(|cli| {
|
||||
if let Command::Serve(serve) = &cli.command
|
||||
&& serve.debug_cli
|
||||
{
|
||||
println!(
|
||||
"Original CLI args: {}\n",
|
||||
args.join(OsStr::new(" ")).display()
|
||||
);
|
||||
println!(
|
||||
"Repartitioned CLI args: {}\n",
|
||||
repartitioned_args.join(OsStr::new(" ")).display()
|
||||
);
|
||||
println!(
|
||||
"Passthrough Python args: {}",
|
||||
serve.managed_engine.python_args.join(" ")
|
||||
);
|
||||
std::process::exit(0);
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Supported top-level CLI commands.
|
||||
#[derive(Debug, Subcommand, PartialEq, Eq)]
|
||||
pub enum Command {
|
||||
/// Run the Rust OpenAI frontend as a Python-supervised worker.
|
||||
Frontend(FrontendArgs),
|
||||
/// Launch a managed Python headless engine, then run the Rust OpenAI
|
||||
/// frontend.
|
||||
Serve(ServeArgs),
|
||||
}
|
||||
|
||||
/// Runtime arguments shared by the external-engine and managed-engine paths.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
||||
#[educe(Debug)]
|
||||
pub struct SharedRuntimeArgs {
|
||||
#[serde(rename = "model_tag")]
|
||||
/// Model identifier or local model directory used for backend loading and
|
||||
/// public model ID.
|
||||
pub model: String,
|
||||
|
||||
/// Maximum time to wait for the expected engines to register on the
|
||||
/// frontend transport.
|
||||
#[arg(
|
||||
long = "engine-ready-timeout-secs",
|
||||
env = "VLLM_ENGINE_READY_TIMEOUT_S",
|
||||
default_value_t = default_engine_ready_timeout_secs()
|
||||
)]
|
||||
#[serde(default = "default_engine_ready_timeout_secs")]
|
||||
pub engine_ready_timeout_secs: u64,
|
||||
|
||||
/// Select the tool call parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Select the reasoning parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
#[serde(default, rename = "tokenizer_mode")]
|
||||
pub renderer: RendererSelection,
|
||||
/// Override the maximum model context length. When set, the frontend uses
|
||||
/// this value instead of the model's `max_position_embeddings` from
|
||||
/// `config.json`.
|
||||
#[arg(long)]
|
||||
pub max_model_len: Option<u32>,
|
||||
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
|
||||
/// started.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
pub shutdown_timeout: u64,
|
||||
|
||||
/// The file path to the chat template, or the template in single-line form
|
||||
/// for the specified model.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Default keyword arguments to pass to the chat template renderer.
|
||||
///
|
||||
/// These will be merged with request-level chat_template_kwargs, with
|
||||
/// request values taking precedence. Useful for setting default
|
||||
/// behavior for reasoning models.
|
||||
///
|
||||
/// Example: `{"enable_thinking": false}` to disable thinking mode by
|
||||
/// default for Qwen3/DeepSeek models.
|
||||
#[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
|
||||
#[serde(default)]
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// The format to render message content within a chat template.
|
||||
///
|
||||
/// * "auto" detects the format from the template
|
||||
/// * "string" renders content as a string. Example: `"Hello World"`
|
||||
/// * "openai" renders content as a list of dictionaries, similar to OpenAI schema. Example:
|
||||
/// `[{"type": "text", "text": "Hello world!"}]`
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
|
||||
/// Log a summary line for each completed request, including prompt/output
|
||||
/// token counts and finish reason.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub enable_log_requests: bool,
|
||||
|
||||
/// Disable periodic logging of engine statistics (throughput, queue depth,
|
||||
/// cache usage).
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub disable_log_stats: bool,
|
||||
|
||||
/// The model name(s) used in the API. If multiple names are provided, the
|
||||
/// server will respond to any of the provided names. The model name in the
|
||||
/// model field of a response will be the first name in this list. If not
|
||||
/// specified, the model name will be the same as the `--model` argument.
|
||||
/// Noted that this name(s) will also be used in `model_name` tag
|
||||
/// content of prometheus metrics, if multiple names provided, metrics
|
||||
/// tag will take the first one.
|
||||
#[arg(long, num_args = 0..)]
|
||||
#[serde(default)]
|
||||
pub served_model_name: Vec<String>,
|
||||
|
||||
/// Unsupported Python vLLM frontend arguments recognized but not yet
|
||||
/// implemented in Rust.
|
||||
#[educe(Debug(ignore))]
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
pub unsupported: UnsupportedArgs,
|
||||
}
|
||||
|
||||
impl SharedRuntimeArgs {
|
||||
/// Maximum time to wait for the expected engines to register on the
|
||||
/// frontend transport.
|
||||
pub fn ready_timeout(&self) -> Duration {
|
||||
Duration::from_secs(self.engine_ready_timeout_secs)
|
||||
}
|
||||
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
pub fn shutdown_timeout(&self) -> Duration {
|
||||
Duration::from_secs(self.shutdown_timeout)
|
||||
}
|
||||
|
||||
/// Build the OpenAI-server config for the Python-bootstrap worker contract.
|
||||
///
|
||||
/// The resulting config binds the Python-supplied transport addresses and
|
||||
/// inherits an already open HTTP listener from the supervisor process.
|
||||
fn into_bootstrapped_config(
|
||||
self,
|
||||
listen_fd: i32,
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
coordinator_address: Option<String>,
|
||||
engine_count: usize,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
input_address,
|
||||
output_address,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
},
|
||||
coordinator_mode: match coordinator_address {
|
||||
Some(address) => CoordinatorMode::External { address },
|
||||
None => CoordinatorMode::None,
|
||||
},
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the OpenAI-server config for the managed `serve` path that still
|
||||
/// owns the startup handshake and binds its own HTTP listener.
|
||||
fn into_managed_config(
|
||||
self,
|
||||
listener_mode: HttpListenerMode,
|
||||
handshake_address: String,
|
||||
advertised_host: String,
|
||||
engine_count: usize,
|
||||
local_input_address: Option<String>,
|
||||
local_output_address: Option<String>,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_engine_ready_timeout_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||
}
|
||||
|
||||
fn parse_runtime_args_json(value: &str) -> Result<SharedRuntimeArgs, String> {
|
||||
let args: SharedRuntimeArgs = serde_json::from_str(value)
|
||||
.map_err(|e| format!("invalid JSON arguments: {}", e.as_report()))?;
|
||||
args.unsupported.check()?;
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
/// Arguments for running the Rust frontend as a Python-bootstrapped worker.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq)]
|
||||
#[educe(Debug)]
|
||||
pub struct FrontendArgs {
|
||||
/// Inherited listening socket file descriptor passed by the Python
|
||||
/// supervisor.
|
||||
#[arg(long)]
|
||||
pub listen_fd: i32,
|
||||
/// Frontend input ROUTER socket address that the Python engines will
|
||||
/// connect to.
|
||||
#[arg(long)]
|
||||
pub input_address: String,
|
||||
/// Frontend output PULL socket address that the Python engines will push
|
||||
/// responses to.
|
||||
#[arg(long)]
|
||||
pub output_address: String,
|
||||
/// Optional Python-owned frontend-side DP coordinator socket address for
|
||||
/// external coordinator mode in the bootstrapped frontend path, i.e.,
|
||||
/// `stats_update_address`.
|
||||
#[arg(long)]
|
||||
pub coordinator_address: Option<String>,
|
||||
/// Total number of data-parallel engines expected for this frontend.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub engine_count: usize,
|
||||
|
||||
/// Shared frontend arguments as one JSON object.
|
||||
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
|
||||
pub runtime: SharedRuntimeArgs,
|
||||
}
|
||||
|
||||
impl FrontendArgs {
|
||||
/// Convert the CLI arguments into the OpenAI server's runtime config.
|
||||
pub fn into_config(self) -> Config {
|
||||
self.runtime.into_bootstrapped_config(
|
||||
self.listen_fd,
|
||||
self.input_address,
|
||||
self.output_address,
|
||||
self.coordinator_address,
|
||||
self.engine_count,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments for the managed-engine mode that spawns Python on behalf of the
|
||||
/// user.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq)]
|
||||
#[educe(Debug)]
|
||||
#[command(override_usage = "vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]")]
|
||||
pub struct ServeArgs {
|
||||
/// Only launch the managed Python headless engine and do not start the Rust
|
||||
/// frontend.
|
||||
#[arg(long)]
|
||||
pub headless: bool,
|
||||
/// HTTP bind host for the OpenAI-compatible server.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
pub host: String,
|
||||
/// HTTP bind port for the OpenAI-compatible server.
|
||||
#[arg(long, default_value_t = 8000)]
|
||||
pub port: u16,
|
||||
/// Unix domain socket path. If set, host and port arguments are ignored.
|
||||
#[arg(long)]
|
||||
pub uds: Option<String>,
|
||||
|
||||
/// Flag to print debug information about CLI argument parsing and exit.
|
||||
#[educe(Debug(ignore))]
|
||||
#[arg(long, hide = true, env = "VLLM_RS_DEBUG_CLI")]
|
||||
pub debug_cli: bool,
|
||||
|
||||
/// Shared frontend arguments.
|
||||
#[command(flatten)]
|
||||
pub runtime: SharedRuntimeArgs,
|
||||
|
||||
/// Managed Python headless-engine arguments.
|
||||
#[command(flatten)]
|
||||
pub managed_engine: ManagedEngineArgs,
|
||||
}
|
||||
|
||||
impl ServeArgs {
|
||||
/// Build the OpenAI-server runtime config used after the managed Python
|
||||
/// engine starts.
|
||||
pub fn to_frontend_config(&self, handshake_address: String) -> Config {
|
||||
// Prefer IPC sockets for local engine input/output.
|
||||
let (local_input_address, local_output_address) =
|
||||
self.managed_engine.frontend_local_only().then(frontend_ipc_addresses).unzip();
|
||||
let listener_mode = match &self.uds {
|
||||
Some(path) => HttpListenerMode::BindUnix { path: path.clone() },
|
||||
None => HttpListenerMode::BindTcp {
|
||||
host: self.host.clone(),
|
||||
port: self.port,
|
||||
},
|
||||
};
|
||||
|
||||
self.runtime.clone().into_managed_config(
|
||||
listener_mode,
|
||||
handshake_address,
|
||||
self.managed_engine.handshake_host.clone(),
|
||||
self.managed_engine.data_parallel_size,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the managed Python-engine spawn configuration with the given
|
||||
/// handshake port.
|
||||
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
handshake_port,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
||||
fn frontend_ipc_addresses() -> (String, String) {
|
||||
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(std::env::temp_dir);
|
||||
let input_name = format!("vllm-rs-i-{}", Uuid::new_v4().simple());
|
||||
let output_name = format!("vllm-rs-o-{}", Uuid::new_v4().simple());
|
||||
|
||||
let input = preferred_base_path.join(input_name);
|
||||
let output = preferred_base_path.join(output_name);
|
||||
|
||||
(
|
||||
format!("ipc://{}", input.to_string_lossy()),
|
||||
format!("ipc://{}", output.to_string_lossy()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,905 @@
|
||||
use expect_test::expect;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
|
||||
|
||||
use super::{Cli, Command};
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_python_flags_with_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"../vllm/.venv/bin/python",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Serve(
|
||||
ServeArgs {
|
||||
headless: false,
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: Some(
|
||||
512,
|
||||
),
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "../vllm/.venv/bin/python",
|
||||
handshake_host: "127.0.0.1",
|
||||
handshake_port: None,
|
||||
data_parallel_size: 1,
|
||||
data_parallel_size_local: None,
|
||||
python_args: [
|
||||
"--dtype",
|
||||
"float16",
|
||||
],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_flags_without_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--quantization",
|
||||
"awq",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--quantization", "awq"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_explicit_deepseek_v32_renderer() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--tokenizer-mode",
|
||||
"deepseek_v32",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unknown_renderer_value() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--tokenizer-mode",
|
||||
"definitely_missing",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_flag_arg() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--allow-credentials"])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'true' for '--allow-credentials [<ALLOW_CREDENTIALS>]': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_no_flag_alias() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--no-enable-log-deltas",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'true' for '--enable-log-deltas [<ENABLE_LOG_DELTAS>]': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_accept_json() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--coordinator-address",
|
||||
"tcp://127.0.0.1:7000",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_count":2}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Frontend(
|
||||
FrontendArgs {
|
||||
listen_fd: 3,
|
||||
input_address: "ipc:///tmp/input.sock",
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
coordinator_address: Some(
|
||||
"tcp://127.0.0.1:7000",
|
||||
),
|
||||
engine_count: 1,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_applies_defaults() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_model_len, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","max_model_len":8192,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
|
||||
assert_eq!(
|
||||
args.runtime.tool_call_parser,
|
||||
ParserSelection::Explicit("hermes".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
args.runtime.reasoning_parser,
|
||||
ParserSelection::Explicit("qwen3_thinking".to_string())
|
||||
);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
assert_eq!(args.runtime.max_model_len, Some(8192));
|
||||
assert_eq!(args.runtime.shutdown_timeout, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_none_reasoning_parser() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--reasoning-parser",
|
||||
"none",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_ignores_unknown_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","uds":"/tmp/vllm.sock","nested_unknown":{"x":1}}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_noop_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_unsupported_fields() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_aggregates_multiple_unsupported_fields() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
- api_key
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_malformed_json() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B""#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B"' for '--args-json <JSON>': invalid JSON arguments: EOF while parsing an object at line 1 column 30
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_flags_before_model() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: the model must appear immediately after the command
|
||||
|
||||
Usage: vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_headless_mode() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--headless"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert!(args.headless);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_passthrough_flags_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2", "--dtype", "float16"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_multi_char_alias_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--",
|
||||
"-tp",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-tp", "2", "--dtype", "float16"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_frontend_arg_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--",
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--uds", "/tmp/vllm.sock"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--",
|
||||
"-dpr",
|
||||
"1",
|
||||
"-dpl",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-dpr", "1", "-dpl", "2"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_unknown_flags_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--foo", "bar"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_negative_value_without_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--num-gpu-blocks-override",
|
||||
"-1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--num-gpu-blocks-override", "-1"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_handshake_aliases() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--handshake-host",
|
||||
"10.99.48.128",
|
||||
"--handshake-port",
|
||||
"13345",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Serve(
|
||||
ServeArgs {
|
||||
headless: false,
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "python3",
|
||||
handshake_host: "10.99.48.128",
|
||||
handshake_port: Some(
|
||||
13345,
|
||||
),
|
||||
data_parallel_size: 4,
|
||||
data_parallel_size_local: None,
|
||||
python_args: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_data_parallel_primary_flags() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--data-parallel-address",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-rpc-port",
|
||||
"13345",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert!(!args.headless);
|
||||
assert_eq!(args.managed_engine.handshake_host, "10.99.48.128");
|
||||
assert_eq!(args.managed_engine.handshake_port, Some(13345));
|
||||
assert_eq!(args.managed_engine.data_parallel_size, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--handshake-host",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
|
||||
|
||||
let TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
} = &config.transport_mode
|
||||
else {
|
||||
panic!("expected handshake-owned transport");
|
||||
};
|
||||
|
||||
assert_eq!(handshake_address, "tcp://10.99.48.128:29550");
|
||||
assert_eq!(advertised_host, "10.99.48.128");
|
||||
assert_eq!(*engine_count, 4);
|
||||
assert!(
|
||||
local_input_address
|
||||
.as_deref()
|
||||
.is_some_and(|address| address.starts_with("ipc://"))
|
||||
);
|
||||
assert!(
|
||||
local_output_address
|
||||
.as_deref()
|
||||
.is_some_and(|address| address.starts_with("ipc://"))
|
||||
);
|
||||
assert_ne!(local_input_address, local_output_address);
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: HandshakeOwner {
|
||||
handshake_address: "tcp://10.99.48.128:29550",
|
||||
advertised_host: "10.99.48.128",
|
||||
engine_count: 4,
|
||||
ready_timeout: 600s,
|
||||
local_input_address: Some(
|
||||
"<ipc input>",
|
||||
),
|
||||
local_output_address: Some(
|
||||
"<ipc output>",
|
||||
),
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: handshake_address.clone(),
|
||||
advertised_host: advertised_host.clone(),
|
||||
engine_count: *engine_count,
|
||||
ready_timeout: *ready_timeout,
|
||||
local_input_address: Some("<ipc input>".to_string()),
|
||||
local_output_address: Some("<ipc output>".to_string()),
|
||||
},
|
||||
..config.clone()
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--data-parallel-address",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
"--data-parallel-size-local",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: HandshakeOwner {
|
||||
handshake_address: "tcp://10.99.48.128:29550",
|
||||
advertised_host: "10.99.48.128",
|
||||
engine_count: 4,
|
||||
ready_timeout: 600s,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_reject_legacy_handshake_flags() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
"--handshake-address",
|
||||
"tcp://127.0.0.1:62100",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("--handshake-address"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--coordinator-address",
|
||||
"tcp://127.0.0.1:7000",
|
||||
"--engine-count",
|
||||
"2",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: Bootstrapped {
|
||||
input_address: "ipc:///tmp/input.sock",
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
engine_count: 2,
|
||||
ready_timeout: 600s,
|
||||
},
|
||||
coordinator_mode: External {
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
fd: 3,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
|
||||
|
||||
assert_eq!(
|
||||
config.listener_mode,
|
||||
HttpListenerMode::BindUnix {
|
||||
path: "/tmp/vllm.sock".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,660 @@
|
||||
#![allow(clippy::doc_lazy_continuation)]
|
||||
|
||||
use std::fmt::Display;
|
||||
use std::str::FromStr;
|
||||
|
||||
use clap::Args;
|
||||
use clap::builder::{TypedValueParser, ValueParserFactory};
|
||||
use itertools::Itertools;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
/// Marker type for frontend-owned `serve` arguments that `vllm-rs` recognizes
|
||||
/// but does not support yet.
|
||||
///
|
||||
/// When passed as JSON args, it can be deserialized from any value, and
|
||||
/// serializes back to the original value.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct Unsupported(pub serde_json::Value);
|
||||
|
||||
impl FromStr for Unsupported {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(_s: &str) -> Result<Self, Self::Err> {
|
||||
Err("argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument."
|
||||
.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Marker type for no-op arguments that are accepted by the Rust frontend but
|
||||
/// have no effect.
|
||||
///
|
||||
/// When passed as JSON args, it can be deserialized from any value, but always
|
||||
/// serializes back to `null`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Noop;
|
||||
|
||||
impl<'de> Deserialize<'de> for Noop {
|
||||
fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Ok(Noop)
|
||||
}
|
||||
}
|
||||
|
||||
impl ValueParserFactory for Noop {
|
||||
type Parser = NoopValueParser;
|
||||
|
||||
fn value_parser() -> Self::Parser {
|
||||
NoopValueParser
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct NoopValueParser;
|
||||
|
||||
#[track_caller]
|
||||
fn noop_warn(arg: impl Display) {
|
||||
tracing::warn!("argument '{arg}' currently has no effect in Rust frontend, ignoring");
|
||||
}
|
||||
|
||||
impl TypedValueParser for NoopValueParser {
|
||||
type Value = Noop;
|
||||
|
||||
fn parse_ref(
|
||||
&self,
|
||||
_cmd: &clap::Command,
|
||||
arg: Option<&clap::Arg>,
|
||||
_value: &std::ffi::OsStr,
|
||||
) -> Result<Self::Value, clap::Error> {
|
||||
if let Some(arg) = arg {
|
||||
noop_warn(arg);
|
||||
}
|
||||
Ok(Noop)
|
||||
}
|
||||
}
|
||||
|
||||
/// Frontend-owned Python `serve` arguments that `vllm-rs` recognizes but does
|
||||
/// not support yet.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
#[command(next_help_heading = "Options not implemented in Rust frontend yet")]
|
||||
pub struct UnsupportedArgs {
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
top_level: TopLevelUnsupportedArgs,
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
engine: EngineUnsupportedArgs,
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
server: ServerUnsupportedArgs,
|
||||
}
|
||||
|
||||
impl UnsupportedArgs {
|
||||
/// Check whether any unsupported arguments are set, and if so, return an
|
||||
/// error listing them. Also warn about any no-op arguments that are set
|
||||
/// but will be ignored.
|
||||
pub(crate) fn check(&self) -> Result<(), String> {
|
||||
let value = serde_json::to_value(self).unwrap();
|
||||
let map = value.as_object().unwrap();
|
||||
let mut unsupported = Vec::new();
|
||||
|
||||
for (key, value) in map {
|
||||
if value.is_null() {
|
||||
noop_warn(key);
|
||||
} else {
|
||||
unsupported.push(key.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
if !unsupported.is_empty() {
|
||||
unsupported.sort_unstable();
|
||||
let bullets = unsupported.into_iter().map(|key| format!("- {key}")).join("\n");
|
||||
return Err(format!(
|
||||
"
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
{bullets}
|
||||
|
||||
Remove these arguments to continue."
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Frontend-owned Python `vllm serve` top-level arguments that `vllm-rs`
|
||||
/// recognizes but does not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.entrypoints.openai.cli_args.make_arg_parser(...)`
|
||||
/// - `vllm.entrypoints.cli.serve.ServeSubcommand.subparser_init(...)`
|
||||
///
|
||||
/// These are not part of `EngineArgs`, `AsyncEngineArgs`, `BaseFrontendArgs`,
|
||||
/// or `FrontendArgs`. They live on the `serve` command itself and control
|
||||
/// managed-engine / multi-process orchestration rather than the shared frontend
|
||||
/// runtime config.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct TopLevelUnsupportedArgs {
|
||||
/// How many API server processes to run. Defaults to data_parallel_size if
|
||||
/// not specified.
|
||||
#[arg(long, hide = true)]
|
||||
pub api_server_count: Option<Noop>,
|
||||
|
||||
/// Read CLI options from a config file. Must be a YAML with the following
|
||||
/// options: https://docs.vllm.ai/en/latest/configuration/serve_args.html
|
||||
#[arg(long)]
|
||||
pub config: Option<Unsupported>,
|
||||
|
||||
/// Launch a gRPC server instead of the HTTP OpenAI-compatible server.
|
||||
/// Requires: pip install vllm[grpc].
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub grpc: Option<Unsupported>,
|
||||
}
|
||||
|
||||
/// Frontend-owned Python engine arguments that `vllm-rs` recognizes but does
|
||||
/// not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.engine.arg_utils.EngineArgs.add_cli_args(...)`
|
||||
/// - `vllm.engine.arg_utils.AsyncEngineArgs.add_cli_args(...)`
|
||||
///
|
||||
/// These arguments are declared through the Python engine-args surface, but
|
||||
/// they are still frontend-owned: the API server / AsyncLLM layer reads them
|
||||
/// for tokenizer setup, request validation, routing, logging, and other
|
||||
/// frontend behavior, so Rust must recognize them rather than treating them as
|
||||
/// pure engine passthrough.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct EngineUnsupportedArgs {
|
||||
/// Name or path of the Hugging Face tokenizer to use. If unspecified, model
|
||||
/// name or path will be used.
|
||||
#[arg(long)]
|
||||
pub tokenizer: Option<Unsupported>,
|
||||
|
||||
/// Name or path of the Hugging Face config to use. If unspecified, model
|
||||
/// name or path will be used.
|
||||
#[arg(long)]
|
||||
pub hf_config_path: Option<Unsupported>,
|
||||
|
||||
/// Allowing API requests to read local images or videos from directories
|
||||
/// specified by the server file system. This is a security risk. Should
|
||||
/// only be enabled in trusted environments.
|
||||
#[arg(long)]
|
||||
pub allowed_local_media_path: Option<Unsupported>,
|
||||
|
||||
/// If set, only media URLs that belong to this domain can be used for
|
||||
/// multi-modal inputs.
|
||||
#[arg(long)]
|
||||
pub allowed_media_domains: Option<Unsupported>,
|
||||
|
||||
/// The specific revision to use for the tokenizer on the Hugging Face Hub.
|
||||
/// It can be a branch name, a tag name, or a commit id. If unspecified,
|
||||
/// will use the default version.
|
||||
#[arg(long)]
|
||||
pub tokenizer_revision: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in `SamplingParams`. The default value comes the default for
|
||||
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
|
||||
/// (output_length * vocab_size) logprobs are allowed to be returned and
|
||||
/// it may cause OOM.
|
||||
#[arg(long)]
|
||||
pub max_logprobs: Option<Unsupported>,
|
||||
|
||||
/// Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
/// `prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
/// output will contain token ids.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-skip-tokenizer-init",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub skip_tokenizer_init: Option<Unsupported>,
|
||||
|
||||
/// If `True`, enables passing text embeddings as inputs via the
|
||||
/// `prompt_embeds` key.
|
||||
///
|
||||
/// WARNING: The vLLM engine may crash if incorrect shape of embeddings is
|
||||
/// passed. Only enable this flag for trusted users!
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-prompt-embeds",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_prompt_embeds: Option<Unsupported>,
|
||||
|
||||
/// The token to use as HTTP bearer authorization for remote files. If
|
||||
/// `True`, will use the token generated when running `hf auth login`
|
||||
/// (stored in `~/.cache/huggingface/token`).
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub hf_token: Option<Unsupported>,
|
||||
|
||||
/// If a dictionary, contains arguments to be forwarded to the Hugging Face
|
||||
/// config. If a callable, it is called to update the HuggingFace config.
|
||||
#[arg(long)]
|
||||
pub hf_overrides: Option<Unsupported>,
|
||||
|
||||
/// The folder path to the generation config. Defaults to `"auto"`, the
|
||||
/// generation config will be loaded from model path. If set to `"vllm"`, no
|
||||
/// generation config is loaded, vLLM defaults will be used. If set to a
|
||||
/// folder path, the generation config will be loaded from the specified
|
||||
/// folder path. If `max_new_tokens` is specified in generation config,
|
||||
/// then it sets a server-wide limit on the number of output tokens for
|
||||
/// all requests.
|
||||
#[arg(long)]
|
||||
pub generation_config: Option<Unsupported>,
|
||||
|
||||
/// IOProcessor plugin name to load at model startup
|
||||
#[arg(long)]
|
||||
pub io_processor_plugin: Option<Unsupported>,
|
||||
|
||||
/// Path to a dynamically reasoning parser plugin that can be dynamically
|
||||
/// loaded and registered.
|
||||
#[arg(long)]
|
||||
pub reasoning_parser_plugin: Option<Unsupported>,
|
||||
|
||||
/// Rank of the data parallel group.
|
||||
#[arg(long, env = "VLLM_DP_RANK")]
|
||||
pub data_parallel_rank: Option<Unsupported>,
|
||||
|
||||
/// Whether to use "hybrid" DP LB mode. Applies only to online serving
|
||||
/// and when data_parallel_size > 0. Enables running an AsyncLLM
|
||||
/// and API server on a "per-node" basis where vLLM load balances
|
||||
/// between local data parallel ranks, but an external LB balances
|
||||
/// between vLLM nodes/replicas. Set explicitly in conjunction with
|
||||
/// --data-parallel-start-rank.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-data-parallel-hybrid-lb",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub data_parallel_hybrid_lb: Option<Unsupported>,
|
||||
|
||||
/// Whether to use "external" DP LB mode. Applies only to online serving
|
||||
/// and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"
|
||||
/// wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank
|
||||
/// is provided explicitly to vllm serve.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-data-parallel-external-lb",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub data_parallel_external_lb: Option<Unsupported>,
|
||||
|
||||
/// This feature is work in progress and no prefill optimization takes place
|
||||
/// with this flag enabled currently.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-kv-sharing-fast-prefill",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub kv_sharing_fast_prefill: Option<Unsupported>,
|
||||
|
||||
/// The maximum number of input items and options allowed per
|
||||
/// prompt for each modality.
|
||||
#[arg(long)]
|
||||
pub limit_mm_per_prompt: Option<Unsupported>,
|
||||
|
||||
/// Additional args passed to process media inputs, keyed by modalities.
|
||||
#[arg(long)]
|
||||
pub media_io_kwargs: Option<Unsupported>,
|
||||
|
||||
/// Arguments to be forwarded to the model's processor for multi-modal data,
|
||||
/// e.g., image processor.
|
||||
#[arg(long)]
|
||||
pub mm_processor_kwargs: Option<Unsupported>,
|
||||
|
||||
/// The size (in GiB) of the multi-modal processor cache.
|
||||
#[arg(long)]
|
||||
pub mm_processor_cache_gb: Option<Unsupported>,
|
||||
|
||||
/// Type of cache to use for the multi-modal preprocessor/mapper.
|
||||
#[arg(long)]
|
||||
pub mm_processor_cache_type: Option<Unsupported>,
|
||||
|
||||
/// If True, enable handling of LoRA adapters.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-lora",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_lora: Option<Unsupported>,
|
||||
|
||||
/// Dictionary mapping specific modalities to LoRA model paths.
|
||||
#[arg(long)]
|
||||
pub default_mm_loras: Option<Unsupported>,
|
||||
|
||||
/// Target URL to which OpenTelemetry traces will be sent.
|
||||
#[arg(long)]
|
||||
pub otlp_traces_endpoint: Option<Unsupported>,
|
||||
|
||||
/// It makes sense to set this only if `--otlp-traces-endpoint` is set.
|
||||
#[arg(long)]
|
||||
pub collect_detailed_traces: Option<Unsupported>,
|
||||
|
||||
/// The interval (or buffer size) for streaming in terms of token length.
|
||||
#[arg(long)]
|
||||
pub stream_interval: Option<Unsupported>,
|
||||
|
||||
/// Structured outputs configuration.
|
||||
#[arg(long)]
|
||||
pub structured_outputs_config: Option<Noop>,
|
||||
|
||||
/// Log aggregate rather than per-engine statistics when using data
|
||||
/// parallelism.
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub aggregate_engine_logging: Option<Unsupported>,
|
||||
}
|
||||
|
||||
/// Frontend-owned Python OpenAI server arguments that `vllm-rs` recognizes but
|
||||
/// does not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.entrypoints.openai.cli_args.BaseFrontendArgs`
|
||||
/// - `vllm.entrypoints.openai.cli_args.FrontendArgs`
|
||||
///
|
||||
/// These are not engine args. They belong to the Python OpenAI-compatible
|
||||
/// frontend / API-server layer itself, for example chat-template configuration,
|
||||
/// tool/frontend behavior, TLS / CORS / HTTP server settings, and other
|
||||
/// northbound server knobs.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct ServerUnsupportedArgs {
|
||||
/// LoRA modules configurations in either 'name=path' format or JSON format
|
||||
/// or JSON list format. Example (old format): `'name=path'` Example (new
|
||||
/// format): `{"name": "name", "path": "lora_path",
|
||||
/// "base_model_name": "id"}`
|
||||
#[arg(long)]
|
||||
pub lora_modules: Option<Unsupported>,
|
||||
|
||||
/// Whether to trust the chat template provided in the request. If False,
|
||||
/// the server will always use the chat template specified by
|
||||
/// `--chat-template` or the ones from tokenizer.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-trust-request-chat-template",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub trust_request_chat_template: Option<Unsupported>,
|
||||
|
||||
/// The role name to return if `request.add_generation_prompt=true`.
|
||||
#[arg(long)]
|
||||
pub response_role: Option<Unsupported>,
|
||||
|
||||
/// When `--max-logprobs` is specified, represents single tokens as
|
||||
/// strings of the form 'token_id:{token_id}' so that tokens that are not
|
||||
/// JSON-encodable can be identified.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-return-tokens-as-token-ids",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub return_tokens_as_token_ids: Option<Unsupported>,
|
||||
|
||||
/// Enable auto tool choice for supported models. Use `--tool-call-parser`
|
||||
/// to specify which parser to use.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-auto-tool-choice",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1,
|
||||
hide = true
|
||||
)]
|
||||
pub enable_auto_tool_choice: Option<Noop>,
|
||||
|
||||
/// If specified, exclude tool definitions in prompts when
|
||||
/// tool_choice='none'.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-exclude-tools-when-tool-choice-none",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub exclude_tools_when_tool_choice_none: Option<Unsupported>,
|
||||
|
||||
/// Special the tool parser plugin write to parse the model-generated tool
|
||||
/// into OpenAI API format, the name register in this plugin can be used in
|
||||
/// `--tool-call-parser`.
|
||||
#[arg(long)]
|
||||
pub tool_parser_plugin: Option<Unsupported>,
|
||||
|
||||
/// Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
/// Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
/// built-in demo tools (browser and Python code interpreter). WARNING:
|
||||
/// The `demo` Python tool executes model-generated code in Docker without
|
||||
/// network isolation by default. See the security guide for more
|
||||
/// information.
|
||||
#[arg(long)]
|
||||
pub tool_server: Option<Unsupported>,
|
||||
|
||||
/// Path to logging config JSON file for both vllm and uvicorn
|
||||
#[arg(long, /* env = "VLLM_LOGGING_CONFIG_PATH" */)]
|
||||
pub log_config_file: Option<Unsupported>,
|
||||
|
||||
/// Max number of prompt characters or prompt ID numbers being printed in
|
||||
/// log. The default of None means unlimited.
|
||||
#[arg(long)]
|
||||
pub max_log_len: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable prompt_tokens_details in usage.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-prompt-tokens-details",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_prompt_tokens_details: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable tracking server_load_metrics in the app state.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-server-load-tracking",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_server_load_tracking: Option<Noop>,
|
||||
|
||||
/// If set to True, including usage on every request.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-force-include-usage",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_force_include_usage: Option<Unsupported>,
|
||||
|
||||
/// Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
/// templates and other tokenizer configuration.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-tokenizer-info-endpoint",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_tokenizer_info_endpoint: Option<Unsupported>,
|
||||
|
||||
/// If set to True, log model outputs (generations).
|
||||
/// Requires `--enable-log-requests`. As with `--enable-log-requests`,
|
||||
/// information is only logged at INFO level at maximum.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-log-outputs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_log_outputs: Option<Unsupported>,
|
||||
|
||||
/// If set to False, output deltas will not be logged. Relevant only if
|
||||
/// --enable-log-outputs is set.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-log-deltas",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_log_deltas: Option<Unsupported>,
|
||||
|
||||
/// If set to True, log the stack trace of error responses
|
||||
#[arg(
|
||||
long,
|
||||
// env = "VLLM_SERVER_DEV_MODE",
|
||||
visible_alias = "no-log-error-stack",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub log_error_stack: Option<Unsupported>,
|
||||
|
||||
/// If set to True, only enable the Tokens In<>Out endpoint.
|
||||
/// This is intended for use in a Disaggregated Everything setup.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-tokens-only",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub tokens_only: Option<Unsupported>,
|
||||
|
||||
/// Log level for uvicorn.
|
||||
#[arg(long)]
|
||||
pub uvicorn_log_level: Option<Unsupported>,
|
||||
|
||||
/// Disable uvicorn access log.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-disable-uvicorn-access-log",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub disable_uvicorn_access_log: Option<Noop>,
|
||||
|
||||
/// Comma-separated list of endpoint paths to exclude from uvicorn access
|
||||
/// logs. This is useful to reduce log noise from high-frequency endpoints
|
||||
/// like health checks. Example: "/health,/metrics,/ping".
|
||||
/// When set, access logs for requests to these paths will be suppressed
|
||||
/// while keeping logs for other endpoints.
|
||||
#[arg(long)]
|
||||
pub disable_access_log_for_endpoints: Option<Noop>,
|
||||
|
||||
/// Allow credentials.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-allow-credentials",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub allow_credentials: Option<Unsupported>,
|
||||
|
||||
/// Allowed origins.
|
||||
#[arg(long)]
|
||||
pub allowed_origins: Option<Unsupported>,
|
||||
|
||||
/// Allowed methods.
|
||||
#[arg(long)]
|
||||
pub allowed_methods: Option<Unsupported>,
|
||||
|
||||
/// Allowed headers.
|
||||
#[arg(long)]
|
||||
pub allowed_headers: Option<Unsupported>,
|
||||
|
||||
/// If provided, the server will require one of these keys to be presented
|
||||
/// in the header.
|
||||
#[arg(long)]
|
||||
pub api_key: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL key file.
|
||||
#[arg(long)]
|
||||
pub ssl_keyfile: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL cert file.
|
||||
#[arg(long)]
|
||||
pub ssl_certfile: Option<Unsupported>,
|
||||
|
||||
/// The CA certificates file.
|
||||
#[arg(long)]
|
||||
pub ssl_ca_certs: Option<Unsupported>,
|
||||
|
||||
/// Refresh SSL Context when SSL certificate files change
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-ssl-refresh",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_ssl_refresh: Option<Unsupported>,
|
||||
|
||||
/// Whether client certificate is required (see stdlib ssl module's).
|
||||
#[arg(long)]
|
||||
pub ssl_cert_reqs: Option<Unsupported>,
|
||||
|
||||
/// SSL cipher suites for HTTPS (TLS 1.2 and below only).
|
||||
/// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
|
||||
#[arg(long)]
|
||||
pub ssl_ciphers: Option<Unsupported>,
|
||||
|
||||
/// FastAPI root_path when app is behind a path based routing proxy.
|
||||
#[arg(long)]
|
||||
pub root_path: Option<Unsupported>,
|
||||
|
||||
/// Additional ASGI middleware to apply to the app. We accept multiple
|
||||
/// --middleware arguments. The value should be an import path. If a
|
||||
/// function is provided, vLLM will add it to the server using
|
||||
/// `@app.middleware('http')`. If a class is provided, vLLM will
|
||||
/// add it to the server using `app.add_middleware()`.
|
||||
#[arg(long)]
|
||||
pub middleware: Option<Unsupported>,
|
||||
|
||||
/// If specified, API server will add X-Request-Id header to responses.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-request-id-headers",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_request_id_headers: Option<Unsupported>,
|
||||
|
||||
/// Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-disable-fastapi-docs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub disable_fastapi_docs: Option<Unsupported>,
|
||||
|
||||
/// Maximum size (bytes) of an incomplete HTTP event (header or body) for
|
||||
/// h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB).
|
||||
#[arg(long)]
|
||||
pub h11_max_incomplete_event_size: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of HTTP headers allowed in a request for h11 parser.
|
||||
/// Helps mitigate header abuse. Default: 256.
|
||||
#[arg(long)]
|
||||
pub h11_max_header_count: Option<Unsupported>,
|
||||
|
||||
/// Enable offline FastAPI documentation for air-gapped environments.
|
||||
/// Uses vendored static assets bundled with vLLM.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-offline-docs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_offline_docs: Option<Unsupported>,
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
use std::{env, fmt, process};
|
||||
|
||||
use time::UtcOffset;
|
||||
use time::macros::format_description;
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::Layer as _;
|
||||
use tracing_subscriber::filter::Targets;
|
||||
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
|
||||
use tracing_subscriber::fmt::time::FormatTime;
|
||||
use tracing_subscriber::fmt::{FmtContext, FormattedFields};
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
use tracing_subscriber::util::SubscriberInitExt as _;
|
||||
|
||||
const CYAN: &str = "\x1b[0;36m";
|
||||
const GREY: &str = "\x1b[90m";
|
||||
const GREEN: &str = "\x1b[32m";
|
||||
const YELLOW: &str = "\x1b[33m";
|
||||
const RED: &str = "\x1b[31m";
|
||||
const WHITE: &str = "\x1b[37m";
|
||||
const RESET: &str = "\x1b[0m";
|
||||
const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
|
||||
format_description!("[month]-[day] [hour]:[minute]:[second]");
|
||||
|
||||
const PROCESS_LABEL: &str = "RustFrontend";
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
|
||||
pub(crate) fn init_tracing() {
|
||||
let filter = build_targets_filter(
|
||||
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
|
||||
env::var("RUST_LOG").ok().as_deref(),
|
||||
);
|
||||
let formatter = VllmEventFormatter::new();
|
||||
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().event_format(formatter).with_filter(filter))
|
||||
.try_init();
|
||||
}
|
||||
|
||||
/// Build the CLI log filter by merging the vLLM-style default level with
|
||||
/// Rust-style target overrides.
|
||||
///
|
||||
/// Precedence:
|
||||
/// - Start from `VLLM_LOGGING_LEVEL` as the default level for all targets.
|
||||
/// - If `RUST_LOG` contains a global default level such as `warn`, it overrides
|
||||
/// `VLLM_LOGGING_LEVEL`.
|
||||
/// - Any explicit target directives in `RUST_LOG`, such as `hyper=info`, override whichever default
|
||||
/// level is active for those targets only.
|
||||
fn build_targets_filter(vllm_logging_level: Option<&str>, rust_log: Option<&str>) -> Targets {
|
||||
let mut filter =
|
||||
Targets::new().with_default(map_python_log_level(vllm_logging_level.unwrap_or("INFO")));
|
||||
|
||||
if let Some(rust_log) = rust_log
|
||||
&& !rust_log.is_empty()
|
||||
{
|
||||
let rust_log_targets: Targets = rust_log.parse().expect("failed to parse `RUST_LOG`");
|
||||
if let Some(default_level) = rust_log_targets.default_level() {
|
||||
filter = filter.with_default(default_level);
|
||||
}
|
||||
filter = filter.with_targets(rust_log_targets);
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct VllmLocalTimer {
|
||||
local_offset: UtcOffset,
|
||||
}
|
||||
|
||||
impl Default for VllmLocalTimer {
|
||||
fn default() -> Self {
|
||||
let local_offset = UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC);
|
||||
Self { local_offset }
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatTime for VllmLocalTimer {
|
||||
fn format_time(&self, w: &mut Writer<'_>) -> fmt::Result {
|
||||
let now = time::OffsetDateTime::now_utc().to_offset(self.local_offset);
|
||||
let formatted = now.format(VLLM_TIME_FORMAT).map_err(|_| fmt::Error)?;
|
||||
w.write_str(&formatted)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VllmEventFormatter {
|
||||
prefix: String,
|
||||
timer: VllmLocalTimer,
|
||||
}
|
||||
|
||||
impl VllmEventFormatter {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
prefix: format!("({} pid={})", PROCESS_LABEL, process::id()),
|
||||
timer: VllmLocalTimer::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_process_prefix(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
|
||||
write_colored(writer, ansi, Some(CYAN), &self.prefix)?;
|
||||
writer.write_char(' ')
|
||||
}
|
||||
|
||||
fn write_level(&self, writer: &mut Writer<'_>, level: &Level, ansi: bool) -> fmt::Result {
|
||||
let (text, color) = match *level {
|
||||
Level::TRACE => ("TRACE", WHITE),
|
||||
Level::DEBUG => ("DEBUG", WHITE),
|
||||
Level::INFO => ("INFO", GREEN),
|
||||
Level::WARN => ("WARNING", YELLOW),
|
||||
Level::ERROR => ("ERROR", RED),
|
||||
};
|
||||
write_colored(writer, ansi, Some(color), text)
|
||||
}
|
||||
|
||||
fn write_timestamp(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
|
||||
if ansi {
|
||||
writer.write_str(GREY)?;
|
||||
}
|
||||
if self.timer.format_time(writer).is_err() {
|
||||
writer.write_str("<unknown time>")?;
|
||||
}
|
||||
if ansi {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_location(
|
||||
&self,
|
||||
writer: &mut Writer<'_>,
|
||||
file: Option<&str>,
|
||||
line: Option<u32>,
|
||||
full_path: bool,
|
||||
ansi: bool,
|
||||
) -> fmt::Result {
|
||||
let Some(file) = file else {
|
||||
return Ok(());
|
||||
};
|
||||
let file = if full_path {
|
||||
file
|
||||
} else {
|
||||
shorten_file_path(file)
|
||||
};
|
||||
if ansi {
|
||||
writer.write_str(GREY)?;
|
||||
}
|
||||
match line {
|
||||
Some(line) => write!(writer, "[{file}:{line}]")?,
|
||||
None => write!(writer, "[{file}]")?,
|
||||
}
|
||||
if ansi {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_scope<S, N>(&self, ctx: &FmtContext<'_, S, N>, writer: &mut Writer<'_>) -> fmt::Result
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
N: for<'writer> FormatFields<'writer> + 'static,
|
||||
{
|
||||
let Some(scope) = ctx.event_scope() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut seen = false;
|
||||
for span in scope.from_root() {
|
||||
if seen {
|
||||
writer.write_str(":")?;
|
||||
}
|
||||
seen = true;
|
||||
writer.write_str(span.metadata().name())?;
|
||||
|
||||
let ext = span.extensions();
|
||||
if let Some(fields) = ext.get::<FormattedFields<N>>()
|
||||
&& !fields.is_empty()
|
||||
{
|
||||
write!(writer, "{{{fields}}}")?;
|
||||
}
|
||||
}
|
||||
|
||||
if seen {
|
||||
writer.write_str(": ")?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for VllmEventFormatter
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
N: for<'writer> FormatFields<'writer> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
let meta = event.metadata();
|
||||
let ansi = writer.has_ansi_escapes();
|
||||
|
||||
self.write_process_prefix(&mut writer, ansi)?;
|
||||
self.write_level(&mut writer, meta.level(), ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
self.write_timestamp(&mut writer, ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
// Use the full file path only when DEBUG (or more verbose) is enabled anywhere,
|
||||
// independent of the level of this particular event. Filenames alone are often
|
||||
// ambiguous, but full paths are too noisy for normal INFO-level operation.
|
||||
let full_path = LevelFilter::current() >= LevelFilter::DEBUG;
|
||||
self.write_location(&mut writer, meta.file(), meta.line(), full_path, ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
self.write_scope(ctx, &mut writer)?;
|
||||
ctx.format_fields(writer.by_ref(), event)?;
|
||||
writer.write_char('\n')
|
||||
}
|
||||
}
|
||||
|
||||
/// Shorten a source file path for log output while preserving enough context
|
||||
/// for common Rust entrypoint and module filenames.
|
||||
///
|
||||
/// - For `mod.rs`, keep the parent directory as `parent/mod.rs`.
|
||||
/// - For `src/lib.rs` and `src/main.rs`, keep one additional component as `crate/src/lib.rs` or
|
||||
/// `crate/src/main.rs` when available.
|
||||
/// - Other files are displayed as just the basename.
|
||||
fn shorten_file_path(file: &str) -> &str {
|
||||
let mut parts = file.rsplit('/');
|
||||
let name = parts.next().unwrap_or(file);
|
||||
let parent = parts.next();
|
||||
let grandparent = parts.next();
|
||||
|
||||
let Some(parent) = parent else {
|
||||
return file;
|
||||
};
|
||||
|
||||
if name == "mod.rs" {
|
||||
return &file[file.len() - parent.len() - 1 - name.len()..];
|
||||
}
|
||||
|
||||
if !matches!(name, "lib.rs" | "main.rs") || parent != "src" {
|
||||
return name;
|
||||
}
|
||||
let Some(grandparent) = grandparent else {
|
||||
return file;
|
||||
};
|
||||
|
||||
&file[file.len() - grandparent.len() - 1 - parent.len() - 1 - name.len()..]
|
||||
}
|
||||
|
||||
fn write_colored(
|
||||
writer: &mut Writer<'_>,
|
||||
ansi: bool,
|
||||
color: Option<&str>,
|
||||
text: &str,
|
||||
) -> fmt::Result {
|
||||
if ansi {
|
||||
if let Some(color) = color {
|
||||
writer.write_str(color)?;
|
||||
}
|
||||
writer.write_str(text)?;
|
||||
if color.is_some() {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
writer.write_str(text)
|
||||
}
|
||||
|
||||
/// Map a Python logging level name to the corresponding Rust tracing level.
|
||||
fn map_python_log_level(level: &str) -> LevelFilter {
|
||||
match level.to_ascii_uppercase().as_str() {
|
||||
"CRITICAL" | "FATAL" => LevelFilter::ERROR,
|
||||
"ERROR" => LevelFilter::ERROR,
|
||||
"WARNING" | "WARN" => LevelFilter::WARN,
|
||||
"INFO" => LevelFilter::INFO,
|
||||
"DEBUG" => LevelFilter::DEBUG,
|
||||
"NOTSET" => LevelFilter::TRACE,
|
||||
_ => LevelFilter::INFO,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rust_log_target_overrides_are_merged_with_vllm_default_level() {
|
||||
let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error"));
|
||||
|
||||
assert_eq!(filter.to_string(), "tower=error,hyper=warn,debug");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rust_log_default_level_overrides_vllm_default_level() {
|
||||
let filter = build_targets_filter(Some("DEBUG"), Some("warn,hyper=info"));
|
||||
|
||||
assert_eq!(filter.to_string(), "hyper=info,warn");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_vllm_level_falls_back_to_info() {
|
||||
let filter = build_targets_filter(Some("bogus"), None);
|
||||
|
||||
assert_eq!(filter.to_string(), "info");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_path_uses_filename_for_non_ambiguous_files() {
|
||||
assert_eq!(shorten_file_path("src/cmd/src/logging.rs"), "logging.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/lib.rs"), "lib.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/main.rs"), "main.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/xmod.rs"), "xmod.rs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_path_keeps_more_context_for_common_entrypoint_filenames() {
|
||||
assert_eq!(shorten_file_path("src/lib.rs"), "src/lib.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/lib.rs"), "chat/src/lib.rs");
|
||||
assert_eq!(shorten_file_path("src/cmd/src/main.rs"), "cmd/src/main.rs");
|
||||
assert_eq!(shorten_file_path("mod.rs"), "mod.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/tool/mod.rs"), "tool/mod.rs");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
mod cli;
|
||||
mod logging;
|
||||
|
||||
use std::env;
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use vllm_managed_engine::ManagedEngineHandle;
|
||||
|
||||
use crate::cli::{Cli, Command};
|
||||
|
||||
const TOKIO_WORKER_THREADS_ENV: &str = "TOKIO_WORKER_THREADS";
|
||||
const DEFAULT_MAX_TOKIO_WORKER_THREADS: usize = 32;
|
||||
|
||||
/// Cap the default number of Tokio worker threads if the user did not
|
||||
/// explicitly set `TOKIO_WORKER_THREADS` to avoid spawning too many threads on
|
||||
/// machines with a large number of CPUs, which may lead to excessive context
|
||||
/// switching and degraded performance.
|
||||
fn tokio_worker_threads() -> Option<usize> {
|
||||
if env::var_os(TOKIO_WORKER_THREADS_ENV).is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
std::thread::available_parallelism()
|
||||
.map(|parallelism| {
|
||||
let available = parallelism.get();
|
||||
let worker_threads = available.min(DEFAULT_MAX_TOKIO_WORKER_THREADS);
|
||||
if worker_threads < available {
|
||||
info!(
|
||||
available_parallelism = available,
|
||||
capped_worker_threads = worker_threads,
|
||||
"capping tokio worker threads, set {TOKIO_WORKER_THREADS_ENV} to override"
|
||||
);
|
||||
}
|
||||
worker_threads
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Reason that caused a managed `serve` session to stop.
|
||||
#[derive(Debug)]
|
||||
enum ShutdownReason {
|
||||
Signal,
|
||||
Server(anyhow::Error),
|
||||
EngineExited(ExitStatus),
|
||||
}
|
||||
|
||||
/// Cancellation token tripped by Ctrl-C or SIGTERM.
|
||||
fn shutdown_signal() -> CancellationToken {
|
||||
let token = CancellationToken::new();
|
||||
let shutdown = token.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let ctrl_c = async {
|
||||
tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
|
||||
};
|
||||
|
||||
let sigterm = async {
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("failed to install SIGTERM signal handler")
|
||||
.recv()
|
||||
.await;
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => info!("received shutdown signal (Ctrl-C), shutting down..."),
|
||||
_ = sigterm => info!("received shutdown signal (SIGTERM), shutting down..."),
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
});
|
||||
|
||||
token
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
logging::init_tracing();
|
||||
let cli = Cli::parse();
|
||||
|
||||
let mut runtime = tokio::runtime::Builder::new_multi_thread();
|
||||
runtime.enable_all();
|
||||
if let Some(worker_threads) = tokio_worker_threads() {
|
||||
runtime.worker_threads(worker_threads);
|
||||
}
|
||||
|
||||
runtime
|
||||
.build()
|
||||
.context("failed to build Tokio runtime")?
|
||||
.block_on(async_main(cli))
|
||||
}
|
||||
|
||||
async fn async_main(cli: Cli) -> Result<()> {
|
||||
match cli.command {
|
||||
Command::Frontend(args) => vllm_server::serve(args.into_config(), shutdown_signal()).await,
|
||||
Command::Serve(args) => {
|
||||
let handshake_port = args.managed_engine.resolve_handshake_port()?;
|
||||
|
||||
if args.managed_engine.data_parallel_size_local == Some(0) {
|
||||
if args.headless {
|
||||
bail!("cannot combine `--headless` with `--data-parallel-size-local 0`");
|
||||
}
|
||||
|
||||
let handshake_address = args.managed_engine.handshake_address(handshake_port);
|
||||
info!(
|
||||
%handshake_address,
|
||||
engine_count = args.managed_engine.data_parallel_size,
|
||||
"running Rust frontend without a managed local Python engine"
|
||||
);
|
||||
let config = args.to_frontend_config(handshake_address);
|
||||
return vllm_server::serve(config, shutdown_signal()).await;
|
||||
}
|
||||
|
||||
let shutdown_timeout = args.runtime.shutdown_timeout();
|
||||
let engine_config = args.to_managed_engine_config(handshake_port);
|
||||
let handshake_address = engine_config.handshake_address();
|
||||
|
||||
let engine = ManagedEngineHandle::spawn(engine_config)
|
||||
.await
|
||||
.context("failed to start managed Python headless engine")?;
|
||||
|
||||
let shutdown = shutdown_signal();
|
||||
|
||||
let mut serve_task = if args.headless {
|
||||
info!("running managed Python headless engine without Rust frontend");
|
||||
let shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
shutdown.cancelled().await;
|
||||
Ok(())
|
||||
})
|
||||
} else {
|
||||
let config = args.to_frontend_config(handshake_address);
|
||||
let shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = vllm_server::serve(config, shutdown).await;
|
||||
if result.is_ok() {
|
||||
info!("OpenAI server shut down gracefully");
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
let shutdown_reason = tokio::select! {
|
||||
biased;
|
||||
|
||||
// Received shutdown signal via Ctrl-C or SIGTERM.
|
||||
_ = shutdown.cancelled() => ShutdownReason::Signal,
|
||||
|
||||
// Engine process exited unexpectedly.
|
||||
status = engine.wait_for_exit() => {
|
||||
warn!(%status, "managed Python headless engine exited, shutting down...");
|
||||
ShutdownReason::EngineExited(status)
|
||||
}
|
||||
|
||||
// Serve task exited unexpectedly.
|
||||
serve_result = &mut serve_task => {
|
||||
let serve_result = serve_result.context("serve task join failed")?;
|
||||
match serve_result {
|
||||
Ok(()) => ShutdownReason::Server(anyhow!("OpenAI server shut down unexpectedly without error")),
|
||||
Err(error) => ShutdownReason::Server(error),
|
||||
}
|
||||
}
|
||||
};
|
||||
// Regardless of the shutdown reason, broadcast shutdown signal here to ensure
|
||||
// that all serving tasks are notified.
|
||||
shutdown.cancel();
|
||||
|
||||
// Shutdown begins. Terminate the managed engine first.
|
||||
engine.shutdown(shutdown_timeout).await?;
|
||||
info!("managed engine shut down gracefully");
|
||||
// Wait for the API server to shut down gracefully by draining in-flight
|
||||
// requests.
|
||||
if !matches!(shutdown_reason, ShutdownReason::Server(_)) {
|
||||
serve_task.await.context("serve task join failed")??;
|
||||
}
|
||||
|
||||
match shutdown_reason {
|
||||
ShutdownReason::Signal => Ok(()),
|
||||
ShutdownReason::Server(error) => {
|
||||
Err(error.context("OpenAI server shut down unexpectedly"))
|
||||
}
|
||||
ShutdownReason::EngineExited(status) => Err(anyhow!(
|
||||
"managed Python headless engine exited unexpectedly with status {status}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
[package]
|
||||
name = "vllm-engine-core-client"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[features]
|
||||
test-util = ["dep:tempfile"]
|
||||
|
||||
[dependencies]
|
||||
arc-swap.workspace = true
|
||||
bytemuck.workspace = true
|
||||
byteorder.workspace = true
|
||||
bytes.workspace = true
|
||||
easy-ext.workspace = true
|
||||
enum-as-inner.workspace = true
|
||||
futures.workspace = true
|
||||
half.workspace = true
|
||||
hex.workspace = true
|
||||
itertools.workspace = true
|
||||
parking_lot.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
rmpv.workspace = true
|
||||
serde.workspace = true
|
||||
serde_default.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_repr.workspace = true
|
||||
serde_tuple.workspace = true
|
||||
serde_with.workspace = true
|
||||
task-local.workspace = true
|
||||
tempfile = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
vllm-metrics.workspace = true
|
||||
zeromq.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
hex.workspace = true
|
||||
tempfile.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,53 @@
|
||||
# Engine-Core Smoke Tests
|
||||
|
||||
Start headless `vllm`:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--enable-sleep-mode \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
|
||||
smoke:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1 \
|
||||
--skip-sleep-wake
|
||||
```
|
||||
|
||||
Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
|
||||
longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
|
||||
The Rust client decodes those payloads into semantic per-position records rather than exposing the
|
||||
raw ndarray/tensor wire shape.
|
||||
|
||||
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
|
||||
@@ -0,0 +1,190 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::{
|
||||
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
|
||||
const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 120)]
|
||||
output_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
max_tokens: u32,
|
||||
#[arg(long, default_value_t = 2)]
|
||||
logprobs: i32,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
prompt_logprobs: i32,
|
||||
#[arg(long, default_value_t = 96)]
|
||||
prompt_repeats: usize,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock should be after unix epoch")
|
||||
.as_nanos();
|
||||
format!("rust-engine-core-logprobs-{nanos}")
|
||||
}
|
||||
|
||||
fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
|
||||
let repeats = prompt_repeats.max(1);
|
||||
BASE_PROMPT_TOKEN_IDS.repeat(repeats)
|
||||
}
|
||||
|
||||
fn build_request(
|
||||
request_id: String,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
max_tokens: u32,
|
||||
logprobs: i32,
|
||||
prompt_logprobs: i32,
|
||||
client_index: u32,
|
||||
) -> EngineCoreRequest {
|
||||
EngineCoreRequest {
|
||||
request_id,
|
||||
prompt_token_ids: Some(prompt_token_ids),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
logprobs: Some(logprobs),
|
||||
prompt_logprobs: Some(prompt_logprobs),
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 0.0,
|
||||
client_index,
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_final_output(
|
||||
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
|
||||
) -> Result<EngineCoreStreamOutput> {
|
||||
while let Some(output) = stream.next().await {
|
||||
let output = output.context("failed to receive engine-core output")?;
|
||||
if output.finished() {
|
||||
return Ok(output);
|
||||
}
|
||||
}
|
||||
bail!("request stream ended without a final output")
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(args.output_timeout_secs);
|
||||
let request_id = unique_request_id();
|
||||
let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let request = build_request(
|
||||
request_id.clone(),
|
||||
prompt_token_ids.clone(),
|
||||
args.max_tokens,
|
||||
args.logprobs,
|
||||
args.prompt_logprobs,
|
||||
args.client_index,
|
||||
);
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt_len={}", prompt_token_ids.len());
|
||||
println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
|
||||
println!("prompt_repeats={}", args.prompt_repeats);
|
||||
println!("requested_logprobs={}", args.logprobs);
|
||||
println!("requested_prompt_logprobs={}", args.prompt_logprobs);
|
||||
|
||||
let stream = client.call(request).await.context("failed to submit engine-core request")?;
|
||||
let output = timeout(output_timeout, wait_for_final_output(stream))
|
||||
.await
|
||||
.context("timed out waiting for final output")??;
|
||||
|
||||
let finish_reason = output.finish_reason;
|
||||
let token_ids = output.new_token_ids.clone();
|
||||
let logprobs = output
|
||||
.new_logprobs
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded sample logprobs")?;
|
||||
let prompt_logprobs = output
|
||||
.new_prompt_logprobs_tensors
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded prompt logprobs")?;
|
||||
|
||||
println!("token_ids={token_ids:?}");
|
||||
println!("finish_reason={finish_reason:?}");
|
||||
println!("new_logprobs={logprobs:#?}");
|
||||
println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
if finish_reason != Some(EngineCoreFinishReason::Length) {
|
||||
bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
|
||||
}
|
||||
if token_ids.is_empty() {
|
||||
bail!("engine returned no generated token ids");
|
||||
}
|
||||
if logprobs.is_empty() {
|
||||
bail!("decoded logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.is_empty() {
|
||||
bail!("decoded prompt logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
|
||||
bail!(
|
||||
"prompt logprobs rows look too short: prompt_len={}, rows={}",
|
||||
prompt_token_ids.len(),
|
||||
prompt_logprobs.len()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Expected initial result of is_sleeping() before running the smoke steps."
|
||||
)]
|
||||
expected_is_sleeping: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_running_requests: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_external: bool,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
sleep_level: u32,
|
||||
#[arg(long, default_value = "abort")]
|
||||
sleep_mode: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
|
||||
)]
|
||||
skip_sleep_wake: bool,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let initial_is_sleeping =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping utility")?;
|
||||
|
||||
println!("is_sleeping(initial)={initial_is_sleeping}");
|
||||
|
||||
if initial_is_sleeping != args.expected_is_sleeping {
|
||||
bail!(
|
||||
"unexpected initial is_sleeping state: expected {}, got {}",
|
||||
args.expected_is_sleeping,
|
||||
initial_is_sleeping
|
||||
);
|
||||
}
|
||||
|
||||
let reset_prefix_cache = client
|
||||
.reset_prefix_cache(args.reset_running_requests, args.reset_external)
|
||||
.await
|
||||
.context("failed to call reset_prefix_cache utility")?;
|
||||
println!("reset_prefix_cache={reset_prefix_cache}");
|
||||
|
||||
client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
|
||||
println!("reset_mm_cache=ok");
|
||||
|
||||
client
|
||||
.reset_encoder_cache()
|
||||
.await
|
||||
.context("failed to call reset_encoder_cache utility")?;
|
||||
println!("reset_encoder_cache=ok");
|
||||
|
||||
if args.skip_sleep_wake {
|
||||
println!("sleep_wake=skipped");
|
||||
} else {
|
||||
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
|
||||
format!(
|
||||
"failed to call sleep utility with level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
)
|
||||
})?;
|
||||
println!(
|
||||
"sleep=ok level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
);
|
||||
|
||||
let sleeping_after_sleep =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
|
||||
println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
|
||||
|
||||
if !sleeping_after_sleep {
|
||||
bail!("engine should report sleeping=true after sleep()");
|
||||
}
|
||||
|
||||
client.wake_up(None).await.context("failed to call wake_up utility")?;
|
||||
println!("wake_up=ok");
|
||||
|
||||
let sleeping_after_wake =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
|
||||
println!("is_sleeping(after_wake)={sleeping_after_wake}");
|
||||
|
||||
if sleeping_after_wake {
|
||||
bail!("engine should report sleeping=false after wake_up()");
|
||||
}
|
||||
}
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{join_all, try_join_all};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::{
|
||||
EngineCoreRequest, EngineCoreRequestType, EngineCoreUtilityRequest, ModelDtype,
|
||||
};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
pub(crate) mod imp;
|
||||
mod state;
|
||||
mod stream;
|
||||
|
||||
pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
|
||||
|
||||
/// How the frontend acquires its request/response transport with Python
|
||||
/// `EngineCoreProc`s.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TransportMode {
|
||||
/// The Rust process owns the startup handshake and allocates or binds the
|
||||
/// frontend transport addresses itself before replying to engine
|
||||
/// `HELLO` messages.
|
||||
HandshakeOwner {
|
||||
/// Shared handshake endpoint that engines dial during startup.
|
||||
handshake_address: String,
|
||||
/// Host/IP that engines should use to connect back to the frontend
|
||||
/// transport sockets.
|
||||
advertised_host: String,
|
||||
/// Total number of engines expected to join this transport.
|
||||
engine_count: usize,
|
||||
/// Maximum time to wait for each startup phase to complete.
|
||||
ready_timeout: Duration,
|
||||
/// Optional explicit bind address for the input ROUTER socket.
|
||||
local_input_address: Option<String>,
|
||||
/// Optional explicit bind address for the output PULL socket.
|
||||
local_output_address: Option<String>,
|
||||
},
|
||||
|
||||
/// The Python supervisor has already chosen the frontend transport
|
||||
/// addresses, and the Rust process only needs to bind them and wait for
|
||||
/// engine registration frames.
|
||||
Bootstrapped {
|
||||
/// Input ROUTER socket address that engines will connect to for
|
||||
/// requests.
|
||||
input_address: String,
|
||||
/// Output PULL socket address that engines will connect to for
|
||||
/// responses.
|
||||
output_address: String,
|
||||
/// Total number of engines expected to register on this transport.
|
||||
engine_count: usize,
|
||||
/// Maximum time to wait for all expected engines to register.
|
||||
ready_timeout: Duration,
|
||||
},
|
||||
}
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CoordinatorMode {
|
||||
/// Run the Rust in-process coordinator for managed `serve` deployments.
|
||||
InProc,
|
||||
/// Connect to an external coordinator owned by another process.
|
||||
External { address: String },
|
||||
}
|
||||
|
||||
/// Configuration for connecting a Rust frontend client to an already running
|
||||
/// Python `EngineCoreProc`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct EngineCoreClientConfig {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
/// Frontend-side coordinator behavior, or `None` when requests should flow
|
||||
/// directly to engines without any coordinator involvement.
|
||||
pub coordinator_mode: Option<CoordinatorMode>,
|
||||
/// Model name used for frontend-side metrics labels.
|
||||
pub model_name: String,
|
||||
/// Frontend client index stamped onto every request.
|
||||
pub client_index: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreClientConfig {
|
||||
/// Create a new client config with the given handshake address, expecting a
|
||||
/// single engine, and default values for all other fields.
|
||||
pub fn new_single(handshake_address: impl Into<String>) -> Self {
|
||||
Self {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: handshake_address.into(),
|
||||
advertised_host: "127.0.0.1".to_string(),
|
||||
engine_count: 1,
|
||||
ready_timeout: Duration::from_secs(30),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: String::new(),
|
||||
client_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the model name used by frontend-side metrics and diagnostics.
|
||||
pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
|
||||
self.model_name = model_name.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the client index stamped onto every outgoing request.
|
||||
pub fn with_client_index(mut self, client_index: u32) -> Self {
|
||||
self.client_index = client_index;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the optional coordinator mode for this client config.
|
||||
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
|
||||
self.coordinator_mode = coordinator_mode;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the locally bound input/output addresses for handshake-owned
|
||||
/// transport mode.
|
||||
///
|
||||
/// This is primarily used by tests that want deterministic IPC endpoints
|
||||
/// while still exercising the handshake-owned startup path.
|
||||
pub fn with_local_input_output_addresses(
|
||||
mut self,
|
||||
local_input_address: Option<String>,
|
||||
local_output_address: Option<String>,
|
||||
) -> Self {
|
||||
let TransportMode::HandshakeOwner {
|
||||
local_input_address: current_input,
|
||||
local_output_address: current_output,
|
||||
..
|
||||
} = &mut self.transport_mode
|
||||
else {
|
||||
panic!("local input/output overrides are only valid in handshake-owned mode");
|
||||
};
|
||||
*current_input = local_input_address;
|
||||
*current_output = local_output_address;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// The reason a request stream is being aborted when its output stream is
|
||||
/// dropped.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AbortCause {
|
||||
/// The consumer dropped the stream before the request reached a terminal
|
||||
/// engine output.
|
||||
#[default]
|
||||
DroppedStream,
|
||||
/// The frontend matched a stop string locally and intentionally stopped
|
||||
/// consuming the stream.
|
||||
StopStringMatched,
|
||||
}
|
||||
|
||||
task_local::task_local! {
|
||||
static ABORT_CAUSE: AbortCause;
|
||||
}
|
||||
|
||||
impl AbortCause {
|
||||
/// Return the abort cause currently associated with this task, or
|
||||
/// [`AbortCause::DroppedStream`] by default.
|
||||
pub fn current() -> Self {
|
||||
ABORT_CAUSE.try_get().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Drop one value while marking the drop as happening for this abort cause.
|
||||
pub fn drop_as<T>(self, value: T) {
|
||||
ABORT_CAUSE.sync_scope(self, move || drop(value));
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal auto-abort work item sent from stream `Drop` handlers to the abort
|
||||
/// worker.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct AbortRequest {
|
||||
request_id: String,
|
||||
cause: AbortCause,
|
||||
}
|
||||
|
||||
/// Default ZMQ-based implementation that talks directly to a Python
|
||||
/// `EngineCoreProc`.
|
||||
pub struct EngineCoreClient {
|
||||
config: EngineCoreClientConfig,
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
engines: Vec<ConnectedEngine>,
|
||||
inner: Arc<ClientInner>,
|
||||
coordinator: Option<CoordinatorHandle>,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
|
||||
// Background tasks
|
||||
output_task: AbortOnDropHandle<()>,
|
||||
dispatcher_task: AbortOnDropHandle<()>,
|
||||
abort_task: AbortOnDropHandle<()>,
|
||||
coordinator_output_task: Option<AbortOnDropHandle<()>>,
|
||||
coordinator_task: Option<AbortOnDropHandle<()>>,
|
||||
}
|
||||
|
||||
impl EngineCoreClient {
|
||||
/// Connect to Python `EngineCoreProc`s using the configured
|
||||
/// transport/coordinator modes.
|
||||
///
|
||||
/// In handshake-owned mode this method drives the full engine startup
|
||||
/// handshake. In bootstrapped mode it binds the provided frontend
|
||||
/// sockets and waits for the expected engine registration frames.
|
||||
pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
|
||||
let connected = match &config.transport_mode {
|
||||
TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
} => {
|
||||
let enable_inproc_coordinator = match config.coordinator_mode {
|
||||
None => false,
|
||||
Some(CoordinatorMode::InProc) => true,
|
||||
Some(CoordinatorMode::External { .. }) => {
|
||||
return Err(Error::UnsupportedExternalCoordinator);
|
||||
}
|
||||
};
|
||||
|
||||
transport::connect_handshake(
|
||||
handshake_address,
|
||||
*engine_count,
|
||||
advertised_host,
|
||||
local_input_address.as_deref(),
|
||||
local_output_address.as_deref(),
|
||||
enable_inproc_coordinator,
|
||||
*ready_timeout,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
TransportMode::Bootstrapped {
|
||||
input_address,
|
||||
output_address,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
} => {
|
||||
if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
|
||||
panic!("cannot use in-process coordinator with bootstrapped transport mode")
|
||||
}
|
||||
|
||||
transport::connect_bootstrapped(
|
||||
input_address,
|
||||
output_address,
|
||||
*engine_count,
|
||||
*ready_timeout,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
|
||||
Self::from_connected(config, connected).await
|
||||
}
|
||||
|
||||
/// Create a new client instance from the connected transport state after
|
||||
/// the startup handshake completes.
|
||||
async fn from_connected(
|
||||
config: EngineCoreClientConfig,
|
||||
connected: transport::ConnectedTransport,
|
||||
) -> Result<Self> {
|
||||
let (output_tx, output_rx) = mpsc::channel(64);
|
||||
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
|
||||
let engines = connected.engines;
|
||||
let inner = Arc::new(ClientInner::new(
|
||||
connected.input_send,
|
||||
config.model_name.clone(),
|
||||
&engines,
|
||||
));
|
||||
let output_task = AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
connected.output_socket,
|
||||
output_tx,
|
||||
)));
|
||||
let dispatcher_task = AbortOnDropHandle::new(tokio::spawn(run_output_dispatcher_loop(
|
||||
inner.clone(),
|
||||
output_rx,
|
||||
)));
|
||||
let abort_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(run_abort_loop(inner.clone(), abort_rx)));
|
||||
|
||||
// If any engine reported a dp_stats_address in its ready response, use it
|
||||
// as the external coordinator address.
|
||||
let dp_stats_address: Option<String> = engines
|
||||
.iter()
|
||||
.filter_map(|e| e.ready_response.as_ref())
|
||||
.find_map(|r| r.dp_stats_address.clone());
|
||||
|
||||
let (coordinator, coordinator_output_task, coordinator_task) =
|
||||
if let Some(coordinator_transport) = connected.coordinator {
|
||||
let (handle, runner) =
|
||||
CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
|
||||
let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
|
||||
let coordinator_output_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
coordinator_transport.output_socket,
|
||||
coordinator_output_tx,
|
||||
)));
|
||||
let coordinator_task = AbortOnDropHandle::new(tokio::spawn(
|
||||
runner.run(coordinator_output_rx, inner.clone()),
|
||||
));
|
||||
(
|
||||
Some(handle),
|
||||
Some(coordinator_output_task),
|
||||
Some(coordinator_task),
|
||||
)
|
||||
} else if let Some(address) =
|
||||
dp_stats_address.as_deref().or(match config.coordinator_mode.as_ref() {
|
||||
Some(CoordinatorMode::External { address }) => Some(address.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
{
|
||||
let (handle, service) = CoordinatorHandle::connect_external(address).await?;
|
||||
let coordinator_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(service.run(inner.clone())));
|
||||
(Some(handle), None, Some(coordinator_task))
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
input_address: connected.input_address,
|
||||
output_address: connected.output_address,
|
||||
engines,
|
||||
inner,
|
||||
coordinator,
|
||||
abort_tx,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
coordinator_output_task,
|
||||
coordinator_task,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the address of the input socket that the client uses to send
|
||||
/// requests to the engine.
|
||||
pub fn input_address(&self) -> &str {
|
||||
&self.input_address
|
||||
}
|
||||
|
||||
/// Return the address of the output socket that the client listens on for
|
||||
/// engine responses.
|
||||
pub fn output_address(&self) -> &str {
|
||||
&self.output_address
|
||||
}
|
||||
|
||||
/// Return the number of engines connected to this client.
|
||||
pub fn engine_count(&self) -> usize {
|
||||
self.engines.len()
|
||||
}
|
||||
|
||||
/// Return the engine identities of all engines connected to this client.
|
||||
pub fn engine_identities(&self) -> Vec<&[u8]> {
|
||||
self.engines.iter().map(|engine| &*engine.engine_id).collect()
|
||||
}
|
||||
|
||||
/// Return the ready responses received from all engines on the input
|
||||
/// socket.
|
||||
pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return the engine-reported effective model dtype, when available.
|
||||
pub fn model_dtype(&self) -> Option<ModelDtype> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.find_map(|response| response.dtype)
|
||||
}
|
||||
|
||||
/// Return the total number of GPU blocks summed across all connected
|
||||
/// engines.
|
||||
pub fn total_num_gpu_blocks(&self) -> u64 {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.map(|r| r.num_gpu_blocks)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Return the minimum engine-reported `max_model_len` across all engines.
|
||||
///
|
||||
/// This is the auto-fitted value after KV cache profiling and may differ
|
||||
/// from the originally configured value.
|
||||
pub fn max_model_len(&self) -> Option<u32> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|e| e.ready_response.as_ref())
|
||||
.map(|r| r.max_model_len as u32)
|
||||
.min()
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.inner.is_healthy()
|
||||
}
|
||||
|
||||
/// Return the first persistent health error observed by the client, if any.
|
||||
pub fn health_error(&self) -> Option<Arc<Error>> {
|
||||
self.inner.health_error()
|
||||
}
|
||||
}
|
||||
|
||||
// Client API implementation.
|
||||
impl EngineCoreClient {
|
||||
/// Add a new request to the engine and return a per-request raw output
|
||||
/// stream.
|
||||
pub async fn call(&self, mut req: EngineCoreRequest) -> Result<EngineCoreOutputStream> {
|
||||
req.client_index = self.config.client_index;
|
||||
req.validate()?;
|
||||
trace!(
|
||||
request_id = %req.request_id,
|
||||
client_index = req.client_index,
|
||||
current_wave = req.current_wave,
|
||||
request = ?req,
|
||||
"sending add request"
|
||||
);
|
||||
|
||||
let request_id = req.request_id.clone();
|
||||
let data_parallel_rank = req.data_parallel_rank;
|
||||
let (engine_id, rx) =
|
||||
self.inner.register_request(request_id.clone(), data_parallel_rank)?;
|
||||
|
||||
let result: Result<()> = async {
|
||||
if let Some(coordinator) = self.coordinator.as_ref() {
|
||||
let snapshot = coordinator.snapshot();
|
||||
req.current_wave = snapshot.current_wave;
|
||||
if !snapshot.engines_running {
|
||||
coordinator.notify_first_request(engine_id.clone())?;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
request_id = req.request_id,
|
||||
?engine_id,
|
||||
"registered request to engine"
|
||||
);
|
||||
|
||||
self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
// Failed to send the request to the engine, roll back the registration.
|
||||
if let Err(error) = result {
|
||||
self.inner.rollback_request(&request_id);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
Ok(EngineCoreOutputStream::new(
|
||||
request_id,
|
||||
self.abort_tx.clone(),
|
||||
rx,
|
||||
))
|
||||
}
|
||||
|
||||
/// Abort currently in-flight requests by request ID.
|
||||
pub async fn abort(&self, ids: &[String]) -> Result<()> {
|
||||
let abortable = self.inner.abortable_request_ids(ids)?;
|
||||
|
||||
trace!(request_ids = ?ids, abortable_request_ids = ?abortable, "sending abort request ids");
|
||||
|
||||
if abortable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for (engine_id, request_ids) in abortable {
|
||||
self.inner.do_abort_requests(&engine_id, &request_ids).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Call a typed utility method on all connected engines, returning one
|
||||
/// decoded result per connected engine if all calls succeed or an error
|
||||
/// if any call fails.
|
||||
///
|
||||
/// Callers should pass utility arguments using Rust tuple semantics so the
|
||||
/// encoded payload matches Python's `(client_index, call_id,
|
||||
/// method_name, args)` contract: `()`, `(arg,)`, `(arg1, arg2)`, etc.
|
||||
pub async fn call_utility<T, A>(&self, method: &str, args: A) -> Result<Vec<T>>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
A: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
trace!(
|
||||
method,
|
||||
client_index = self.config.client_index,
|
||||
engine_count = self.engines.len(),
|
||||
"sending utility request"
|
||||
);
|
||||
|
||||
// Phase 1: allocate one call id per engine and build the per-engine
|
||||
// request payloads up-front. Any failure here (registry closed, encode
|
||||
// error) must roll back the call ids already allocated so they do not
|
||||
// leak in the utility registry until shutdown.
|
||||
let mut pending_calls = Vec::with_capacity(self.engines.len());
|
||||
let mut prepared_sends = Vec::with_capacity(self.engines.len());
|
||||
for engine in &self.engines {
|
||||
let (call_id, rx) = match self.inner.allocate_and_register_utility_call() {
|
||||
Ok(pair) => pair,
|
||||
Err(err) => {
|
||||
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let request = match EngineCoreUtilityRequest::new(
|
||||
self.config.client_index,
|
||||
call_id,
|
||||
method,
|
||||
&args,
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
self.inner.unregister_utility_calls(
|
||||
pending_calls.iter().map(|(id, _)| *id).chain(std::iter::once(call_id)),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
pending_calls.push((call_id, rx));
|
||||
prepared_sends.push((&engine.engine_id, request));
|
||||
}
|
||||
|
||||
// Phase 2: dispatch every utility request concurrently. `try_join_all`
|
||||
// fails fast on the first transport error and drops the remaining send
|
||||
// futures; any engines that already received the request will reply,
|
||||
// but those replies are simply dropped because we roll back the call
|
||||
// ids below.
|
||||
let send_futures = prepared_sends.iter().map(|(engine_id, request)| {
|
||||
self.inner.send_to_engine(engine_id, EngineCoreRequestType::Utility, request)
|
||||
});
|
||||
if let Err(err) = try_join_all(send_futures).await {
|
||||
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Phase 3: wait for all engines to respond and preserve the per-engine
|
||||
// result list.
|
||||
let futures = pending_calls.into_iter().map(|(call_id, rx)| async move {
|
||||
rx.await
|
||||
.map_err(|_| Error::UtilityCallClosed {
|
||||
method: method.to_string(),
|
||||
call_id,
|
||||
})??
|
||||
.into_typed_result(method)
|
||||
});
|
||||
try_join_all(futures).await
|
||||
}
|
||||
|
||||
/// Execute `collective_rpc` on all engines and flatten all engine results
|
||||
/// into one list.
|
||||
pub async fn collective_rpc<A, K>(
|
||||
&self,
|
||||
method: &str,
|
||||
timeout: Option<f64>,
|
||||
args: A,
|
||||
kwargs: K,
|
||||
) -> Result<Vec<rmpv::Value>>
|
||||
where
|
||||
A: serde::Serialize + std::fmt::Debug,
|
||||
K: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
let results = self
|
||||
.call_utility::<rmpv::Value, _>("collective_rpc", (method, timeout, args, kwargs))
|
||||
.await?;
|
||||
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.flat_map(|result| match result {
|
||||
// Each engine's `collective_rpc` result is itself the worker-level result list.
|
||||
rmpv::Value::Array(results) => results,
|
||||
other => vec![other],
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
pub async fn is_sleeping(&self) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self.call_utility("is_sleeping", ()).await?[0])
|
||||
}
|
||||
|
||||
/// Reset the multi-modal cache.
|
||||
pub async fn reset_mm_cache(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("reset_mm_cache", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset the encoder cache.
|
||||
pub async fn reset_encoder_cache(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("reset_encoder_cache", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset the prefix cache and optionally the external connector cache.
|
||||
pub async fn reset_prefix_cache(
|
||||
&self,
|
||||
reset_running_requests: bool,
|
||||
reset_connector: bool,
|
||||
) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self
|
||||
.call_utility(
|
||||
"reset_prefix_cache",
|
||||
(reset_running_requests, reset_connector),
|
||||
)
|
||||
.await?[0])
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
|
||||
self.call_utility::<(), _>("sleep", (level, mode)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
|
||||
self.call_utility::<(), _>("wake_up", (tags,)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shut down local client tasks and close transport state.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
let Self {
|
||||
inner,
|
||||
abort_tx,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
coordinator_output_task,
|
||||
coordinator_task,
|
||||
..
|
||||
} = self;
|
||||
|
||||
info!("shutting down engine-core client");
|
||||
inner.shutdown();
|
||||
drop(abort_tx);
|
||||
|
||||
// Abort all client tasks first, then await them.
|
||||
// Note the aborting orders here.
|
||||
let mut tasks = vec![abort_task, dispatcher_task, output_task];
|
||||
tasks.extend(coordinator_task);
|
||||
tasks.extend(coordinator_output_task);
|
||||
|
||||
tasks.iter().for_each(|t| t.abort());
|
||||
join_all(tasks).await;
|
||||
|
||||
info!("engine-core client shut down");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::slice;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use parking_lot::Mutex;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::RouterSendHalf;
|
||||
|
||||
use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, UtilityRegistry};
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::record_scheduler_stats;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::{
|
||||
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
|
||||
UtilityOutput, encode_msgpack,
|
||||
};
|
||||
use crate::transport::{ConnectedEngine, EngineId};
|
||||
use crate::{Error, Result, transport};
|
||||
|
||||
pub(crate) struct ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
model_name: String,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
health_error: ArcSwapOption<Error>,
|
||||
}
|
||||
|
||||
impl ClientInner {
|
||||
/// Create a new instance with the given input send half after the startup
|
||||
/// handshake completes.
|
||||
pub fn new(
|
||||
input_send: RouterSendHalf,
|
||||
model_name: String,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
Self {
|
||||
input_send,
|
||||
model_name,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
health_error: ArcSwapOption::empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
&self.model_name
|
||||
}
|
||||
|
||||
/// Register a newly added request. Return the selected engine id and the
|
||||
/// per-request output channel bound to its `request_id`.
|
||||
///
|
||||
/// When `data_parallel_rank` is provided, the request is routed to that
|
||||
/// specific engine rank, bypassing load balancing.
|
||||
pub fn register_request(
|
||||
&self,
|
||||
request_id: String,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
registry.register(request_id, data_parallel_rank)
|
||||
}
|
||||
|
||||
/// Allocate the next utility `call_id` and register its waiting receiver.
|
||||
pub fn allocate_and_register_utility_call(&self) -> Result<(i64, UtilityReceiver)> {
|
||||
let mut registry = self.utility_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
Ok(registry.allocate_and_register())
|
||||
}
|
||||
|
||||
/// Undo a batch of utility call allocations when the fan-out send fails
|
||||
/// partway through. Silently ignores unknown call ids so callers can pass
|
||||
/// the full set without first filtering successful sends.
|
||||
pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = i64>) {
|
||||
self.utility_reg.lock().unregister_many(call_ids);
|
||||
}
|
||||
|
||||
/// Undo a request registration when `add_request()` fails.
|
||||
pub fn rollback_request(&self, request_id: &str) {
|
||||
let _ = self.request_reg.lock().remove(request_id);
|
||||
}
|
||||
|
||||
/// Filter the given request IDs to the subset that are still tracked as
|
||||
/// active and can be aborted, grouped by the engine that originally
|
||||
/// accepted them.
|
||||
pub fn abortable_request_ids(
|
||||
&self,
|
||||
request_ids: &[String],
|
||||
) -> Result<BTreeMap<EngineId, Vec<String>>> {
|
||||
let registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
Ok(registry.abortable_request_ids(request_ids))
|
||||
}
|
||||
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn take_sender_for_output(
|
||||
&self,
|
||||
output: &EngineCoreOutput,
|
||||
) -> Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
|
||||
self.request_reg.lock().sender_for_output(output)
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
/// their stream senders.
|
||||
pub fn finish_requests<'a>(
|
||||
&self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
) -> Vec<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
|
||||
self.request_reg.lock().finish_many(request_ids)
|
||||
}
|
||||
|
||||
/// Apply one scheduler stats update for the given engine to the local
|
||||
/// routing state. Returns `false` if the engine is unknown to the
|
||||
/// client.
|
||||
pub fn apply_scheduler_stats(&self, engine_index: u32, stats: &SchedulerStats) -> bool {
|
||||
self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
|
||||
}
|
||||
|
||||
/// Close all active request streams and utility calls with the first
|
||||
/// persistent health error.
|
||||
pub fn close_registries(&self, error: Arc<Error>) {
|
||||
let persistent_error = self.record_health_error(error);
|
||||
let request_senders = self.request_reg.lock().close();
|
||||
let utility_senders = self.utility_reg.lock().close();
|
||||
|
||||
// Notify all ongoing requests that the client is closed.
|
||||
for sender in request_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
}
|
||||
for sender in utility_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the first persistent health error observed by the client, if any.
|
||||
pub fn health_error(&self) -> Option<Arc<Error>> {
|
||||
self.health_error.load_full()
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.health_error.load().is_none()
|
||||
}
|
||||
|
||||
/// Resolve one utility output to the waiting caller. Returns `true` if a
|
||||
/// waiting caller existed.
|
||||
pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
|
||||
match self.utility_reg.lock().resolve(&output.call_id) {
|
||||
Some(sender) => {
|
||||
sender.send(Ok(output)).unwrap_or_default();
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Send the given message to the engine. The request should be first
|
||||
/// registered via `register_request()` to ensure the request stream is
|
||||
/// tracked.
|
||||
pub async fn send_to_engine<T>(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
request_type: EngineCoreRequestType,
|
||||
payload: &T,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
// TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
|
||||
// frames instead of always producing a single msgpack frame.
|
||||
let payload = encode_msgpack(payload)?;
|
||||
let mut input_send = self.input_send.clone();
|
||||
transport::send_message(&mut input_send, engine_id, request_type.to_frame(), payload)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handle an abort request by sending the abort message to the engine.
|
||||
pub async fn do_abort_requests(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
request_ids: &[String],
|
||||
) -> Result<()> {
|
||||
self.send_to_engine(engine_id, EngineCoreRequestType::Abort, &request_ids).await
|
||||
}
|
||||
|
||||
/// Shut down by closing all active request streams and utility calls with a
|
||||
/// sticky client closed error.
|
||||
pub fn shutdown(&self) {
|
||||
self.close_registries(Arc::new(client_closed!("engine-core client shut down")));
|
||||
}
|
||||
|
||||
/// Remove the request from the active registry for auto-abort and return
|
||||
/// the engine that the request was originally routed to, if it is still
|
||||
/// active.
|
||||
pub fn take_auto_abort_target(&self, request_id: &str) -> Option<EngineId> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
let (_, engine_id) = registry.remove(request_id)?;
|
||||
if registry.is_closed() {
|
||||
return None;
|
||||
}
|
||||
Some(engine_id)
|
||||
}
|
||||
|
||||
/// Publish the first persistent health error and return the sticky error
|
||||
/// recorded for this client. Later failures do not overwrite the first
|
||||
/// one so `/health` and post-close callers observe a stable cause.
|
||||
fn record_health_error(&self, error: Arc<Error>) -> Arc<Error> {
|
||||
if let Some(existing) = self.health_error.load_full() {
|
||||
return existing;
|
||||
}
|
||||
self.health_error
|
||||
.rcu(|current| current.clone().unwrap_or_else(|| error.clone()));
|
||||
self.health_error
|
||||
.load_full()
|
||||
.expect("health error must be recorded before registries close")
|
||||
}
|
||||
|
||||
/// Assert there is a recorded health error and return a `Shared` variant
|
||||
/// wrapping it for error returns when the client is already closed.
|
||||
fn closed_error(&self) -> Error {
|
||||
Error::Shared(self.health_error.load_full().expect(
|
||||
"closed registry must have a recorded health error before rejecting new operations",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Background loop that listens for request IDs to abort and sends abort
|
||||
/// messages to the engine. This is used to implement the auto-abort behavior
|
||||
/// when a request stream is dropped without being properly terminated.
|
||||
pub(crate) async fn run_abort_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut abort_rx: mpsc::UnboundedReceiver<AbortRequest>,
|
||||
) {
|
||||
// TODO: receive and abort requests in batch
|
||||
while let Some(AbortRequest { request_id, cause }) = abort_rx.recv().await {
|
||||
let Some(engine_id) = inner.take_auto_abort_target(&request_id) else {
|
||||
debug!(request_id, "skip auto-abort for inactive request");
|
||||
continue;
|
||||
};
|
||||
|
||||
match cause {
|
||||
AbortCause::DroppedStream => {
|
||||
info!(request_id, "auto-aborting request due to dropped stream")
|
||||
}
|
||||
AbortCause::StopStringMatched => {
|
||||
debug!(
|
||||
request_id,
|
||||
"auto-aborting request due to stop string matched"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(error) = inner.do_abort_requests(&engine_id, slice::from_ref(&request_id)).await
|
||||
{
|
||||
warn!(
|
||||
request_id,
|
||||
?engine_id,
|
||||
error = %error.as_report(),
|
||||
"failed to auto-abort dropped request stream"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background loop that listens for engine-core outputs and dispatches them to
|
||||
/// the corresponding request streams based on their `request_id`.
|
||||
pub(crate) async fn run_output_dispatcher_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
let outputs = match output_rx.recv().await {
|
||||
Some(outputs) => outputs,
|
||||
None => Err(dispatcher_closed!(
|
||||
"engine-core output dispatcher channel closed"
|
||||
)),
|
||||
}?;
|
||||
|
||||
match outputs.classify() {
|
||||
ClassifiedEngineCoreOutputs::RequestBatch(batch) => {
|
||||
for output in batch.outputs {
|
||||
let request_id = output.request_id.clone();
|
||||
let Some(sender) = inner.take_sender_for_output(&output) else {
|
||||
debug!(request_id, "dropping output for inactive request");
|
||||
continue;
|
||||
};
|
||||
|
||||
let wrapped_output = EngineCoreStreamOutput {
|
||||
engine_index: batch.engine_index,
|
||||
timestamp: batch.timestamp,
|
||||
output,
|
||||
};
|
||||
if sender.send(Ok(wrapped_output)).is_err() {
|
||||
debug!(request_id, "request output stream receiver dropped");
|
||||
}
|
||||
}
|
||||
|
||||
// The sender for normally-finished requests should have already been removed
|
||||
// from the registry when their final output was dispatched
|
||||
// above. This serves as a safety net to capture any
|
||||
// requests marked as finished by the engine.
|
||||
if let Some(finished_requests) = batch.finished_requests.as_ref() {
|
||||
for request_id in finished_requests {
|
||||
trace!(request_id, "request completed via finished_requests");
|
||||
}
|
||||
drop(inner.finish_requests(finished_requests));
|
||||
}
|
||||
|
||||
if let Some(scheduler_stats) = batch.scheduler_stats.as_ref() {
|
||||
if !inner.apply_scheduler_stats(batch.engine_index, scheduler_stats) {
|
||||
debug!(
|
||||
engine_index = batch.engine_index,
|
||||
"dropping scheduler stats for unknown engine"
|
||||
);
|
||||
}
|
||||
record_scheduler_stats(
|
||||
&METRICS.scheduler,
|
||||
inner.model_name(),
|
||||
batch.engine_index,
|
||||
scheduler_stats,
|
||||
);
|
||||
}
|
||||
}
|
||||
ClassifiedEngineCoreOutputs::Utility(utility) => {
|
||||
let call_id = utility.output.call_id;
|
||||
if inner.resolve_utility_output(utility.output) {
|
||||
trace!(
|
||||
call_id,
|
||||
engine_index = utility.engine_index,
|
||||
"resolved utility output"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
call_id,
|
||||
engine_index = utility.engine_index,
|
||||
"dropping output for unexpected utility call"
|
||||
);
|
||||
}
|
||||
}
|
||||
other @ (ClassifiedEngineCoreOutputs::DpControl { .. }
|
||||
| ClassifiedEngineCoreOutputs::Other(_)) => {
|
||||
Err::<(), _>(unexpected_dispatcher_output!(
|
||||
"received unexpected output on main dispatcher path: {other:?}"
|
||||
))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let Err(error) = result else { return };
|
||||
|
||||
warn!(error = %error.as_report(), "output dispatcher exiting with error");
|
||||
inner.close_registries(Arc::new(error));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use zeromq::{RouterSocket, Socket};
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn test_inner() -> ClientInner {
|
||||
let mut socket = RouterSocket::new();
|
||||
socket.bind("tcp://127.0.0.1:0").await.unwrap();
|
||||
let (send, _) = socket.split();
|
||||
ClientInner::new(
|
||||
send,
|
||||
"test-model".to_string(),
|
||||
&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn close_registries_records_first_health_error_only() {
|
||||
let inner = test_inner().await;
|
||||
|
||||
inner.close_registries(Arc::new(Error::EngineCoreDead));
|
||||
assert!(!inner.is_healthy());
|
||||
assert!(matches!(
|
||||
inner.health_error().as_deref(),
|
||||
Some(Error::EngineCoreDead)
|
||||
));
|
||||
|
||||
inner.close_registries(Arc::new(client_closed!("shutdown")));
|
||||
assert!(matches!(
|
||||
inner.health_error().as_deref(),
|
||||
Some(Error::EngineCoreDead)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,670 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::trace;
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::{EngineCoreOutput, UtilityOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
|
||||
pub type OutputReceiver = mpsc::UnboundedReceiver<Result<EngineCoreStreamOutput>>;
|
||||
pub type UtilitySender = oneshot::Sender<Result<UtilityOutput>>;
|
||||
pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TrackedRequest {
|
||||
sender: OutputSender,
|
||||
engine_id: EngineId,
|
||||
}
|
||||
|
||||
/// The latest real scheduler-side load snapshot observed from one engine.
|
||||
///
|
||||
/// These counters come from `scheduler_stats` on the normal engine output path
|
||||
/// and are the preferred routing signal once available.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct EngineLoadSnapshot {
|
||||
/// Requests still counted on the scheduler's waiting side.
|
||||
waiting: usize,
|
||||
/// Requests currently counted on the scheduler's running side.
|
||||
running: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct EngineRoutingState {
|
||||
/// Requests admitted by this frontend that have not finished yet.
|
||||
///
|
||||
/// This is used both as the bootstrap fallback before real scheduler stats
|
||||
/// exist and as a lower bound afterwards so asynchronous scheduler
|
||||
/// snapshots cannot erase frontend admission history.
|
||||
inflight: usize,
|
||||
/// The latest real scheduler snapshot received from this engine, if any.
|
||||
last_scheduler_stats: Option<EngineLoadSnapshot>,
|
||||
}
|
||||
|
||||
impl EngineRoutingState {
|
||||
/// Compute the routing score used to pick the least-loaded engine.
|
||||
///
|
||||
/// Scheduler stats can raise the load estimate above the frontend-local
|
||||
/// view, but they should not lower it below requests this frontend has
|
||||
/// already admitted. Waiting requests still get the same extra penalty
|
||||
/// as the original `waiting * 4 + running` score.
|
||||
fn routing_score(&self) -> usize {
|
||||
const WAITING_WEIGHT: usize = 4;
|
||||
|
||||
let Some(stats) = self.last_scheduler_stats else {
|
||||
return self.inflight;
|
||||
};
|
||||
|
||||
let scheduler_total = stats.running + stats.waiting;
|
||||
self.inflight.max(scheduler_total) + stats.waiting * (WAITING_WEIGHT - 1)
|
||||
}
|
||||
|
||||
/// Replace the local routing view with a fresh real scheduler snapshot.
|
||||
fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) {
|
||||
self.last_scheduler_stats = Some(next);
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal registry for tracking active requests and their output stream
|
||||
/// senders.
|
||||
///
|
||||
/// This is used to route incoming outputs to the correct request stream, and to
|
||||
/// ensure proper cleanup of senders when requests finish or the client shuts
|
||||
/// down.
|
||||
#[derive(Debug)]
|
||||
pub struct RequestRegistry {
|
||||
closed: bool,
|
||||
requests: BTreeMap<String, TrackedRequest>,
|
||||
routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
|
||||
}
|
||||
|
||||
impl RequestRegistry {
|
||||
pub fn new(engines: &[ConnectedEngine]) -> Self {
|
||||
Self {
|
||||
closed: false,
|
||||
requests: BTreeMap::default(),
|
||||
routing_per_engine: engines
|
||||
.iter()
|
||||
.map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a newly added request. Create the per-request output channel
|
||||
/// bound to its `request_id` and return the selected engine id.
|
||||
///
|
||||
/// When `data_parallel_rank` is provided, the request is routed directly to
|
||||
/// the engine at that rank index, bypassing load balancing. Otherwise
|
||||
/// the engine with the fewest in-flight requests is chosen.
|
||||
pub fn register(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
if self.requests.contains_key(&request_id) {
|
||||
return Err(Error::DuplicateRequestId { request_id });
|
||||
}
|
||||
|
||||
let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
self.requests.insert(
|
||||
request_id,
|
||||
TrackedRequest {
|
||||
sender: tx,
|
||||
engine_id: engine_id.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let state = self
|
||||
.routing_per_engine
|
||||
.get_mut(&engine_id)
|
||||
.expect("request registry must track all known engines");
|
||||
state.inflight += 1;
|
||||
|
||||
Ok((engine_id, rx))
|
||||
}
|
||||
|
||||
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
|
||||
if let Some(rank) = data_parallel_rank {
|
||||
// Route to the engine at the specified rank index.
|
||||
let engine_id = EngineId::from_engine_index(rank);
|
||||
return self
|
||||
.routing_per_engine
|
||||
.contains_key(&engine_id)
|
||||
.then_some(engine_id)
|
||||
.ok_or_else(|| Error::InvalidDataParallelRank {
|
||||
rank,
|
||||
num_engines: self.routing_per_engine.len() as u32,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.routing_per_engine
|
||||
.iter()
|
||||
.min_by_key(|(_, state)| state.routing_score())
|
||||
.map(|(engine_id, _)| engine_id.clone())
|
||||
.expect("request registry must contain at least one engine"))
|
||||
}
|
||||
|
||||
/// Filter the given request IDs to the subset that are still tracked as
|
||||
/// active and can be aborted, grouped by engine.
|
||||
pub fn abortable_request_ids(&self, request_ids: &[String]) -> BTreeMap<EngineId, Vec<String>> {
|
||||
let mut by_engine = BTreeMap::new();
|
||||
for request_id in request_ids {
|
||||
let Some(tracked) = self.requests.get(request_id.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
by_engine
|
||||
.entry(tracked.engine_id.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(request_id.clone());
|
||||
}
|
||||
by_engine
|
||||
}
|
||||
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
|
||||
if output.finished() {
|
||||
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
|
||||
} else {
|
||||
self.requests
|
||||
.get(output.request_id.as_str())
|
||||
.map(|tracked| tracked.sender.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
/// their stream senders.
|
||||
pub fn finish_many<'a>(
|
||||
&mut self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
) -> Vec<OutputSender> {
|
||||
request_ids
|
||||
.into_iter()
|
||||
.filter_map(|request_id| self.remove(request_id.as_str()).map(|tracked| tracked.0))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Apply one scheduler stats update for the given engine to the local
|
||||
/// routing state. Returns `false` if the engine is unknown to the
|
||||
/// client.
|
||||
pub fn apply_scheduler_stats(&mut self, engine_index: u32, stats: &SchedulerStats) -> bool {
|
||||
self.apply_scheduler_counts(
|
||||
engine_index,
|
||||
EngineLoadSnapshot {
|
||||
waiting: stats.num_waiting_reqs as usize,
|
||||
running: stats.num_running_reqs as usize,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Mark the registry as closed, detach and return all tracked senders.
|
||||
pub fn close(&mut self) -> Vec<OutputSender> {
|
||||
if self.closed {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
std::mem::take(&mut self.requests)
|
||||
.into_values()
|
||||
.map(|tracked| tracked.sender)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Remove one request from the local registry. Returns the tracked entry if
|
||||
/// it exists.
|
||||
#[must_use]
|
||||
pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
|
||||
let tracked = self.requests.remove(request_id)?;
|
||||
self.routing_per_engine
|
||||
.get_mut(&tracked.engine_id)
|
||||
.expect("request registry must track all known engines")
|
||||
.inflight -= 1;
|
||||
Some((tracked.sender, tracked.engine_id))
|
||||
}
|
||||
|
||||
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
|
||||
let engine_id = EngineId::from_engine_index(engine_index);
|
||||
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let previous = state.last_scheduler_stats;
|
||||
if previous != Some(next) {
|
||||
trace!(
|
||||
?engine_id,
|
||||
previous_waiting = previous.map(|stats| stats.waiting),
|
||||
previous_running = previous.map(|stats| stats.running),
|
||||
waiting = next.waiting,
|
||||
running = next.running,
|
||||
"updated scheduler routing counts",
|
||||
);
|
||||
}
|
||||
|
||||
state.apply_scheduler_counts(next);
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn contains(&self, request_id: &str) -> bool {
|
||||
self.requests.contains_key(request_id)
|
||||
}
|
||||
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal registry for tracking active utility calls and their waiting
|
||||
/// receivers.
|
||||
#[derive(Debug)]
|
||||
pub struct UtilityRegistry {
|
||||
closed: bool,
|
||||
next_call_id: AtomicI64,
|
||||
utility_calls: BTreeMap<i64, UtilitySender>,
|
||||
}
|
||||
|
||||
impl Default for UtilityRegistry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
closed: false,
|
||||
next_call_id: AtomicI64::new(1),
|
||||
utility_calls: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UtilityRegistry {
|
||||
/// Allocate the next utility `call_id` and register a newly added utility
|
||||
/// call.
|
||||
pub fn allocate_and_register(&mut self) -> (i64, UtilityReceiver) {
|
||||
let call_id = self.next_call_id.fetch_add(1, Ordering::Relaxed);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.utility_calls.insert(call_id, tx);
|
||||
(call_id, rx)
|
||||
}
|
||||
|
||||
/// Resolve a utility output to its waiting receiver.
|
||||
pub fn resolve(&mut self, call_id: &i64) -> Option<UtilitySender> {
|
||||
self.utility_calls.remove(call_id)
|
||||
}
|
||||
|
||||
/// Drop a batch of registered utility calls without delivering a result.
|
||||
/// Used to roll back allocations when the dispatch fan-out fails before
|
||||
/// every engine could accept the request.
|
||||
pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = i64>) {
|
||||
for call_id in call_ids {
|
||||
self.utility_calls.remove(&call_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark the registry as closed, detach and return all tracked senders.
|
||||
pub fn close(&mut self) -> Vec<UtilitySender> {
|
||||
if self.closed {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
std::mem::take(&mut self.utility_calls).into_values().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn contains(&self, call_id: i64) -> bool {
|
||||
self.utility_calls.contains_key(&call_id)
|
||||
}
|
||||
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
|
||||
use crate::EngineId;
|
||||
use crate::client::state::EngineLoadSnapshot;
|
||||
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
#[test]
|
||||
fn registry_rejects_duplicate_request_ids() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
let error = registry.register("req-1".to_string(), None).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_removes_finished_request_on_output() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
|
||||
let sender = registry.sender_for_output(&EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sender.is_some());
|
||||
assert!(!registry.contains("req-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_closes_all_requests_on_failure() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
registry.register("req-2".to_string(), None).unwrap();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
assert_eq!(senders.len(), 2);
|
||||
assert!(registry.is_closed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_tracks_engine_id_per_request() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
assert_eq!(chosen_0_again, engine_0);
|
||||
|
||||
let grouped = registry.abortable_request_ids(&[
|
||||
"req-1".to_string(),
|
||||
"req-2".to_string(),
|
||||
"req-3".to_string(),
|
||||
]);
|
||||
assert_eq!(
|
||||
grouped.get(&engine_0).unwrap(),
|
||||
&vec!["req-1".to_string(), "req-3".to_string()]
|
||||
);
|
||||
assert_eq!(grouped.get(&engine_1).unwrap(), &vec!["req-2".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_uses_inflight_as_waiting_fallback_before_stats_arrive() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
assert_eq!(chosen_0_again, engine_0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_uses_inflight_before_stats_arrive() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 3,
|
||||
last_scheduler_stats: None,
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_uses_inflight_as_scheduler_stats_lower_bound() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 7,
|
||||
last_scheduler_stats: Some(EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 2,
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_keeps_extra_waiting_penalty() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 1,
|
||||
last_scheduler_stats: Some(EngineLoadSnapshot {
|
||||
waiting: 3,
|
||||
running: 2,
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 14);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_prefers_real_scheduler_stats_over_inflight() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
assert!(registry.apply_scheduler_counts(
|
||||
0,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 3,
|
||||
running: 2
|
||||
}
|
||||
));
|
||||
assert!(registry.apply_scheduler_counts(
|
||||
1,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 1
|
||||
}
|
||||
));
|
||||
|
||||
let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_data_parallel_rank_routes_to_specified_engine() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let engine_2 = EngineId::from_engine_index(2);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_2.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
// Explicitly target rank 2 (third engine).
|
||||
let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
|
||||
assert_eq!(chosen, engine_2);
|
||||
|
||||
// Explicitly target rank 0 (first engine).
|
||||
let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
// Explicitly target rank 1.
|
||||
let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_data_parallel_rank_bypasses_load_balancing() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
// Load-balance: first two go to engine_0 and engine_1.
|
||||
registry.register("req-lb-0".to_string(), None).unwrap();
|
||||
|
||||
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
|
||||
// But with dp_rank=0, it should still go to engine_0.
|
||||
let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_out_of_range_rank_returns_error() {
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: EngineId::from_engine_index(0),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: EngineId::from_engine_index(1),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 2,
|
||||
num_engines: 2,
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_rank_on_single_engine_only_accepts_zero() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
}]);
|
||||
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 1,
|
||||
num_engines: 1,
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_tracks_and_removes_call_ids() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id_1, _) = registry.allocate_and_register();
|
||||
let (call_id_2, _) = registry.allocate_and_register();
|
||||
|
||||
assert_eq!(call_id_1, 1);
|
||||
assert_eq!(call_id_2, 2);
|
||||
assert!(registry.contains(1));
|
||||
assert!(registry.contains(2));
|
||||
assert!(registry.resolve(&1).is_some());
|
||||
assert!(!registry.contains(1));
|
||||
assert!(registry.contains(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_closes_all_waiters_on_failure() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
registry.allocate_and_register();
|
||||
registry.allocate_and_register();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
assert_eq!(senders.len(), 2);
|
||||
assert!(!registry.contains(1));
|
||||
assert!(!registry.contains(2));
|
||||
assert!(registry.is_closed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_unregister_many_drops_pending_calls() {
|
||||
use tokio::sync::oneshot::error::TryRecvError;
|
||||
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id_1, mut rx_1) = registry.allocate_and_register();
|
||||
let (call_id_2, mut rx_2) = registry.allocate_and_register();
|
||||
let (call_id_3, _rx_3) = registry.allocate_and_register();
|
||||
|
||||
// Drop two of the three allocated calls; the third stays pending.
|
||||
registry.unregister_many([call_id_1, call_id_2]);
|
||||
|
||||
assert!(!registry.contains(call_id_1));
|
||||
assert!(!registry.contains(call_id_2));
|
||||
assert!(registry.contains(call_id_3));
|
||||
// The receivers must observe the sender being dropped (channel closed).
|
||||
assert!(matches!(rx_1.try_recv(), Err(TryRecvError::Closed)));
|
||||
assert!(matches!(rx_2.try_recv(), Err(TryRecvError::Closed)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_unregister_many_ignores_unknown_call_ids() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id, _rx) = registry.allocate_and_register();
|
||||
|
||||
// Unknown call ids are silently ignored — caller doesn't care which were live.
|
||||
registry.unregister_many([call_id, 42, 9999]);
|
||||
|
||||
assert!(!registry.contains(call_id));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user