[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+34
View File
@@ -0,0 +1,34 @@
[package]
name = "vllm-text"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
anyhow.workspace = true
asynk-strim-attr.workspace = true
easy-ext.workspace = true
enum-as-inner.workspace = true
futures.workspace = true
hf-hub.workspace = true
itertools.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_with.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
tracing.workspace = true
trait-set.workspace = true
vllm-engine-core-client.workspace = true
vllm-llm.workspace = true
vllm-tokenizer.workspace = true
[dev-dependencies]
expect-test.workspace = true
futures.workspace = true
tempfile.workspace = true
tokio.workspace = true
vllm-llm = { workspace = true, features = ["test-util"] }
[lints]
workspace = true
+373
View File
@@ -0,0 +1,373 @@
use std::collections::BTreeSet;
use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
use crate::error::{Error, Result};
/// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub struct HfTokenizerConfig {
#[serde(flatten)]
pub special_tokens: HfSpecialTokens,
pub chat_template: Option<String>,
/// The `tokenizer_class` field from HuggingFace tokenizer configs. Some
/// tiktoken-based models (e.g. DeepSeek, Kimi K2) set this to a value
/// containing "Tiktoken" which can be used as a hint for backend
/// selection.
pub tokenizer_class: Option<String>,
}
/// Hugging Face named special tokens may be serialized as a string or an
/// object carrying the token content.
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub enum NamedSpecialToken {
Text(String),
WithContent { content: String },
}
impl Serialize for NamedSpecialToken {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.as_str())
}
}
impl From<NamedSpecialToken> for String {
fn from(value: NamedSpecialToken) -> Self {
match value {
NamedSpecialToken::Text(string) => string,
NamedSpecialToken::WithContent { content } => content,
}
}
}
impl NamedSpecialToken {
pub fn as_str(&self) -> &str {
match self {
Self::Text(value) => value,
Self::WithContent { content } => content,
}
}
}
/// Minimal set of special-token entries needed by chat/EOS handling.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
#[serde(default)]
pub struct HfSpecialTokens {
pub bos_token: Option<NamedSpecialToken>,
pub eos_token: Option<NamedSpecialToken>,
pub unk_token: Option<NamedSpecialToken>,
pub pad_token: Option<NamedSpecialToken>,
}
impl HfSpecialTokens {
/// Returns true if we don't discover any special tokens in the config.
pub fn is_empty(&self) -> bool {
self.bos_token.is_none()
&& self.eos_token.is_none()
&& self.unk_token.is_none()
&& self.pad_token.is_none()
}
}
/// Minimal subset of `config.json` (the model's main HF config).
///
/// This intentionally supports only the two layouts we currently care about in
/// the Rust frontend:
/// - pure text models that keep text metadata at the top level
/// - composite models that expose a single nested `text_config`
///
/// We do not support additional entry points such as `decoder`, `generator`, or
/// `text_encoder`.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub struct ModelConfig {
model_type: Option<String>,
max_position_embeddings: Option<u32>,
num_attention_heads: Option<u32>,
num_experts: Option<OneOrManyExpertCount>,
moe_num_experts: Option<OneOrManyExpertCount>,
n_routed_experts: Option<OneOrManyExpertCount>,
num_local_experts: Option<OneOrManyExpertCount>,
block_configs: Vec<BlockConfig>,
text_config: Option<Box<ModelConfig>>,
}
/// Minimal subset of `generation_config.json`.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub(super) struct GenerationConfig {
pub eos_token_id: Option<OneOrManyTokenIds>,
pub temperature: Option<f32>,
pub top_p: Option<f32>,
pub top_k: Option<u32>,
pub min_p: Option<f32>,
pub repetition_penalty: Option<f32>,
pub max_new_tokens: Option<u32>,
}
/// HF generation configs allow either one EOS id or a list of EOS ids.
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub(super) enum OneOrManyTokenIds {
One(u32),
Many(Vec<u32>),
}
impl OneOrManyTokenIds {
pub(super) fn into_set(self) -> BTreeSet<u32> {
match self {
Self::One(id) => BTreeSet::from([id]),
Self::Many(ids) => ids.into_iter().collect(),
}
}
}
/// Hugging Face configs may expose the expert count either as one integer or
/// as a list of repeated integers.
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub(super) enum OneOrManyExpertCount {
One(u32),
Many(Vec<u32>),
}
impl OneOrManyExpertCount {
fn first_value(&self) -> u32 {
match self {
Self::One(value) => *value,
// Python currently takes the first value for list[int] expert
// counts in remote-code configs.
Self::Many(values) => values.first().copied().unwrap_or(0),
}
}
}
/// Heterogeneous block-level MoE metadata used as a fallback when no top-level
/// expert-count field is available.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub(super) struct BlockConfig {
pub block_type: String,
pub n_routed_experts: u32,
}
impl ModelConfig {
/// Return the config that the Rust frontend treats as the text/LLM config.
///
/// This is deliberately narrower than Python/transformers: we only support
/// either the top-level config itself or a single nested `text_config`.
fn effective_text_config(&self) -> &Self {
self.text_config.as_deref().unwrap_or(self)
}
/// Return the effective Hugging Face `model_type` used by the Rust
/// frontend.
///
/// This follows the same simplified text-config selection as the rest of
/// this type: the top-level config wins, otherwise a single nested
/// `text_config` may provide the value.
pub fn model_type(&self) -> Option<&str> {
self.model_type.as_deref().or_else(|| self.text_config.as_deref()?.model_type())
}
/// Reject partially nested `text_config` payloads that are unlikely to be
/// valid LLM configs for our current use.
///
/// This keeps the simplified Rust-side parsing honest: if a model declares
/// `text_config`, it must at least look like a real text model config.
fn validate_text_config_selection(&self) -> Result<()> {
if let Some(text_config) = self.text_config.as_deref()
&& text_config.num_attention_heads.is_none()
{
return Err(Error::Tokenizer(
"the text config extracted from the model config does not have `num_attention_heads`"
.to_string(),
));
}
Ok(())
}
/// Match Python's current expert-count priority on the selected text
/// config.
///
/// The only intentional simplification here is how we pick the text config:
/// Rust only looks at the top level or `text_config`, not the broader
/// transformers composite-config surface.
fn num_experts_from_block_configs(&self) -> u32 {
self.effective_text_config()
.block_configs
.iter()
.filter(|block| block.block_type == "moe")
.map(|block| block.n_routed_experts)
.max()
.unwrap_or(0)
}
pub(super) fn num_experts(&self) -> u32 {
let config = self.effective_text_config();
let direct = [
config.num_experts.as_ref(),
config.moe_num_experts.as_ref(),
config.n_routed_experts.as_ref(),
config.num_local_experts.as_ref(),
]
.into_iter()
.flatten()
.map(OneOrManyExpertCount::first_value)
.next()
.unwrap_or(0);
if direct > 0 {
direct
} else {
self.num_experts_from_block_configs()
}
}
pub(super) fn is_moe(&self) -> bool {
self.num_experts() > 0
}
pub(super) fn max_position_embeddings(&self) -> Option<u32> {
self.effective_text_config().max_position_embeddings
}
}
/// Load the tokenizer-side EOS metadata if a config file is present.
pub fn load_tokenizer_config(path: Option<&Path>) -> Result<HfTokenizerConfig> {
read_json_file(path)
}
/// Load the generation-side EOS metadata if a config file is present.
pub(super) fn load_generation_config(path: Option<&Path>) -> Result<GenerationConfig> {
read_json_file(path)
}
/// Load the model-side config (`config.json`) if present.
pub fn load_model_config(path: Option<&Path>) -> Result<ModelConfig> {
let config: ModelConfig = read_json_file(path)?;
config.validate_text_config_selection()?;
Ok(config)
}
fn read_json_file<T>(path: Option<&Path>) -> Result<T>
where
T: for<'de> Deserialize<'de> + Default,
{
let Some(path) = path else {
return Ok(T::default());
};
let content = fs::read_to_string(path).map_err(|error| {
Error::Tokenizer(format!(
"failed to read {}: {}",
path.display(),
error.as_report()
))
})?;
serde_json::from_str(&content).map_err(|error| {
Error::Tokenizer(format!(
"failed to parse {}: {}",
path.display(),
error.as_report()
))
})
}
#[cfg(test)]
mod tests {
use super::ModelConfig;
#[test]
fn model_config_detects_moe_from_named_expert_fields() {
let field_names = [
"num_experts",
"moe_num_experts",
"n_routed_experts",
"num_local_experts",
];
for field_name in field_names {
let config: ModelConfig =
serde_json::from_str(&format!(r#"{{"{field_name}": 64}}"#)).unwrap();
assert_eq!(config.num_experts(), 64, "field_name={field_name}");
assert!(config.is_moe(), "field_name={field_name}");
}
}
#[test]
fn model_config_uses_first_value_for_list_expert_counts() {
let config: ModelConfig = serde_json::from_str(r#"{"num_experts":[16,16]}"#).unwrap();
assert_eq!(config.num_experts(), 16);
assert!(config.is_moe());
}
#[test]
fn model_config_falls_back_to_block_configs_maximum() {
let config: ModelConfig = serde_json::from_str(
r#"{
"block_configs": [
{"block_type":"attention","n_routed_experts":9},
{"block_type":"moe","n_routed_experts":32},
{"block_type":"moe","n_routed_experts":64}
]
}"#,
)
.unwrap();
assert_eq!(config.num_experts(), 64);
assert!(config.is_moe());
}
#[test]
fn model_config_prefers_nested_text_config_like_python_hf_text_config() {
let config: ModelConfig = serde_json::from_str(
r#"{
"model_type": "top_level",
"num_experts": 64,
"max_position_embeddings": 8192,
"text_config": {
"model_type": "nested",
"num_attention_heads": 32,
"num_local_experts": 8,
"max_position_embeddings": 4096
}
}"#,
)
.unwrap();
assert_eq!(config.num_experts(), 8);
assert_eq!(config.model_type(), Some("top_level"));
assert_eq!(config.max_position_embeddings(), Some(4096));
assert!(config.is_moe());
}
#[test]
fn model_config_defaults_to_non_moe_when_no_expert_metadata_exists() {
let config: ModelConfig =
serde_json::from_str(r#"{"max_position_embeddings":4096}"#).unwrap();
assert_eq!(config.num_experts(), 0);
assert!(!config.is_moe());
assert_eq!(config.max_position_embeddings(), Some(4096));
}
#[test]
fn model_config_rejects_nested_text_config_without_attention_heads() {
let config: ModelConfig =
serde_json::from_str(r#"{"text_config":{"max_position_embeddings":4096}}"#).unwrap();
let error = config.validate_text_config_selection().unwrap_err();
assert!(error.to_string().contains("does not have `num_attention_heads`"),);
}
}
+120
View File
@@ -0,0 +1,120 @@
mod config;
mod model_files;
use std::collections::BTreeSet;
use std::sync::Arc;
use tracing::info;
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
use self::config::{GenerationConfig, load_generation_config};
pub use self::config::{
HfSpecialTokens, HfTokenizerConfig, ModelConfig, NamedSpecialToken, load_model_config,
load_tokenizer_config,
};
pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
use crate::backend::{SamplingHints, TextBackend};
use crate::error::Result;
fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {
match tokenizer {
TokenizerSource::HuggingFace(path) => Ok(Arc::new(HuggingFaceTokenizer::new(path)?)),
TokenizerSource::Tiktoken(path) => Ok(Arc::new(TiktokenTokenizer::new(path)?)),
TokenizerSource::Tekken(path) => Ok(Arc::new(TekkenTokenizer::new(path)?)),
}
}
/// [`TextBackend`] implementation built on Hugging Face model files.
pub struct HfTextBackend {
model_id: String,
files: ResolvedModelFiles,
tokenizer: DynTokenizer,
/// Primary EOS handled by engine-core's dedicated EOS path.
primary_eos_token_id: Option<u32>,
/// Additional EOS ids that should flow through stop-token handling.
extra_eos_token_ids: BTreeSet<u32>,
/// Generation-config for sampling defaults that may be inherited when the
/// user does not explicitly override them.
generation_config: GenerationConfig,
/// Model config (`config.json`).
model_config: ModelConfig,
}
impl HfTextBackend {
/// Load the text backend with the given model id.
pub async fn from_model(model_id: &str) -> Result<Self> {
let files = ResolvedModelFiles::new(model_id).await?;
Self::from_resolved_model_files(files, model_id.to_string())
}
/// Load the text backend from resolved Hugging Face model files.
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
let tokenizer = load_tokenizer(&files.tokenizer)?;
let primary_eos_token_id = tokenizer_config
.special_tokens
.eos_token
.as_ref()
.and_then(|token| tokenizer.token_to_id(token.as_str()));
let model_config = load_model_config(files.config_path.as_deref())?;
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
let mut extra_eos_token_ids = generation_config
.eos_token_id
.clone()
.map(|value| value.into_set())
.unwrap_or_default();
if let Some(primary_eos_token_id) = primary_eos_token_id {
extra_eos_token_ids.remove(&primary_eos_token_id);
}
info!(
model_id,
"loaded text backend with Hugging Face model files"
);
Ok(Self {
model_id,
files,
tokenizer,
primary_eos_token_id,
extra_eos_token_ids,
generation_config,
model_config,
})
}
/// Expose the resolved model files for use by the chat backend to load the
/// chat template.
pub fn resolved_model_files(&self) -> &ResolvedModelFiles {
&self.files
}
}
impl TextBackend for HfTextBackend {
fn tokenizer(&self) -> DynTokenizer {
self.tokenizer.clone()
}
fn is_moe(&self) -> bool {
self.model_config.is_moe()
}
fn model_id(&self) -> &str {
&self.model_id
}
fn sampling_hints(&self) -> Result<SamplingHints> {
Ok(SamplingHints {
primary_eos_token_id: self.primary_eos_token_id,
extra_eos_token_ids: self.extra_eos_token_ids.clone(),
default_temperature: self.generation_config.temperature,
default_top_p: self.generation_config.top_p,
default_top_k: self.generation_config.top_k,
default_min_p: self.generation_config.min_p,
default_repetition_penalty: self.generation_config.repetition_penalty,
default_max_tokens: self.generation_config.max_new_tokens,
max_model_len: self.model_config.max_position_embeddings(),
})
}
}
+459
View File
@@ -0,0 +1,459 @@
use std::path::{Path, PathBuf};
use hf_hub::Cache;
use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
use thiserror_ext::AsReport as _;
use super::config::{HfTokenizerConfig, load_tokenizer_config};
use crate::error::{Error, Result};
const HF_TOKEN_ENV: &str = "HF_TOKEN";
/// The tokenizer source selected for a model.
#[derive(Debug, Clone)]
pub enum TokenizerSource {
/// Path to `tokenizer.json` in HuggingFace format.
HuggingFace(PathBuf),
/// Path to `tiktoken.model` or `*.tiktoken` file for tiktoken-based models.
Tiktoken(PathBuf),
/// Path to `tekken.json` when present (Mistral native tokenizer format).
///
/// When set, the Tekken tokenizer should be preferred over the Hugging Face
/// tokenizer because the HuggingFace `tokenizer.json` for Mistral
/// models has a known regex bug that produces incorrect token IDs for
/// some inputs.
Tekken(PathBuf),
}
impl TokenizerSource {
pub fn path(&self) -> &Path {
match self {
Self::HuggingFace(path) | Self::Tiktoken(path) | Self::Tekken(path) => path,
}
}
}
/// Concrete tokenizer/config file locations resolved for one HF model id.
#[derive(Debug, Clone)]
pub struct ResolvedModelFiles {
/// The selected tokenizer source for this model.
pub tokenizer: TokenizerSource,
pub tokenizer_config_path: Option<PathBuf>,
pub generation_config_path: Option<PathBuf>,
pub preprocessor_config_path: Option<PathBuf>,
pub chat_template_path: Option<PathBuf>,
pub config_path: Option<PathBuf>,
}
impl ResolvedModelFiles {
/// Resolve tokenizer/config files from a local model directory first when
/// `model_id` points to one, otherwise consult the local HF cache and
/// finally the Hub.
pub async fn new(model_id: &str) -> Result<Self> {
if Path::new(model_id).is_dir() {
return resolve_local_model_files(Path::new(model_id));
}
if let Some(files) = resolve_cached_model_files(model_id)? {
return Ok(files);
}
resolve_remote_model_files(model_id).await
}
}
fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
let tokenizer_config_path = local_file_if_exists(model_dir, "tokenizer_config.json");
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
let tokenizer = resolve_local_tokenizer_source(model_dir, &tokenizer_config)?;
Ok(ResolvedModelFiles {
tokenizer,
tokenizer_config_path,
generation_config_path: local_file_if_exists(model_dir, "generation_config.json"),
preprocessor_config_path: local_file_if_exists(model_dir, "preprocessor_config.json"),
chat_template_path: discover_chat_template_in_dir(model_dir),
config_path: local_file_if_exists(model_dir, "config.json"),
})
}
async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles> {
let api = build_api().map_err(|error| Error::Tokenizer(error.to_report_string()))?;
let repo = api.model(model_id.to_string());
let info = repo.info().await.map_err(|error| {
Error::Tokenizer(format!(
"failed to fetch model '{model_id}': {}",
error.as_report()
))
})?;
let siblings = info
.siblings
.iter()
.map(|sibling| sibling.rfilename.as_str())
.collect::<std::collections::BTreeSet<_>>();
let tokenizer_config_path =
download_if_present(&repo, model_id, &siblings, "tokenizer_config.json").await?;
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
let tokenizer = resolve_remote_tokenizer_source(
&repo,
model_id,
&siblings,
tokenizer_config.tokenizer_class.as_deref(),
)
.await?;
let generation_config_path =
download_if_present(&repo, model_id, &siblings, "generation_config.json").await?;
let preprocessor_config_path =
download_if_present(&repo, model_id, &siblings, "preprocessor_config.json").await?;
let chat_template_name = siblings
.contains("chat_template.json")
.then_some("chat_template.json")
.or_else(|| siblings.contains("chat_template.jinja").then_some("chat_template.jinja"))
.or_else(|| siblings.iter().copied().find(|name| name.ends_with(".jinja")));
let chat_template_path = match chat_template_name {
Some(name) => Some(download_known_file(&repo, model_id, name).await?),
None => None,
};
let config_path = download_if_present(&repo, model_id, &siblings, "config.json").await?;
Ok(ResolvedModelFiles {
tokenizer,
tokenizer_config_path,
generation_config_path,
preprocessor_config_path,
chat_template_path,
config_path,
})
}
fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFiles>> {
let cache_repo = Cache::from_env().model(model_id.to_string());
let tokenizer_config_path = cache_repo.get("tokenizer_config.json");
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
let tokenizer = match resolve_cached_tokenizer_source(&cache_repo, &tokenizer_config)? {
Some(tokenizer) => tokenizer,
None => return Ok(None),
};
let model_dir = tokenizer.path().parent().ok_or_else(|| {
Error::Tokenizer("resolved tokenizer file has no parent directory".to_string())
})?;
let generation_config_path = cache_repo.get("generation_config.json");
let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
let chat_template_path = discover_chat_template_in_dir(model_dir);
let config_path = cache_repo.get("config.json");
Ok(Some(ResolvedModelFiles {
tokenizer,
tokenizer_config_path,
generation_config_path,
preprocessor_config_path,
chat_template_path,
config_path,
}))
}
async fn resolve_remote_tokenizer_source(
repo: &ApiRepo,
model_id: &str,
siblings: &std::collections::BTreeSet<&str>,
tokenizer_class: Option<&str>,
) -> Result<TokenizerSource> {
if let Some(tekken_path) = download_if_present(repo, model_id, siblings, "tekken.json").await? {
return Ok(TokenizerSource::Tekken(tekken_path));
}
let tokenizer_path = if siblings.contains("tokenizer.json") {
download_known_file(repo, model_id, "tokenizer.json").await?
} else if let Some(tiktoken_name) = find_tiktoken_sibling(siblings) {
download_known_file(repo, model_id, tiktoken_name).await?
} else {
return Err(Error::Tokenizer(format!(
"model '{model_id}' does not expose a supported tokenizer file \
(tokenizer.json, tiktoken.model, or *.tiktoken) on Hugging Face"
)));
};
Ok(resolve_tokenizer_source(
tokenizer_path,
tokenizer_class,
None,
))
}
fn resolve_cached_tokenizer_source(
cache_repo: &hf_hub::CacheRepo,
tokenizer_config: &HfTokenizerConfig,
) -> Result<Option<TokenizerSource>> {
let tekken_path = cache_repo.get("tekken.json");
if let Some(tekken_path) = tekken_path {
return Ok(Some(TokenizerSource::Tekken(tekken_path)));
}
let Some(tokenizer_path) = cache_repo.get("tokenizer.json").or_else(|| {
// tiktoken.model is the most common name, try it first.
cache_repo.get("tiktoken.model").or_else(|| {
// Scan for any *.tiktoken file in the cache snapshot directory.
let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
discover_tiktoken_in_dir(&snapshot_dir)
})
}) else {
return Ok(None);
};
Ok(Some(resolve_tokenizer_source(
tokenizer_path,
tokenizer_config.tokenizer_class.as_deref(),
None,
)))
}
fn resolve_local_tokenizer_source(
model_dir: &Path,
tokenizer_config: &HfTokenizerConfig,
) -> Result<TokenizerSource> {
let tekken_path = local_file_if_exists(model_dir, "tekken.json");
if let Some(tekken_path) = tekken_path {
return Ok(TokenizerSource::Tekken(tekken_path));
}
let tokenizer_path = local_file_if_exists(model_dir, "tokenizer.json")
.or_else(|| local_file_if_exists(model_dir, "tiktoken.model"))
.or_else(|| discover_tiktoken_in_dir(model_dir))
.ok_or_else(|| {
Error::Tokenizer(format!(
"local model directory '{}' does not contain a supported tokenizer file \
(tokenizer.json, tiktoken.model, or *.tiktoken)",
model_dir.display()
))
})?;
Ok(resolve_tokenizer_source(
tokenizer_path,
tokenizer_config.tokenizer_class.as_deref(),
None,
))
}
/// Choose the tokenizer.
///
/// Selection order:
/// 1. `tekken.json` — Mistral native tokenizer (preferred over HF `tokenizer.json` because the HF
/// version has a known regex bug for Mistral models).
/// 2. File extension — `.tiktoken` / `tiktoken.model` files use tiktoken from BPE data.
/// 3. `tokenizer_class` in `tokenizer_config.json` — classes containing "Tiktoken" (case-
/// insensitive) trigger tiktoken loading from a sibling BPE file.
/// 4. Default — `tokenizer.json` in HuggingFace format.
fn resolve_tokenizer_source(
tokenizer_path: PathBuf,
tokenizer_class: Option<&str>,
tekken_path: Option<PathBuf>,
) -> TokenizerSource {
if let Some(tekken_path) = tekken_path {
return TokenizerSource::Tekken(tekken_path);
}
if is_tiktoken_file(&tokenizer_path) {
return TokenizerSource::Tiktoken(tokenizer_path);
}
if tokenizer_class.is_some_and(|cls| cls.to_ascii_lowercase().contains("tiktoken"))
&& let Some(dir) = tokenizer_path.parent()
&& let Some(tiktoken_path) = discover_tiktoken_in_dir(dir)
{
return TokenizerSource::Tiktoken(tiktoken_path);
}
TokenizerSource::HuggingFace(tokenizer_path)
}
/// Download `filename` only if it exists in `siblings`.
async fn download_if_present(
repo: &ApiRepo,
model_id: &str,
siblings: &std::collections::BTreeSet<&str>,
filename: &str,
) -> Result<Option<PathBuf>> {
match siblings.contains(filename) {
true => download_known_file(repo, model_id, filename).await.map(Some),
false => Ok(None),
}
}
async fn download_known_file(repo: &ApiRepo, model_id: &str, filename: &str) -> Result<PathBuf> {
repo.get(filename).await.map_err(|error| {
Error::Tokenizer(format!(
"failed to download '{filename}' for model '{model_id}': {}",
error.as_report()
))
})
}
fn build_api() -> anyhow::Result<Api> {
let mut builder = ApiBuilder::from_env().with_progress(true);
if let Ok(token) = std::env::var(HF_TOKEN_ENV)
&& !token.is_empty()
{
builder = builder.with_token(Some(token));
}
Ok(builder.build()?)
}
fn local_file_if_exists(dir: &Path, filename: &str) -> Option<PathBuf> {
let path = dir.join(filename);
path.is_file().then_some(path)
}
/// Find a tiktoken file name among repo siblings, preferring `tiktoken.model`.
fn find_tiktoken_sibling<'a>(siblings: &std::collections::BTreeSet<&'a str>) -> Option<&'a str> {
if siblings.contains("tiktoken.model") {
return Some("tiktoken.model");
}
siblings.iter().copied().find(|name| name.ends_with(".tiktoken"))
}
/// Discover a tiktoken model file in a local directory.
pub(super) fn discover_tiktoken_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
let tiktoken_model = dir.join("tiktoken.model");
if tiktoken_model.exists() {
return Some(tiktoken_model);
}
std::fs::read_dir(dir).ok()?.flatten().find_map(|entry| {
let path = entry.path();
if path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.ends_with(".tiktoken"))
{
Some(path)
} else {
None
}
})
}
/// Returns `true` if `path` points to a tiktoken-format file (by name).
pub(super) fn is_tiktoken_file(path: &std::path::Path) -> bool {
path.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| name == "tiktoken.model" || name.ends_with(".tiktoken"))
}
/// Chat templates are sometimes stored as dedicated .jinja files rather than as
/// a fixed-name config entry, so we scan the cached model dir.
fn discover_chat_template_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
let json_template_path = dir.join("chat_template.json");
if json_template_path.exists() {
return Some(json_template_path);
}
let jinja_path = dir.join("chat_template.jinja");
if jinja_path.exists() {
return Some(jinja_path);
}
std::fs::read_dir(dir).ok()?.flatten().map(|entry| entry.path()).find(|path| {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(".jinja"))
})
}
#[cfg(test)]
mod tests {
use std::fs;
use tempfile::tempdir;
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
use super::{ResolvedModelFiles, TokenizerSource};
#[tokio::test]
async fn resolved_model_files_prefers_absolute_local_model_dir() {
let dir = tempdir().expect("create temp dir");
fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer");
fs::write(
dir.path().join("tokenizer_config.json"),
r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#,
)
.expect("write tokenizer config");
fs::write(dir.path().join("config.json"), "{}").expect("write config");
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"))
.await
.expect("resolve local model files");
match files.tokenizer {
TokenizerSource::HuggingFace(path) => {
assert_eq!(path, dir.path().join("tokenizer.json"));
}
other => panic!("expected HuggingFace tokenizer, got {other:?}"),
}
assert_eq!(files.config_path, Some(dir.path().join("config.json")));
assert_eq!(
files.tokenizer_config_path,
Some(dir.path().join("tokenizer_config.json"))
);
}
#[tokio::test]
#[ignore = "requires network access to Hugging Face and downloads the real Kimi K2.5 tokenizer"]
async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
.await
.expect("resolve real Kimi K2.5 model files");
let tokenizer_path = match &files.tokenizer {
TokenizerSource::Tiktoken(path) => path.clone(),
other => panic!("expected tiktoken tokenizer source, got {other:?}"),
};
for backend in [
TiktokenTokenizer::new_riptoken(&tokenizer_path).expect("load riptoken backend"),
TiktokenTokenizer::new_tiktoken_rs(&tokenizer_path).expect("load tiktoken-rs backend"),
] {
let think_id = backend.token_to_id("<think>").expect("resolve <think>");
let end_think_id = backend.token_to_id("</think>").expect("resolve </think>");
let tool_section_id = backend
.token_to_id("<|tool_calls_section_begin|>")
.expect("resolve tool call section marker");
let contraction_heavy_text =
"I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.";
let contraction_heavy_ids = backend.encode(contraction_heavy_text, false).unwrap();
assert_eq!(
(think_id, end_think_id, tool_section_id),
(163606, 163607, 163595)
);
assert_eq!(backend.decode(&[think_id], true).unwrap(), "<think>");
assert_eq!(backend.decode(&[end_think_id], true).unwrap(), "</think>");
assert_eq!(
backend.decode(&[tool_section_id], true).unwrap(),
"<|tool_calls_section_begin|>"
);
// This demonstrates that we're using Kimi's custom BPE pattern.
// With CL100K this will be 23 tokens instead.
assert_eq!(
contraction_heavy_ids,
vec![
17172, 3287, 4643, 8201, 11, 996, 374, 8971, 3637, 20020, 8173, 473, 4643,
1573, 56229, 13922, 13,
]
);
assert_eq!(contraction_heavy_ids.len(), 17);
assert_eq!(
backend.decode(&contraction_heavy_ids, false).unwrap(),
contraction_heavy_text
);
// Special-looking text that is not actually registered should fail gracefully.
assert_eq!(backend.token_to_id("◁think▷"), None);
assert_eq!(backend.token_to_id("<|definitely_not_registered|>"), None);
}
}
}
+47
View File
@@ -0,0 +1,47 @@
pub mod hf;
use std::sync::Arc;
use vllm_tokenizer::DynTokenizer;
use crate::error::Result;
/// Tokenizer/model-derived hints used to enrich text-generation requests before
/// they are lowered into engine-core.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SamplingHints {
pub primary_eos_token_id: Option<u32>,
pub extra_eos_token_ids: std::collections::BTreeSet<u32>,
pub default_temperature: Option<f32>,
pub default_top_p: Option<f32>,
pub default_top_k: Option<u32>,
pub default_min_p: Option<f32>,
pub default_repetition_penalty: Option<f32>,
pub default_max_tokens: Option<u32>,
/// Model context window size (`max_position_embeddings` from
/// `config.json`).
pub max_model_len: Option<u32>,
}
/// Minimal text-processing backend needed by `vllm-text`.
pub trait TextBackend: Send + Sync {
/// Return the tokenizer used by this backend.
fn tokenizer(&self) -> DynTokenizer;
/// Return whether the loaded model is a mixture-of-experts model.
fn is_moe(&self) -> bool {
false
}
/// Return the backend model ID.
fn model_id(&self) -> &str;
/// Return tokenizer/model-derived hints used to enrich southbound sampling
/// parameters.
fn sampling_hints(&self) -> Result<SamplingHints> {
Ok(SamplingHints::default())
}
}
/// Shared trait-object form of [`TextBackend`].
pub type DynTextBackend = Arc<dyn TextBackend>;
+30
View File
@@ -0,0 +1,30 @@
use thiserror::Error;
use vllm_engine_core_client::Error as EngineCoreError;
use vllm_llm::Error as LlmError;
#[derive(Debug, Error)]
pub enum Error {
#[error("tokenizer error: {0}")]
Tokenizer(String),
#[error("text request `{request_id}` must contain at least one prompt token ID")]
EmptyPromptTokenIds { request_id: String },
#[error(
"this model's maximum context length is {max_model_len} tokens, \
but the prompt contains {prompt_len} input tokens"
)]
PromptTooLong { max_model_len: u32, prompt_len: u32 },
#[error("text request stream `{request_id}` closed before terminal output")]
StreamClosedBeforeTerminalOutput { request_id: String },
#[error(transparent)]
Llm(#[from] LlmError),
#[error(transparent)]
EngineCore(#[from] EngineCoreError),
}
pub type Result<T> = std::result::Result<T, Error>;
impl From<vllm_tokenizer::TokenizerError> for Error {
fn from(error: vllm_tokenizer::TokenizerError) -> Self {
Self::Tokenizer(error.0)
}
}
+149
View File
@@ -0,0 +1,149 @@
//! Shared text-generation support used by chat and future raw completions.
//!
//! This crate intentionally stays below chat semantics:
//! prompt text handling, tokenizer/model loading, incremental detokenization,
//! and the thin generate-facing backend interface live here.
use std::mem::take;
pub use backend::{DynTextBackend, SamplingHints, TextBackend};
pub use error::{Error, Result};
use futures::Stream;
pub use lower::{
PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,
};
pub use output::{
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
};
pub use request::{Prompt, SamplingParams, TextRequest};
use trait_set::trait_set;
use vllm_engine_core_client::EngineCoreClient;
pub use vllm_llm::FinishReason;
use vllm_llm::{GenerateOutputStream, Llm};
use vllm_tokenizer::DynTokenizer;
pub mod backend;
mod error;
mod lower;
pub mod output;
mod request;
pub use vllm_tokenizer as tokenizer;
trait_set! {
/// Shared streamed text output type used by raw completions and other text-only northbound paths.
pub trait TextOutputStream = Stream<Item = Result<DecodedTextEvent>> + Send + 'static;
}
/// Raw text facade above [`Llm`].
///
/// This layer stays below chat semantics: prompt text or prompt token IDs flow
/// in, decoded text deltas and terminal metadata flow out.
pub struct TextLlm {
/// Generate-only client owned by this text facade.
llm: Llm,
/// Tokenizer/model metadata backend responsible for prompt encode/decode
/// and sampling hints.
backend: DynTextBackend,
/// Context window size derived by the backend or from engine startup
/// handshake, with optional override from config.
max_model_len: Option<u32>,
}
impl TextLlm {
/// Create a new text-generation facade from a shared LLM client plus a text
/// backend.
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
// Prefer the engine-reported max_model_len because it reflects the
// post-profiling, auto-fitted KV cache limit rather than static
// frontend metadata.
let max_model_len = llm.engine_core_client().max_model_len();
Self {
llm,
backend,
max_model_len,
}
}
/// Override the maximum model context length explicitly.
///
/// This takes priority over both the engine-reported default and any
/// tokenizer/model metadata exposed by the backend.
pub fn with_max_model_len(mut self, max_model_len: u32) -> Self {
self.max_model_len = Some(max_model_len);
self
}
/// Return the backend model ID.
pub fn model_id(&self) -> &str {
self.backend.model_id()
}
/// Expose the underlying engine-core client for low-level utility/admin
/// calls.
pub fn engine_core_client(&self) -> &EngineCoreClient {
self.llm.engine_core_client()
}
/// Return the tokenizer used by this text backend.
pub fn tokenizer(&self) -> DynTokenizer {
self.backend.tokenizer()
}
/// Tokenize if needed, lower to a generate request, and return the raw
/// token stream.
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {
let (_, raw_stream) = self.generate_inner(request).await?;
Ok(raw_stream)
}
/// Tokenize if needed, lower to a generate request, and stream
/// incrementally decoded text.
pub async fn generate(&self, request: TextRequest) -> Result<impl TextOutputStream> {
let (text_request, raw_stream) = self.generate_inner(request).await?;
let tokenizer = self.backend.tokenizer();
let decoded_stream = output::decoded_text_event_stream(
text_request.request_id,
tokenizer,
raw_stream,
text_request.decode_options,
text_request.intermediate,
);
Ok(decoded_stream)
}
async fn generate_inner(
&self,
mut request: TextRequest,
) -> Result<(TextRequest, GenerateOutputStream)> {
request.validate()?;
let tokenizer = self.backend.tokenizer();
let prompt_token_ids = match take(&mut request.prompt) {
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
Prompt::TokenIds(token_ids) => token_ids,
};
let mut sampling_hints = self.backend.sampling_hints()?;
if let Some(max_model_len) = self.max_model_len {
sampling_hints.max_model_len = Some(max_model_len);
}
let PreparedTextRequest {
text_request,
generate_request,
} = lower_text_request(request, prompt_token_ids, sampling_hints, &*tokenizer)?;
let raw_stream = self.llm.generate(generate_request).await?;
Ok((text_request, raw_stream))
}
/// Shut down the underlying LLM client and its background tasks.
pub async fn shutdown(self) -> Result<()> {
self.llm.shutdown().await?;
Ok(())
}
}
+739
View File
@@ -0,0 +1,739 @@
use std::collections::BTreeSet;
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
use vllm_llm::GenerateRequest;
use vllm_tokenizer::Tokenizer;
use crate::backend::SamplingHints;
use crate::error::{Error, Result};
use crate::request::{SamplingParams, TextRequest};
/// One text request after it has been lowered into the raw generate boundary.
#[derive(Debug)]
pub struct PreparedTextRequest {
/// The original high-level request, preserved for response-side metadata
/// and decoding options.
pub text_request: TextRequest,
/// The southbound request ready to be sent to `vllm-llm`.
pub generate_request: GenerateRequest,
}
/// Convert a high-level [`TextRequest`] into one lower-level
/// [`GenerateRequest`] ready for the `llm` crate.
pub fn lower_text_request(
request: TextRequest,
prompt_token_ids: Vec<u32>,
sampling_hints: SamplingHints,
tokenizer: &dyn Tokenizer,
) -> Result<PreparedTextRequest> {
let prompt_len = prompt_token_ids.len() as u32;
let generate_request = GenerateRequest {
request_id: request.request_id.clone(),
prompt_token_ids,
mm_features: request.mm_features.clone(),
sampling_params: lower_sampling_params(
request.sampling_params.clone(),
sampling_hints,
prompt_len,
tokenizer,
)?,
cache_salt: request.cache_salt.clone(),
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
// Fields below are currently placeholders.
arrival_time: None,
trace_headers: None,
reasoning_ended: None,
lora_request: None,
};
Ok(PreparedTextRequest {
text_request: request,
generate_request,
})
}
/// Convert [`SamplingParams`] into [`EngineCoreSamplingParams`], enriching
/// omitted user values with tokenizer/model-derived hints when available.
pub fn lower_sampling_params(
sampling_params: SamplingParams,
SamplingHints {
primary_eos_token_id,
extra_eos_token_ids,
default_temperature,
default_top_p,
default_top_k,
default_min_p,
default_repetition_penalty,
default_max_tokens,
max_model_len,
}: SamplingHints,
prompt_len: u32,
tokenizer: &dyn Tokenizer,
) -> Result<EngineCoreSamplingParams> {
let SamplingParams {
temperature,
top_p,
top_k,
seed,
max_tokens,
min_tokens,
logprobs,
prompt_logprobs,
min_p,
frequency_penalty,
presence_penalty,
repetition_penalty,
stop_token_ids,
ignore_eos,
logit_bias,
allowed_token_ids,
bad_words,
logprob_token_ids,
structured_outputs,
skip_reading_prefix_cache,
vllm_xargs,
} = sampling_params;
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
// If neither the caller nor the model provides a value, fall back to 1.0 — the
// default used by the Python vLLM OpenAI-compatible API (via
// `_DEFAULT_SAMPLING_PARAMS`).
let temperature = temperature.or(default_temperature).unwrap_or(1.0);
let top_p = top_p.or(default_top_p).unwrap_or(1.0);
let top_k = top_k.or(default_top_k).unwrap_or(0);
let min_p = min_p.or(default_min_p).unwrap_or(0.0);
let repetition_penalty = repetition_penalty.or(default_repetition_penalty).unwrap_or(1.0);
let max_tokens = resolve_max_tokens(max_tokens, default_max_tokens, max_model_len, prompt_len)?;
let min_tokens = min_tokens.unwrap_or(0);
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
let presence_penalty = presence_penalty.unwrap_or(0.0);
let mut stop_token_ids = stop_token_ids.unwrap_or_default();
let mut all_stop_token_ids = BTreeSet::from_iter(stop_token_ids.iter().copied());
if let Some(primary_eos_token_id) = primary_eos_token_id {
all_stop_token_ids.insert(primary_eos_token_id);
}
all_stop_token_ids.extend(extra_eos_token_ids.iter().copied());
if !ignore_eos {
merge_unique_token_ids(&mut stop_token_ids, extra_eos_token_ids.iter().copied());
}
Ok(EngineCoreSamplingParams {
temperature,
top_p,
top_k,
seed,
max_tokens,
min_tokens,
logprobs,
prompt_logprobs,
min_p,
frequency_penalty,
presence_penalty,
repetition_penalty,
stop_token_ids,
eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(),
all_stop_token_ids,
logit_bias,
allowed_token_ids,
bad_words_token_ids: tokenize_bad_words(bad_words.as_deref(), tokenizer)?,
structured_outputs,
logprob_token_ids,
skip_reading_prefix_cache,
extra_args: vllm_xargs,
})
}
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
/// logic in `SamplingParams.update_from_tokenizer()`.
///
/// Each word is encoded both with and without a leading space so that the ban
/// applies regardless of whether the word appears at the beginning or in the
/// middle of generated text (this accounts for tokenizers that use an
/// `add_prefix_space` convention).
///
/// Reference: <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/sampling_params.py#L555-L594>
fn tokenize_bad_words(
bad_words: Option<&[String]>,
tokenizer: &dyn Tokenizer,
) -> Result<Option<Vec<Vec<u32>>>> {
let bad_words = bad_words.filter(|w| !w.is_empty());
let mut all_token_ids = Vec::new();
for bad_word in bad_words.into_iter().flatten() {
// Without a leading space we always keep the encoding.
// With a leading space we only keep it when the prefix-space variant produces a
// distinct first token but the same sequence length — this mirrors the Python
// dedup condition that avoids redundant entries.
let without_space = tokenizer.encode(bad_word, false)?;
let with_space = tokenizer.encode(&format!(" {}", bad_word.trim_start()), false)?;
if !without_space.is_empty() {
all_token_ids.push(without_space);
}
if !with_space.is_empty()
&& all_token_ids.last().is_some_and(|prev: &Vec<u32>| {
with_space[0] != prev[0] && with_space.len() == prev.len()
})
{
all_token_ids.push(with_space);
}
}
Ok((!all_token_ids.is_empty()).then_some(all_token_ids))
}
/// Resolve the effective `max_tokens` for generation, mirroring vLLM Python's
/// `get_max_tokens()` in `vllm/entrypoints/utils.py`.
///
/// Takes the minimum of all available limits (user-specified, generation-config
/// default, and `max_model_len - prompt_len`). When nothing is known, falls
/// back to `u32::MAX` so the engine-core can apply its own context-window
/// limit.
pub fn resolve_max_tokens(
user_max_tokens: Option<u32>,
default_max_tokens: Option<u32>,
max_model_len: Option<u32>,
prompt_len: u32,
) -> Result<u32> {
let model_max_tokens = match max_model_len {
Some(max_model_len) if prompt_len >= max_model_len => {
return Err(Error::PromptTooLong {
max_model_len,
prompt_len,
});
}
Some(max_model_len) => Some(max_model_len - prompt_len),
None => None,
};
let fallback_max_tokens = user_max_tokens.or(default_max_tokens);
Ok([fallback_max_tokens, model_max_tokens]
.into_iter()
.flatten()
.min()
.unwrap_or(u32::MAX /* TODO: a reasonable fallback? */))
}
fn merge_unique_token_ids(
stop_token_ids: &mut Vec<u32>,
extra_token_ids: impl Iterator<Item = u32>,
) {
// Keep user-provided ordering stable while still folding in backend-derived EOS
// aliases.
for token_id in extra_token_ids {
if !stop_token_ids.contains(&token_id) {
stop_token_ids.push(token_id);
}
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use crate::backend::hf::HfTextBackend;
use crate::backend::{SamplingHints, TextBackend as _};
use crate::request::{Prompt, TextRequest};
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
/// don't exercise bad-words tokenization.
struct StubTokenizer;
impl Tokenizer for StubTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
fn stub_tokenizer() -> StubTokenizer {
StubTokenizer
}
fn sample_request() -> TextRequest {
TextRequest {
prompt: Prompt::TokenIds(vec![1, 2, 3]),
request_id: "text-1".to_string(),
..TextRequest::for_test()
}
}
fn sample_sampling_hints() -> SamplingHints {
SamplingHints {
primary_eos_token_id: Some(99),
extra_eos_token_ids: BTreeSet::from([77]),
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
}
}
#[test]
fn lower_text_request_applies_python_style_eos_hints() {
let prepared = lower_text_request(
sample_request(),
vec![1, 2, 3],
sample_sampling_hints(),
&stub_tokenizer(),
)
.unwrap();
let params = prepared.generate_request.sampling_params;
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: [
77,
],
eos_token_id: Some(
99,
),
all_stop_token_ids: {
77,
99,
},
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
"#]]
.assert_debug_eq(&params);
}
#[test]
fn lower_text_request_respects_ignore_eos_for_stop_token_ids() {
let mut request = sample_request();
request.sampling_params.ignore_eos = true;
let prepared = lower_text_request(
request,
vec![1, 2, 3],
sample_sampling_hints(),
&stub_tokenizer(),
)
.unwrap();
let params = prepared.generate_request.sampling_params;
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {
77,
99,
},
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
"#]]
.assert_debug_eq(&params);
}
#[tokio::test]
#[ignore = "requires network access to Hugging Face"]
async fn lower_text_request_uses_real_qwen_generation_defaults() {
let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")
.await
.expect("load qwen tokenizer and generation config");
let hints = backend.sampling_hints().expect("collect sampling hints");
expect_test::expect![[r#"
SamplingHints {
primary_eos_token_id: Some(
151645,
),
extra_eos_token_ids: {
151643,
},
default_temperature: Some(
0.6,
),
default_top_p: Some(
0.95,
),
default_top_k: Some(
20,
),
default_min_p: Some(
0.1,
),
default_repetition_penalty: Some(
1.2,
),
default_max_tokens: None,
max_model_len: Some(
40960,
),
}
"#]]
.assert_debug_eq(&hints);
let prepared =
lower_text_request(sample_request(), vec![1, 2, 3], hints, &stub_tokenizer())
.expect("lower request");
let params = prepared.generate_request.sampling_params;
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 0.6,
top_p: 0.95,
top_k: 20,
seed: None,
max_tokens: 40957,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.2,
stop_token_ids: [
151643,
],
eos_token_id: Some(
151645,
),
all_stop_token_ids: {
151643,
151645,
},
}
"#]]
.assert_debug_eq(&params);
}
#[test]
fn lower_sampling_params_preserves_explicit_stop_token_ids_in_all_stop_set() {
let sampling_params = SamplingParams {
stop_token_ids: Some(vec![11, 77]),
..SamplingParams::default()
};
let params = lower_sampling_params(
sampling_params,
SamplingHints {
primary_eos_token_id: Some(99),
extra_eos_token_ids: BTreeSet::from([77, 88]),
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
},
3,
&stub_tokenizer(),
)
.unwrap();
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: [
11,
77,
88,
],
eos_token_id: Some(
99,
),
all_stop_token_ids: {
11,
77,
88,
99,
},
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
"#]]
.assert_debug_eq(&params);
}
#[test]
fn lower_sampling_params_prefers_user_values_over_generation_defaults() {
let sampling_params = SamplingParams {
temperature: Some(0.2),
top_p: Some(0.3),
top_k: Some(4),
max_tokens: Some(32),
min_tokens: Some(2),
..Default::default()
};
let params = lower_sampling_params(
sampling_params,
SamplingHints {
primary_eos_token_id: None,
extra_eos_token_ids: BTreeSet::new(),
default_temperature: Some(0.8),
default_top_p: Some(0.9),
default_top_k: Some(12),
default_min_p: Some(0.1),
default_repetition_penalty: Some(1.2),
default_max_tokens: Some(128),
max_model_len: None,
},
3,
&stub_tokenizer(),
)
.unwrap();
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 0.2,
top_p: 0.3,
top_k: 4,
seed: None,
max_tokens: 32,
min_tokens: 2,
logprobs: None,
prompt_logprobs: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.2,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {},
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
"#]]
.assert_debug_eq(&params);
}
#[test]
fn lower_sampling_params_passes_logprobs_fields_through() {
let sampling_params = SamplingParams {
logprobs: Some(3),
prompt_logprobs: Some(-1),
..Default::default()
};
let params = lower_sampling_params(
sampling_params,
SamplingHints {
primary_eos_token_id: None,
extra_eos_token_ids: BTreeSet::new(),
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
},
3,
&stub_tokenizer(),
)
.unwrap();
assert_eq!(params.logprobs, Some(3));
assert_eq!(params.prompt_logprobs, Some(-1));
}
#[test]
fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() {
let params = lower_sampling_params(
SamplingParams::default(),
SamplingHints {
primary_eos_token_id: None,
extra_eos_token_ids: BTreeSet::new(),
default_temperature: Some(0.8),
default_top_p: Some(0.9),
default_top_k: Some(12),
default_min_p: Some(0.1),
default_repetition_penalty: Some(1.2),
default_max_tokens: Some(128),
max_model_len: None,
},
3,
&stub_tokenizer(),
)
.unwrap();
expect_test::expect![[r#"
EngineCoreSamplingParams {
temperature: 0.8,
top_p: 0.9,
top_k: 12,
seed: None,
max_tokens: 128,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.2,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {},
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
"#]]
.assert_debug_eq(&params);
}
#[test]
fn resolve_max_tokens_caps_by_model_len() {
let result = resolve_max_tokens(Some(150), None, Some(200), 100);
assert_eq!(result.unwrap(), 100);
}
#[test]
fn lower_text_request_preserves_non_streaming_request_metadata() {
let mut request = sample_request();
request.intermediate = false;
let prepared = lower_text_request(
request,
vec![1, 2, 3],
sample_sampling_hints(),
&stub_tokenizer(),
)
.unwrap();
assert!(!prepared.text_request.intermediate);
assert_eq!(prepared.generate_request.request_id, "text-1");
}
#[test]
fn resolve_max_tokens_user_smaller_than_model_limit() {
let result = resolve_max_tokens(Some(50), None, Some(200), 100);
assert_eq!(result.unwrap(), 50);
}
#[test]
fn resolve_max_tokens_uses_default_when_user_omits() {
let result = resolve_max_tokens(None, Some(64), Some(200), 100);
assert_eq!(result.unwrap(), 64);
}
#[test]
fn resolve_max_tokens_default_capped_by_model_len() {
let result = resolve_max_tokens(None, Some(256), Some(200), 100);
assert_eq!(result.unwrap(), 100);
}
#[test]
fn resolve_max_tokens_no_model_len_falls_back() {
let result = resolve_max_tokens(Some(9999), None, None, 100);
assert_eq!(result.unwrap(), 9999);
}
#[test]
fn resolve_max_tokens_no_limits_known_falls_back_to_u32_max() {
let result = resolve_max_tokens(None, None, None, 100);
assert_eq!(result.unwrap(), u32::MAX);
}
#[test]
fn resolve_max_tokens_prompt_too_long() {
let result = resolve_max_tokens(Some(10), None, Some(100), 100);
assert!(matches!(
result,
Err(Error::PromptTooLong {
max_model_len: 100,
prompt_len: 100,
})
));
}
#[test]
fn resolve_max_tokens_prompt_exceeds_model_len() {
let result = resolve_max_tokens(Some(10), None, Some(100), 200);
assert!(matches!(
result,
Err(Error::PromptTooLong {
max_model_len: 100,
prompt_len: 200,
})
));
}
}
+607
View File
@@ -0,0 +1,607 @@
use std::sync::Arc;
use asynk_strim_attr::{TryYielder, try_stream};
use futures::{Stream, StreamExt};
use serde::{Deserialize, Serialize};
use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::StopReason;
use vllm_llm::{FinishReason, GenerateOutput};
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
use super::logprobs::{
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
};
use crate::error::Error;
/// Request-neutral options for incremental text decoding.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TextDecodeOptions {
pub skip_special_tokens: bool,
pub include_stop_str_in_output: bool,
pub stop_strings: Option<Vec<String>>,
/// Minimum number of tokens to generate before stop-string checking kicks
/// in. Stop strings found within the first `min_tokens` tokens are
/// ignored.
pub min_tokens: u32,
}
impl Default for TextDecodeOptions {
fn default() -> Self {
Self {
skip_special_tokens: true,
include_stop_str_in_output: false,
stop_strings: None,
min_tokens: 0,
}
}
}
/// Terminal metadata carried on the final [`DecodedTextEvent`].
#[derive(Debug, Clone, PartialEq)]
pub struct Finished {
pub prompt_token_count: usize,
pub output_token_count: usize,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
/// Internal decoded-text event emitted before higher-level assistant
/// adaptation.
#[derive(Debug, Clone, PartialEq)]
pub enum DecodedTextEvent {
/// The request has reached the point where prompt-scoped decoding metadata
/// is ready.
Start {
/// The actual prompt token IDs for this request.
prompt_token_ids: Arc<[u32]>,
/// Once-only prompt logprobs metadata, when requested.
///
/// The first prompt token is carried separately because it has no left
/// context to score against; `scored_positions` covers the
/// remaining prompt positions.
prompt_logprobs: Option<DecodedPromptLogprobs>,
},
/// A delta of text has been decoded, optionally alongside token-position
/// logprobs.
///
/// `delta` is the newly visible decoded text fragment for this update.
///
/// `logprobs` covers the newly generated token positions from the same
/// update, but is not guaranteed to align with `delta` by character
/// span. One update may carry token logprobs but no newly visible text
/// yet, and one visible text fragment may reflect multiple token
/// positions becoming decodable together.
///
/// Upper-level may further parse `delta` as reasoning or tool calls.
///
/// When `finished` is `Some`, this is the terminal event for the request.
TextDelta {
delta: String,
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
},
}
/// Convert the output token stream from the `vllm_llm` layer into incrementally
/// decoded text.
#[try_stream]
pub async fn decoded_text_event_stream(
request_id: String,
tokenizer: DynTokenizer,
mut raw_stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>> + Unpin,
mut decode_options: TextDecodeOptions,
intermediate: bool,
mut y: TryYielder<DecodedTextEvent, Error>,
) -> crate::Result<()> {
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
let mut prompt_token_count = 0_usize;
let mut token_ids = Vec::new();
let mut output_token_count: usize = 0;
let mut logprobs: Option<DecodedLogprobs> = None;
while let Some(next) = raw_stream.next().await {
let output = next?;
// If it's the first output, init states and yield `Start` event.
if decoder.is_none() {
let prompt_token_ids =
output.prompt_token_ids().expect("first llm output must carry prompt token ids");
prompt_token_count = prompt_token_ids.len();
let dec = tokenizer.create_decode_stream(
prompt_token_ids,
decode_options.skip_special_tokens,
// If we are excluding stop strings from output, we need to buffer
// the output so that we don't return the beginning of a stop string
// when streaming the outputs.
match decode_options.include_stop_str_in_output {
true => 0,
false => {
decode_options
.stop_strings
.as_ref()
.and_then(|stops| stops.iter().map(|ss| ss.len()).max())
.unwrap_or(1)
- 1
}
},
);
decoder = Some(dec);
y.yield_ok(DecodedTextEvent::Start {
prompt_token_ids: prompt_token_ids.clone(),
prompt_logprobs: output
.prompt_logprobs()
.map(|logprobs| {
decode_prompt_logprobs(
tokenizer.as_ref(),
prompt_token_ids,
logprobs,
decode_options.skip_special_tokens,
)
})
.transpose()?,
})
.await;
};
let decoder = decoder.as_mut().unwrap();
let kv_transfer_params = output.kv_transfer_params;
let mut finish_reason = output.finish_reason;
let mut stop_str_matched = false;
let suppress_terminal_stop_token = finish_reason.as_ref().is_some_and(|r| r.is_stop())
&& !decode_options.include_stop_str_in_output;
let decodable_token_ids = if suppress_terminal_stop_token {
// Match Python V1 token-stop detokenization by keeping the stop token
// in metadata while excluding it from user-visible text.
output.token_ids.split_last().map(|(_, rest)| rest).unwrap_or(&[])
} else {
&output.token_ids
};
let mut delta: Option<String> = None;
let mut truncate_output_to = None;
let mut truncate_tokens_to = None;
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
let new_bytes = decoder.push_token(token_id)?;
if output_token_count + tok_idx + 1 > decode_options.min_tokens as usize
&& let Some(stops) = decode_options.stop_strings.as_mut()
&& let Some((idx, off)) = matches_stop_string(stops, decoder.output(), new_bytes)
{
let stop_str = stops.swap_remove(idx);
truncate_output_to = match decode_options.include_stop_str_in_output {
true => Some(off + stop_str.len()),
false => Some(off),
};
finish_reason = Some(FinishReason::Stop(Some(StopReason::Text(stop_str))));
truncate_tokens_to = Some(tok_idx + 1);
stop_str_matched = true;
break;
}
if intermediate && let Some(chunk) = decoder.next_chunk() {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
}
}
let mut new_token_ids = output.token_ids;
let mut new_logprobs = output.logprobs;
// Trim tokens and logprobs if we matched stop string.
if let Some(num_tokens) = truncate_tokens_to {
new_token_ids.truncate(num_tokens);
if let Some(logprobs) = &mut new_logprobs {
logprobs.positions.truncate(num_tokens);
}
}
output_token_count += new_token_ids.len();
let decoded_logprobs = new_logprobs
.as_ref()
.map(|logprobs| {
decode_logprobs(
tokenizer.as_ref(),
logprobs,
decode_options.skip_special_tokens,
)
})
.transpose()?;
if !intermediate {
token_ids.extend(&new_token_ids);
if let Some(dlp) = decoded_logprobs.as_ref() {
logprobs
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
.positions
.extend_from_slice(&dlp.positions);
}
}
if let Some(reason) = finish_reason {
// Flush any remaining buffered text.
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
let text_len = text.len();
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
if intermediate {
if let Some(chunk) = last_chunk {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
}
token_ids = new_token_ids;
logprobs = decoded_logprobs;
text = delta.unwrap_or_default();
}
debug!(
finish_reason = ?reason,
text_length_bytes = text_len,
output_token_count = output_token_count,
"request finished with terminal output"
);
if let Some(full_text) = full_text {
trace!(full_text, "request finished with terminal decoded text");
}
// Intentionally drop the stream with explicit cause, so that the engine core
// can distinguish between such normal completion vs an unexpected
// early drop.
if stop_str_matched {
AbortCause::StopStringMatched.drop_as(raw_stream);
}
y.yield_ok(DecodedTextEvent::TextDelta {
delta: text,
token_ids,
logprobs,
finished: Some(Finished {
prompt_token_count,
output_token_count,
finish_reason: reason,
kv_transfer_params,
}),
})
.await;
return Ok(());
}
if intermediate {
y.yield_ok(DecodedTextEvent::TextDelta {
delta: delta.unwrap_or_default(),
token_ids: new_token_ids,
logprobs: decoded_logprobs,
finished: None,
})
.await;
}
}
Err(Error::StreamClosedBeforeTerminalOutput { request_id })
}
/// If stop string matches, returns tuple
/// (index into stop string vec, byte index of first byte of stop string in
/// output)
fn matches_stop_string(stops: &[String], output: &str, new_bytes: usize) -> Option<(usize, usize)> {
// We compare byte subslices to avoid utf8 boundary problem
let output = output.as_bytes();
let next_off = (output.len() + 1) - new_bytes;
stops
.iter()
.map(|ss| (ss.as_bytes(), ss.len(), next_off.saturating_sub(ss.len())))
.enumerate()
.find_map(|(ss_idx, (ss, len, start_off))| {
output[start_off..]
.windows(len)
.rposition(|w| w == ss)
.map(|pos| (ss_idx, start_off + pos))
})
}
#[cfg(test)]
mod tests {
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use futures::{Stream, stream};
use vllm_engine_core_client::AbortCause;
use vllm_llm::GenerateOutput;
use vllm_tokenizer::Tokenizer;
use super::*;
use crate::output::TextOutputStreamExt as _;
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
struct ByteTokenizer;
impl Tokenizer for ByteTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
unreachable!()
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
/// Helper: run `decoded_text_event_stream` to completion and return the
/// collected output.
async fn run_to_completion(
token_ids: Vec<u32>,
decode_options: TextDecodeOptions,
) -> crate::output::CollectedTextOutput {
let prompt: Arc<[u32]> = Arc::from([]);
let raw_stream = stream::iter(vec![Ok(GenerateOutput::for_test(
Some(prompt),
token_ids,
Some(FinishReason::Length),
))]);
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)
.collect_output()
.await
.unwrap()
}
/// Convert ASCII string to token IDs (one byte per token).
fn ascii_tokens(s: &str) -> Vec<u32> {
s.bytes().map(u32::from).collect()
}
fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions {
TextDecodeOptions {
stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()),
min_tokens,
..Default::default()
}
}
struct DropRecordingStream {
next: Option<vllm_llm::Result<GenerateOutput>>,
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
}
impl Stream for DropRecordingStream {
type Item = vllm_llm::Result<GenerateOutput>;
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Ready(self.next.take())
}
}
impl Drop for DropRecordingStream {
fn drop(&mut self) {
*self.dropped_cause.lock().unwrap() = Some(AbortCause::current());
}
}
// --- stop string stream tests ---
#[tokio::test]
async fn stream_stop_string_sets_task_local_abort_cause_on_raw_stream_drop() {
let prompt: Arc<[u32]> = Arc::from([]);
let dropped_cause = Arc::new(Mutex::new(None));
let raw_stream = DropRecordingStream {
next: Some(Ok(GenerateOutput::for_test(
Some(prompt),
ascii_tokens("hello"),
Some(FinishReason::Length),
))),
dropped_cause: Arc::clone(&dropped_cause),
};
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
let output = decoded_text_event_stream(
"test".into(),
tokenizer,
raw_stream,
opts(&["ll"], 0),
false,
)
.collect_output()
.await
.unwrap();
assert_eq!(output.text, "he");
assert!(output.finish_reason.is_stop());
assert_eq!(
*dropped_cause.lock().unwrap(),
Some(AbortCause::StopStringMatched)
);
}
#[tokio::test]
async fn stream_stop_string_truncates_at_match() {
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 0)).await;
assert_eq!(output.text, "h");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_at_end() {
let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await;
assert_eq!(output.text, "abc");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_first_token() {
let output = run_to_completion(ascii_tokens("xhello"), opts(&["x"], 0)).await;
assert_eq!(output.text, "");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_no_match_runs_to_completion() {
let output = run_to_completion(ascii_tokens("hello"), opts(&["z"], 0)).await;
assert_eq!(output.text, "hello");
assert_eq!(output.finish_reason, FinishReason::Length);
}
#[tokio::test]
async fn stream_stop_string_multi_char() {
let output = run_to_completion(ascii_tokens("say hello world"), opts(&["lo"], 0)).await;
assert_eq!(output.text, "say hel");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_first_of_multiple_wins() {
// Both "ll" and "lo" are present; "ll" appears first in the output.
let output = run_to_completion(ascii_tokens("hello"), opts(&["ll", "lo"], 0)).await;
assert_eq!(output.text, "he");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn stream_stop_string_include_in_output() {
let output = run_to_completion(
ascii_tokens("hello"),
TextDecodeOptions {
stop_strings: Some(vec!["ll".to_string()]),
include_stop_str_in_output: true,
..Default::default()
},
)
.await;
assert_eq!(output.text, "hell");
assert!(output.finish_reason.is_stop());
}
// --- min_tokens + stop string interaction ---
#[tokio::test]
async fn min_tokens_suppresses_early_stop_string() {
// stop="e", min_tokens=3: the 'e' at token 2 is within the first 3 tokens,
// so it should be skipped. No later 'e' exists, so output runs to completion.
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 3)).await;
assert_eq!(output.text, "hello");
assert_eq!(output.finish_reason, FinishReason::Length);
}
#[tokio::test]
async fn min_tokens_allows_stop_string_after_threshold() {
// stop="e", min_tokens=2: the first 'e' at token 3 is past the threshold.
let output = run_to_completion(ascii_tokens("greet"), opts(&["e"], 2)).await;
assert_eq!(output.text, "gr");
assert!(output.finish_reason.is_stop());
}
#[tokio::test]
async fn min_tokens_zero_behaves_like_absent() {
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 0)).await;
assert_eq!(output.text, "h");
assert!(output.finish_reason.is_stop());
}
#[test]
fn stop_string_matches_at_end() {
let stops = vec!["wor".to_string()];
// Output: "say wor", last byte 'r' was just added (new_bytes=1)
let result = matches_stop_string(&stops, "say wor", 1);
assert_eq!(result, Some((0, 4)));
}
#[test]
fn stop_string_no_match() {
let stops = vec!["xyz".to_string()];
let result = matches_stop_string(&stops, "say wor", 1);
assert_eq!(result, None);
}
#[test]
fn stop_string_matches_first_of_multiple() {
let stops = vec!["wor".to_string(), "say".to_string()];
// "say" appears earlier but "wor" is checked first (index 0)
let result = matches_stop_string(&stops, "say wor", 1);
assert_eq!(result, Some((0, 4)));
}
#[test]
fn stop_string_matches_second_of_multiple() {
let stops = vec!["xyz".to_string(), "wor".to_string()];
let result = matches_stop_string(&stops, "say wor", 1);
assert_eq!(result, Some((1, 4)));
}
#[test]
fn stop_string_matches_with_multiple_new_bytes() {
let stops = vec!["wor".to_string()];
// "say wor" where last 3 bytes "wor" were added at once
let result = matches_stop_string(&stops, "say wor", 3);
assert_eq!(result, Some((0, 4)));
}
#[test]
fn stop_string_matches_at_beginning() {
let stops = vec!["say".to_string()];
let result = matches_stop_string(&stops, "say wor", 7);
assert_eq!(result, Some((0, 0)));
}
#[test]
fn stop_string_exact_output() {
let stops = vec!["abc".to_string()];
let result = matches_stop_string(&stops, "abc", 3);
assert_eq!(result, Some((0, 0)));
}
#[test]
fn stop_string_single_char() {
let stops = vec!["!".to_string()];
let result = matches_stop_string(&stops, "hello!", 1);
assert_eq!(result, Some((0, 5)));
}
#[test]
fn stop_string_not_in_new_bytes_region() {
let stops = vec!["say".to_string()];
// "say" is in the output but before the new byte region.
// new_bytes=1 means only 'r' was added; "say" ended at byte 3,
// but the search window starts at next_off - stop_len = 7+1-1 - 3 = 4.
let result = matches_stop_string(&stops, "say wor", 1);
assert_eq!(result, None);
}
#[test]
fn stop_string_empty_list() {
let stops: Vec<String> = vec![];
let result = matches_stop_string(&stops, "hello", 1);
assert_eq!(result, None);
}
#[test]
fn stop_string_multibyte_utf8() {
let stops = vec!["世界".to_string()];
// "你好世界" is 12 bytes: 你(3) + 好(3) + 世(3) + 界(3)
// "世界" starts at byte 6
let result = matches_stop_string(&stops, "你好世界", 3);
assert_eq!(result, Some((0, 6)));
}
}
+236
View File
@@ -0,0 +1,236 @@
use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use vllm_llm::{Logprobs, PositionLogprobs};
use vllm_tokenizer::Tokenizer;
use crate::error::Error;
/// One decoded token candidate and its logprob metadata.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecodedTokenLogprob {
/// Original vocabulary token ID for this candidate.
pub token_id: u32,
/// Best-effort decoded token string for this candidate.
pub token: String,
/// Log probability of this token candidate.
pub logprob: f32,
/// Vocabulary rank of this token candidate.
pub rank: u32,
}
/// One position's decoded token candidates and their logprobs.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecodedPositionLogprobs {
/// Candidate tokens for this position.
pub entries: Vec<DecodedTokenLogprob>,
}
/// Decoded sample logprobs for generated token positions.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecodedLogprobs {
/// Generated token positions covered by this payload.
pub positions: Vec<DecodedPositionLogprobs>,
}
/// Decoded prompt logprobs for prompt token positions.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecodedPromptLogprobs {
/// Original vocabulary token ID for the first prompt token.
pub first_token_id: u32,
/// Best-effort decoded string for the first prompt token.
///
/// The first prompt token has no left context to score against, so it is
/// stored separately instead of appearing in `scored_positions`.
pub first_token: String,
/// Scored prompt positions after the first prompt token.
///
/// `scored_positions[i]` corresponds to the prompt token at position `i +
/// 1`.
pub scored_positions: Vec<DecodedPositionLogprobs>,
}
/// Decode generated-token logprobs from the raw `llm` token-ID shape into the
/// text-layer decoded-token representation.
///
/// Each returned position corresponds to one generated token position from the
/// same `llm` update.
pub(super) fn decode_logprobs<T: Tokenizer + ?Sized>(
tokenizer: &T,
logprobs: &Logprobs,
skip_special_tokens: bool,
) -> Result<DecodedLogprobs, Error> {
Ok(DecodedLogprobs {
positions: logprobs
.positions
.iter()
.map(|position| decode_position_logprobs(tokenizer, position, skip_special_tokens))
.try_collect()?,
})
}
/// Decode prompt logprobs from the raw `llm` token-ID shape into the text-layer
/// decoded-token representation.
///
/// The returned payload stores the first prompt token separately and decodes
/// the remaining scored prompt positions into `scored_positions`, matching
/// vLLM's prompt-logprobs semantics.
pub(super) fn decode_prompt_logprobs<T: Tokenizer + ?Sized>(
tokenizer: &T,
prompt_token_ids: &[u32],
logprobs: &Logprobs,
skip_special_tokens: bool,
) -> Result<DecodedPromptLogprobs, Error> {
let first_token_id = prompt_token_ids
.first()
.copied()
.expect("prompt logprobs require at least one prompt token");
let first_token = tokenizer.decode(&[first_token_id], skip_special_tokens)?;
let scored_positions = logprobs
.positions
.iter()
.map(|position| decode_position_logprobs(tokenizer, position, skip_special_tokens))
.try_collect()?;
Ok(DecodedPromptLogprobs {
first_token_id,
first_token,
scored_positions,
})
}
/// Decode one token position's raw candidate set into decoded token strings
/// plus logprob metadata.
///
/// This decodes every candidate token ID independently through the active text
/// backend.
fn decode_position_logprobs<T: Tokenizer + ?Sized>(
tokenizer: &T,
position: &PositionLogprobs,
skip_special_tokens: bool,
) -> Result<DecodedPositionLogprobs, Error> {
Ok(DecodedPositionLogprobs {
entries: position
.entries
.iter()
.map(|entry| {
tokenizer.decode(&[entry.token_id], skip_special_tokens).map(|token| {
DecodedTokenLogprob {
token_id: entry.token_id,
token,
logprob: entry.logprob,
rank: entry.rank,
}
})
})
.try_collect()?,
})
}
#[cfg(test)]
mod tests {
use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
use super::*;
#[derive(Debug)]
struct ByteTokenizer;
impl vllm_tokenizer::Tokenizer for ByteTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
unreachable!()
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::from_utf8_lossy(
&token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
)
.into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
#[test]
fn decode_logprobs_decodes_every_candidate_token() {
let tokenizer = ByteTokenizer;
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: b'a' as u32,
logprob: -0.1,
rank: 3,
},
TokenLogprob {
token_id: b'b' as u32,
logprob: -0.2,
rank: 1,
},
],
}],
};
assert_eq!(
decode_logprobs(&tokenizer, &logprobs, false).unwrap(),
DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
token_id: b'a' as u32,
token: "a".to_string(),
logprob: -0.1,
rank: 3,
},
DecodedTokenLogprob {
token_id: b'b' as u32,
token: "b".to_string(),
logprob: -0.2,
rank: 1,
},
],
}],
}
);
}
#[test]
fn decode_prompt_logprobs_separates_first_prompt_token() {
let tokenizer = ByteTokenizer;
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![TokenLogprob {
token_id: b'x' as u32,
logprob: -0.4,
rank: 1,
}],
}],
};
assert_eq!(
decode_prompt_logprobs(&tokenizer, &[b'p' as u32, b'x' as u32], &logprobs, false)
.unwrap(),
DecodedPromptLogprobs {
first_token_id: b'p' as u32,
first_token: "p".to_string(),
scored_positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: b'x' as u32,
token: "x".to_string(),
logprob: -0.4,
rank: 1,
}],
}],
}
);
}
}
+323
View File
@@ -0,0 +1,323 @@
//! Output processing helpers shared by text and chat layers.
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
pub use logprobs::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
};
mod decoded;
mod logprobs;
use std::sync::Arc;
use futures::{StreamExt as _, pin_mut};
use crate::{Error, FinishReason, Result, TextOutputStream};
/// Final decoded text plus terminal stream metadata.
#[derive(Debug, Clone, PartialEq)]
pub struct CollectedTextOutput {
pub text: String,
pub prompt_token_ids: Arc<[u32]>,
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
pub logprobs: Option<DecodedLogprobs>,
pub token_ids: Vec<u32>,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
#[easy_ext::ext(TextOutputStreamExt)]
impl<T: TextOutputStream> T {
/// Collect the stream to completion and return the final decoded text plus
/// terminal metadata.
pub fn collect_output(self) -> impl Future<Output = Result<CollectedTextOutput>> + Send {
async move {
let stream = self;
pin_mut!(stream);
let mut prompt_logprobs = None;
let mut prompt_token_ids: Arc<[u32]> = Arc::from([]);
let mut collected: Option<CollectedTextOutput> = None;
while let Some(event) = stream.next().await.transpose()? {
match event {
DecodedTextEvent::Start {
prompt_logprobs: start_prompt_logprobs,
prompt_token_ids: start_prompt_token_ids,
..
} => {
prompt_logprobs = start_prompt_logprobs;
prompt_token_ids = start_prompt_token_ids;
}
DecodedTextEvent::TextDelta {
delta,
token_ids: delta_token_ids,
logprobs: mut delta_logprobs,
finished,
} => {
if let Some(c) = collected.as_mut() {
c.text.push_str(&delta);
c.token_ids.extend(delta_token_ids);
if let Some(dlp) = delta_logprobs.as_mut() {
if let Some(lp) = c.logprobs.as_mut() {
lp.positions.extend_from_slice(&dlp.positions);
} else {
c.logprobs = delta_logprobs;
}
}
} else {
collected = Some(CollectedTextOutput {
text: delta,
prompt_token_ids: Arc::clone(&prompt_token_ids),
prompt_logprobs: prompt_logprobs.take(),
logprobs: delta_logprobs,
token_ids: delta_token_ids,
finish_reason: FinishReason::Error,
kv_transfer_params: None,
})
};
if let Some(finished) = finished {
let mut collected = collected.unwrap();
collected.finish_reason = finished.finish_reason;
collected.kv_transfer_params = finished.kv_transfer_params;
return Ok(collected);
}
}
}
}
// Note: this is actually unreachable, as the underlying stream always emit an
// error on unexpected close.
Err(Error::StreamClosedBeforeTerminalOutput {
request_id: "unknown".to_string(),
})
}
}
}
#[cfg(test)]
mod tests {
use futures::stream;
use vllm_llm::FinishReason;
use super::*;
#[tokio::test]
async fn collect_output_retains_prompt_and_sample_logprobs() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![10, 11].into(),
prompt_logprobs: Some(DecodedPromptLogprobs {
first_token_id: 0,
first_token: "o".to_string(),
scored_positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "p".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: "bc".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
prompt_token_count: 2,
output_token_count: 2,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}),
]);
let collected = stream.collect_output().await.unwrap();
assert_eq!(collected.text, "bc");
assert_eq!(
collected.prompt_logprobs,
Some(DecodedPromptLogprobs {
first_token_id: 0,
first_token: "o".to_string(),
scored_positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "p".to_string(),
logprob: -0.1,
rank: 1,
}],
}],
})
);
assert_eq!(
collected.logprobs,
Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
})
);
}
#[tokio::test]
async fn collect_output_accumulates_intermediate_deltas() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![10, 11].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "he".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
],
}),
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "llo".to_string(),
token_ids: vec![3, 4, 5],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
prompt_token_count: 2,
output_token_count: 5,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}),
]);
let collected = stream.collect_output().await.unwrap();
assert_eq!(collected.text, "hello");
assert_eq!(collected.prompt_logprobs, None);
assert_eq!(collected.token_ids, vec![1, 2, 3, 4, 5]);
assert_eq!(
collected.logprobs,
Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
})
);
}
}
+197
View File
@@ -0,0 +1,197 @@
use std::collections::HashMap;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::StructuredOutputsParams;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use crate::error::{Error, Result};
use crate::output::TextDecodeOptions;
/// One raw text-generation prompt.
///
/// This supports either ordinary text that still needs tokenization or
/// already-tokenized prompt IDs that should bypass tokenizer work entirely.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, EnumAsInner)]
#[serde(untagged)]
pub enum Prompt {
/// Untokenized prompt text that still needs tokenizer work before
/// generation.
Text(String),
/// Pre-tokenized prompt IDs that should be forwarded southbound without
/// re-encoding.
TokenIds(Vec<u32>),
}
impl Default for Prompt {
fn default() -> Self {
Self::Text(String::new()) // placeholder
}
}
/// User-facing sampling parameters accepted by `vllm-text`.
///
/// This intentionally keeps only the subset that the current Rust text layer
/// supports as northbound request semantics. Engine-core-specific normalized
/// fields are derived later during lowering.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct SamplingParams {
/// Controls randomness. Lower values are more deterministic; zero means
/// greedy sampling. `None` means no explicit user override.
pub temperature: Option<f32>,
/// Cumulative probability threshold for nucleus sampling.
pub top_p: Option<f32>,
/// Maximum number of top tokens to consider. `Some(0)` means all tokens.
pub top_k: Option<u32>,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate. `None` means no explicit user
/// override.
pub max_tokens: Option<u32>,
/// Minimum number of tokens to generate before EOS or stop-token handling.
pub min_tokens: Option<u32>,
/// Number of log probabilities to return per generated token.
///
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
pub logprobs: Option<i32>,
/// Number of log probabilities to return per prompt token.
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling. `None` means no
/// explicit user override.
pub min_p: Option<f32>,
/// Frequency penalty applied by the sampler. `None` means no explicit user
/// override.
pub frequency_penalty: Option<f32>,
/// Presence penalty applied by the sampler. `None` means no explicit user
/// override.
pub presence_penalty: Option<f32>,
/// Repetition penalty applied by the sampler. `None` means no explicit user
/// override.
pub repetition_penalty: Option<f32>,
/// Explicit stop token IDs provided by the caller. `None` means no explicit
/// user override.
pub stop_token_ids: Option<Vec<u32>>,
/// If true, do not stop on the model's primary EOS token.
pub ignore_eos: bool,
/// Modify the likelihood of specified tokens appearing in the completion.
/// Keys are token IDs.
pub logit_bias: Option<HashMap<u32, f32>>,
/// Restrict output to these token IDs only.
pub allowed_token_ids: Option<Vec<u32>>,
/// Words to avoid during generation (tokenized to IDs during lowering).
pub bad_words: Option<Vec<String>>,
/// Specific token IDs for which log probabilities should be returned at
/// each position.
///
/// When set, the engine returns logprobs for exactly these tokens in
/// addition to the sampled/scored token. Mutually exclusive with
/// `logprobs` in practice.
pub logprob_token_ids: Option<Vec<u32>>,
/// Parameters for configuring structured outputs (guided decoding).
pub structured_outputs: Option<StructuredOutputsParams>,
/// If true, bypass reads from the prefix cache for this request (the prompt
/// will not reuse cached KV blocks from earlier requests, though newly
/// computed blocks may still populate the cache). `None` defers to
/// engine-core defaults.
pub skip_reading_prefix_cache: Option<bool>,
/// Additional request parameters for custom extensions.
pub vllm_xargs: Option<HashMap<String, Value>>,
}
#[allow(clippy::derivable_impls)] // more explicit
impl Default for SamplingParams {
fn default() -> Self {
Self {
temperature: None,
top_p: None,
top_k: None,
seed: None,
max_tokens: None,
min_tokens: None,
logprobs: None,
prompt_logprobs: None,
min_p: None,
frequency_penalty: None,
presence_penalty: None,
repetition_penalty: None,
stop_token_ids: None,
ignore_eos: false,
logit_bias: None,
allowed_token_ids: None,
bad_words: None,
logprob_token_ids: None,
structured_outputs: None,
skip_reading_prefix_cache: None,
vllm_xargs: None,
}
}
}
/// One raw text-generation request ready to be tokenized or sent directly to
/// the engine.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TextRequest {
/// Stable caller-supplied request ID.
pub request_id: String,
/// Prompt text or prompt token IDs for this request.
pub prompt: Prompt,
/// Multimodal features prepared by a higher-level frontend. Raw text
/// requests keep this empty; multimodal chat uses it with pre-tokenized
/// prompt IDs.
pub mm_features: Option<MmFeatures>,
/// User-facing sampling parameters accepted by `vllm-text`.
pub sampling_params: SamplingParams,
/// Incremental detokenization options for the response path.
pub decode_options: TextDecodeOptions,
/// Whether to emit intermediate northbound deltas before the terminal
/// result.
///
/// If `false`, callers only observe the terminal accumulated output. If
/// `true`, callers may receive zero or more incremental decoded updates
/// before the final terminal event.
pub intermediate: bool,
/// Request scheduling priority (lower means earlier handling; default 0).
pub priority: i32,
/// Salt for prefix cache isolation in multi-user environments.
pub cache_salt: Option<String>,
/// Whether to add special tokens (e.g. BOS) during prompt tokenization.
pub add_special_tokens: bool,
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
}
impl TextRequest {
/// Return one minimal valid request fixture for tests.
pub fn for_test() -> Self {
Self {
request_id: "test-request".to_string(),
prompt: Prompt::Text("test".to_string()),
mm_features: None,
sampling_params: SamplingParams::default(),
decode_options: TextDecodeOptions::default(),
intermediate: true,
priority: 0,
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
}
}
/// Validate the minimum invariants before tokenization or request lowering.
pub fn validate(&self) -> Result<()> {
if matches!(&self.prompt, Prompt::TokenIds(ids) if ids.is_empty()) {
return Err(Error::EmptyPromptTokenIds {
request_id: self.request_id.clone(),
});
}
Ok(())
}
}