[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "vllm-text"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
easy-ext.workspace = true
|
||||
enum-as-inner.workspace = true
|
||||
futures.workspace = true
|
||||
hf-hub.workspace = true
|
||||
itertools.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tracing.workspace = true
|
||||
trait-set.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-llm.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
expect-test.workspace = true
|
||||
futures.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
vllm-llm = { workspace = true, features = ["test-util"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,373 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct HfTokenizerConfig {
|
||||
#[serde(flatten)]
|
||||
pub special_tokens: HfSpecialTokens,
|
||||
pub chat_template: Option<String>,
|
||||
/// The `tokenizer_class` field from HuggingFace tokenizer configs. Some
|
||||
/// tiktoken-based models (e.g. DeepSeek, Kimi K2) set this to a value
|
||||
/// containing "Tiktoken" which can be used as a hint for backend
|
||||
/// selection.
|
||||
pub tokenizer_class: Option<String>,
|
||||
}
|
||||
|
||||
/// Hugging Face named special tokens may be serialized as a string or an
|
||||
/// object carrying the token content.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum NamedSpecialToken {
|
||||
Text(String),
|
||||
WithContent { content: String },
|
||||
}
|
||||
|
||||
impl Serialize for NamedSpecialToken {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.serialize_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NamedSpecialToken> for String {
|
||||
fn from(value: NamedSpecialToken) -> Self {
|
||||
match value {
|
||||
NamedSpecialToken::Text(string) => string,
|
||||
NamedSpecialToken::WithContent { content } => content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NamedSpecialToken {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
Self::Text(value) => value,
|
||||
Self::WithContent { content } => content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal set of special-token entries needed by chat/EOS handling.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
|
||||
#[serde(default)]
|
||||
pub struct HfSpecialTokens {
|
||||
pub bos_token: Option<NamedSpecialToken>,
|
||||
pub eos_token: Option<NamedSpecialToken>,
|
||||
pub unk_token: Option<NamedSpecialToken>,
|
||||
pub pad_token: Option<NamedSpecialToken>,
|
||||
}
|
||||
|
||||
impl HfSpecialTokens {
|
||||
/// Returns true if we don't discover any special tokens in the config.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.bos_token.is_none()
|
||||
&& self.eos_token.is_none()
|
||||
&& self.unk_token.is_none()
|
||||
&& self.pad_token.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal subset of `config.json` (the model's main HF config).
|
||||
///
|
||||
/// This intentionally supports only the two layouts we currently care about in
|
||||
/// the Rust frontend:
|
||||
/// - pure text models that keep text metadata at the top level
|
||||
/// - composite models that expose a single nested `text_config`
|
||||
///
|
||||
/// We do not support additional entry points such as `decoder`, `generator`, or
|
||||
/// `text_encoder`.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ModelConfig {
|
||||
model_type: Option<String>,
|
||||
max_position_embeddings: Option<u32>,
|
||||
num_attention_heads: Option<u32>,
|
||||
num_experts: Option<OneOrManyExpertCount>,
|
||||
moe_num_experts: Option<OneOrManyExpertCount>,
|
||||
n_routed_experts: Option<OneOrManyExpertCount>,
|
||||
num_local_experts: Option<OneOrManyExpertCount>,
|
||||
block_configs: Vec<BlockConfig>,
|
||||
text_config: Option<Box<ModelConfig>>,
|
||||
}
|
||||
|
||||
/// Minimal subset of `generation_config.json`.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub(super) struct GenerationConfig {
|
||||
pub eos_token_id: Option<OneOrManyTokenIds>,
|
||||
pub temperature: Option<f32>,
|
||||
pub top_p: Option<f32>,
|
||||
pub top_k: Option<u32>,
|
||||
pub min_p: Option<f32>,
|
||||
pub repetition_penalty: Option<f32>,
|
||||
pub max_new_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// HF generation configs allow either one EOS id or a list of EOS ids.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(super) enum OneOrManyTokenIds {
|
||||
One(u32),
|
||||
Many(Vec<u32>),
|
||||
}
|
||||
|
||||
impl OneOrManyTokenIds {
|
||||
pub(super) fn into_set(self) -> BTreeSet<u32> {
|
||||
match self {
|
||||
Self::One(id) => BTreeSet::from([id]),
|
||||
Self::Many(ids) => ids.into_iter().collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hugging Face configs may expose the expert count either as one integer or
|
||||
/// as a list of repeated integers.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(super) enum OneOrManyExpertCount {
|
||||
One(u32),
|
||||
Many(Vec<u32>),
|
||||
}
|
||||
|
||||
impl OneOrManyExpertCount {
|
||||
fn first_value(&self) -> u32 {
|
||||
match self {
|
||||
Self::One(value) => *value,
|
||||
// Python currently takes the first value for list[int] expert
|
||||
// counts in remote-code configs.
|
||||
Self::Many(values) => values.first().copied().unwrap_or(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Heterogeneous block-level MoE metadata used as a fallback when no top-level
|
||||
/// expert-count field is available.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub(super) struct BlockConfig {
|
||||
pub block_type: String,
|
||||
pub n_routed_experts: u32,
|
||||
}
|
||||
|
||||
impl ModelConfig {
|
||||
/// Return the config that the Rust frontend treats as the text/LLM config.
|
||||
///
|
||||
/// This is deliberately narrower than Python/transformers: we only support
|
||||
/// either the top-level config itself or a single nested `text_config`.
|
||||
fn effective_text_config(&self) -> &Self {
|
||||
self.text_config.as_deref().unwrap_or(self)
|
||||
}
|
||||
|
||||
/// Return the effective Hugging Face `model_type` used by the Rust
|
||||
/// frontend.
|
||||
///
|
||||
/// This follows the same simplified text-config selection as the rest of
|
||||
/// this type: the top-level config wins, otherwise a single nested
|
||||
/// `text_config` may provide the value.
|
||||
pub fn model_type(&self) -> Option<&str> {
|
||||
self.model_type.as_deref().or_else(|| self.text_config.as_deref()?.model_type())
|
||||
}
|
||||
|
||||
/// Reject partially nested `text_config` payloads that are unlikely to be
|
||||
/// valid LLM configs for our current use.
|
||||
///
|
||||
/// This keeps the simplified Rust-side parsing honest: if a model declares
|
||||
/// `text_config`, it must at least look like a real text model config.
|
||||
fn validate_text_config_selection(&self) -> Result<()> {
|
||||
if let Some(text_config) = self.text_config.as_deref()
|
||||
&& text_config.num_attention_heads.is_none()
|
||||
{
|
||||
return Err(Error::Tokenizer(
|
||||
"the text config extracted from the model config does not have `num_attention_heads`"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Match Python's current expert-count priority on the selected text
|
||||
/// config.
|
||||
///
|
||||
/// The only intentional simplification here is how we pick the text config:
|
||||
/// Rust only looks at the top level or `text_config`, not the broader
|
||||
/// transformers composite-config surface.
|
||||
fn num_experts_from_block_configs(&self) -> u32 {
|
||||
self.effective_text_config()
|
||||
.block_configs
|
||||
.iter()
|
||||
.filter(|block| block.block_type == "moe")
|
||||
.map(|block| block.n_routed_experts)
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub(super) fn num_experts(&self) -> u32 {
|
||||
let config = self.effective_text_config();
|
||||
let direct = [
|
||||
config.num_experts.as_ref(),
|
||||
config.moe_num_experts.as_ref(),
|
||||
config.n_routed_experts.as_ref(),
|
||||
config.num_local_experts.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(OneOrManyExpertCount::first_value)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
|
||||
if direct > 0 {
|
||||
direct
|
||||
} else {
|
||||
self.num_experts_from_block_configs()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_moe(&self) -> bool {
|
||||
self.num_experts() > 0
|
||||
}
|
||||
|
||||
pub(super) fn max_position_embeddings(&self) -> Option<u32> {
|
||||
self.effective_text_config().max_position_embeddings
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the tokenizer-side EOS metadata if a config file is present.
|
||||
pub fn load_tokenizer_config(path: Option<&Path>) -> Result<HfTokenizerConfig> {
|
||||
read_json_file(path)
|
||||
}
|
||||
|
||||
/// Load the generation-side EOS metadata if a config file is present.
|
||||
pub(super) fn load_generation_config(path: Option<&Path>) -> Result<GenerationConfig> {
|
||||
read_json_file(path)
|
||||
}
|
||||
|
||||
/// Load the model-side config (`config.json`) if present.
|
||||
pub fn load_model_config(path: Option<&Path>) -> Result<ModelConfig> {
|
||||
let config: ModelConfig = read_json_file(path)?;
|
||||
config.validate_text_config_selection()?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn read_json_file<T>(path: Option<&Path>) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de> + Default,
|
||||
{
|
||||
let Some(path) = path else {
|
||||
return Ok(T::default());
|
||||
};
|
||||
let content = fs::read_to_string(path).map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to read {}: {}",
|
||||
path.display(),
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
serde_json::from_str(&content).map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to parse {}: {}",
|
||||
path.display(),
|
||||
error.as_report()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ModelConfig;
|
||||
|
||||
#[test]
|
||||
fn model_config_detects_moe_from_named_expert_fields() {
|
||||
let field_names = [
|
||||
"num_experts",
|
||||
"moe_num_experts",
|
||||
"n_routed_experts",
|
||||
"num_local_experts",
|
||||
];
|
||||
|
||||
for field_name in field_names {
|
||||
let config: ModelConfig =
|
||||
serde_json::from_str(&format!(r#"{{"{field_name}": 64}}"#)).unwrap();
|
||||
assert_eq!(config.num_experts(), 64, "field_name={field_name}");
|
||||
assert!(config.is_moe(), "field_name={field_name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_uses_first_value_for_list_expert_counts() {
|
||||
let config: ModelConfig = serde_json::from_str(r#"{"num_experts":[16,16]}"#).unwrap();
|
||||
|
||||
assert_eq!(config.num_experts(), 16);
|
||||
assert!(config.is_moe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_falls_back_to_block_configs_maximum() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"block_configs": [
|
||||
{"block_type":"attention","n_routed_experts":9},
|
||||
{"block_type":"moe","n_routed_experts":32},
|
||||
{"block_type":"moe","n_routed_experts":64}
|
||||
]
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.num_experts(), 64);
|
||||
assert!(config.is_moe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_prefers_nested_text_config_like_python_hf_text_config() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"model_type": "top_level",
|
||||
"num_experts": 64,
|
||||
"max_position_embeddings": 8192,
|
||||
"text_config": {
|
||||
"model_type": "nested",
|
||||
"num_attention_heads": 32,
|
||||
"num_local_experts": 8,
|
||||
"max_position_embeddings": 4096
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.num_experts(), 8);
|
||||
assert_eq!(config.model_type(), Some("top_level"));
|
||||
assert_eq!(config.max_position_embeddings(), Some(4096));
|
||||
assert!(config.is_moe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_defaults_to_non_moe_when_no_expert_metadata_exists() {
|
||||
let config: ModelConfig =
|
||||
serde_json::from_str(r#"{"max_position_embeddings":4096}"#).unwrap();
|
||||
|
||||
assert_eq!(config.num_experts(), 0);
|
||||
assert!(!config.is_moe());
|
||||
assert_eq!(config.max_position_embeddings(), Some(4096));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_rejects_nested_text_config_without_attention_heads() {
|
||||
let config: ModelConfig =
|
||||
serde_json::from_str(r#"{"text_config":{"max_position_embeddings":4096}}"#).unwrap();
|
||||
|
||||
let error = config.validate_text_config_selection().unwrap_err();
|
||||
assert!(error.to_string().contains("does not have `num_attention_heads`"),);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
mod config;
|
||||
mod model_files;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::info;
|
||||
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
|
||||
|
||||
use self::config::{GenerationConfig, load_generation_config};
|
||||
pub use self::config::{
|
||||
HfSpecialTokens, HfTokenizerConfig, ModelConfig, NamedSpecialToken, load_model_config,
|
||||
load_tokenizer_config,
|
||||
};
|
||||
pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
|
||||
use crate::backend::{SamplingHints, TextBackend};
|
||||
use crate::error::Result;
|
||||
|
||||
fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {
|
||||
match tokenizer {
|
||||
TokenizerSource::HuggingFace(path) => Ok(Arc::new(HuggingFaceTokenizer::new(path)?)),
|
||||
TokenizerSource::Tiktoken(path) => Ok(Arc::new(TiktokenTokenizer::new(path)?)),
|
||||
TokenizerSource::Tekken(path) => Ok(Arc::new(TekkenTokenizer::new(path)?)),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`TextBackend`] implementation built on Hugging Face model files.
|
||||
pub struct HfTextBackend {
|
||||
model_id: String,
|
||||
files: ResolvedModelFiles,
|
||||
tokenizer: DynTokenizer,
|
||||
/// Primary EOS handled by engine-core's dedicated EOS path.
|
||||
primary_eos_token_id: Option<u32>,
|
||||
/// Additional EOS ids that should flow through stop-token handling.
|
||||
extra_eos_token_ids: BTreeSet<u32>,
|
||||
/// Generation-config for sampling defaults that may be inherited when the
|
||||
/// user does not explicitly override them.
|
||||
generation_config: GenerationConfig,
|
||||
/// Model config (`config.json`).
|
||||
model_config: ModelConfig,
|
||||
}
|
||||
|
||||
impl HfTextBackend {
|
||||
/// Load the text backend with the given model id.
|
||||
pub async fn from_model(model_id: &str) -> Result<Self> {
|
||||
let files = ResolvedModelFiles::new(model_id).await?;
|
||||
Self::from_resolved_model_files(files, model_id.to_string())
|
||||
}
|
||||
|
||||
/// Load the text backend from resolved Hugging Face model files.
|
||||
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
|
||||
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = load_tokenizer(&files.tokenizer)?;
|
||||
let primary_eos_token_id = tokenizer_config
|
||||
.special_tokens
|
||||
.eos_token
|
||||
.as_ref()
|
||||
.and_then(|token| tokenizer.token_to_id(token.as_str()));
|
||||
|
||||
let model_config = load_model_config(files.config_path.as_deref())?;
|
||||
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
|
||||
let mut extra_eos_token_ids = generation_config
|
||||
.eos_token_id
|
||||
.clone()
|
||||
.map(|value| value.into_set())
|
||||
.unwrap_or_default();
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
extra_eos_token_ids.remove(&primary_eos_token_id);
|
||||
}
|
||||
|
||||
info!(
|
||||
model_id,
|
||||
"loaded text backend with Hugging Face model files"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
model_id,
|
||||
files,
|
||||
tokenizer,
|
||||
primary_eos_token_id,
|
||||
extra_eos_token_ids,
|
||||
generation_config,
|
||||
model_config,
|
||||
})
|
||||
}
|
||||
|
||||
/// Expose the resolved model files for use by the chat backend to load the
|
||||
/// chat template.
|
||||
pub fn resolved_model_files(&self) -> &ResolvedModelFiles {
|
||||
&self.files
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for HfTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
self.tokenizer.clone()
|
||||
}
|
||||
|
||||
fn is_moe(&self) -> bool {
|
||||
self.model_config.is_moe()
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
&self.model_id
|
||||
}
|
||||
|
||||
fn sampling_hints(&self) -> Result<SamplingHints> {
|
||||
Ok(SamplingHints {
|
||||
primary_eos_token_id: self.primary_eos_token_id,
|
||||
extra_eos_token_ids: self.extra_eos_token_ids.clone(),
|
||||
default_temperature: self.generation_config.temperature,
|
||||
default_top_p: self.generation_config.top_p,
|
||||
default_top_k: self.generation_config.top_k,
|
||||
default_min_p: self.generation_config.min_p,
|
||||
default_repetition_penalty: self.generation_config.repetition_penalty,
|
||||
default_max_tokens: self.generation_config.max_new_tokens,
|
||||
max_model_len: self.model_config.max_position_embeddings(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use hf_hub::Cache;
|
||||
use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use super::config::{HfTokenizerConfig, load_tokenizer_config};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
const HF_TOKEN_ENV: &str = "HF_TOKEN";
|
||||
|
||||
/// The tokenizer source selected for a model.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TokenizerSource {
|
||||
/// Path to `tokenizer.json` in HuggingFace format.
|
||||
HuggingFace(PathBuf),
|
||||
/// Path to `tiktoken.model` or `*.tiktoken` file for tiktoken-based models.
|
||||
Tiktoken(PathBuf),
|
||||
/// Path to `tekken.json` when present (Mistral native tokenizer format).
|
||||
///
|
||||
/// When set, the Tekken tokenizer should be preferred over the Hugging Face
|
||||
/// tokenizer because the HuggingFace `tokenizer.json` for Mistral
|
||||
/// models has a known regex bug that produces incorrect token IDs for
|
||||
/// some inputs.
|
||||
Tekken(PathBuf),
|
||||
}
|
||||
|
||||
impl TokenizerSource {
|
||||
pub fn path(&self) -> &Path {
|
||||
match self {
|
||||
Self::HuggingFace(path) | Self::Tiktoken(path) | Self::Tekken(path) => path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Concrete tokenizer/config file locations resolved for one HF model id.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResolvedModelFiles {
|
||||
/// The selected tokenizer source for this model.
|
||||
pub tokenizer: TokenizerSource,
|
||||
pub tokenizer_config_path: Option<PathBuf>,
|
||||
pub generation_config_path: Option<PathBuf>,
|
||||
pub preprocessor_config_path: Option<PathBuf>,
|
||||
pub chat_template_path: Option<PathBuf>,
|
||||
pub config_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl ResolvedModelFiles {
|
||||
/// Resolve tokenizer/config files from a local model directory first when
|
||||
/// `model_id` points to one, otherwise consult the local HF cache and
|
||||
/// finally the Hub.
|
||||
pub async fn new(model_id: &str) -> Result<Self> {
|
||||
if Path::new(model_id).is_dir() {
|
||||
return resolve_local_model_files(Path::new(model_id));
|
||||
}
|
||||
if let Some(files) = resolve_cached_model_files(model_id)? {
|
||||
return Ok(files);
|
||||
}
|
||||
resolve_remote_model_files(model_id).await
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
|
||||
let tokenizer_config_path = local_file_if_exists(model_dir, "tokenizer_config.json");
|
||||
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = resolve_local_tokenizer_source(model_dir, &tokenizer_config)?;
|
||||
|
||||
Ok(ResolvedModelFiles {
|
||||
tokenizer,
|
||||
tokenizer_config_path,
|
||||
generation_config_path: local_file_if_exists(model_dir, "generation_config.json"),
|
||||
preprocessor_config_path: local_file_if_exists(model_dir, "preprocessor_config.json"),
|
||||
chat_template_path: discover_chat_template_in_dir(model_dir),
|
||||
config_path: local_file_if_exists(model_dir, "config.json"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles> {
|
||||
let api = build_api().map_err(|error| Error::Tokenizer(error.to_report_string()))?;
|
||||
let repo = api.model(model_id.to_string());
|
||||
let info = repo.info().await.map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to fetch model '{model_id}': {}",
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
|
||||
let siblings = info
|
||||
.siblings
|
||||
.iter()
|
||||
.map(|sibling| sibling.rfilename.as_str())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
|
||||
let tokenizer_config_path =
|
||||
download_if_present(&repo, model_id, &siblings, "tokenizer_config.json").await?;
|
||||
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
|
||||
|
||||
let tokenizer = resolve_remote_tokenizer_source(
|
||||
&repo,
|
||||
model_id,
|
||||
&siblings,
|
||||
tokenizer_config.tokenizer_class.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let generation_config_path =
|
||||
download_if_present(&repo, model_id, &siblings, "generation_config.json").await?;
|
||||
let preprocessor_config_path =
|
||||
download_if_present(&repo, model_id, &siblings, "preprocessor_config.json").await?;
|
||||
let chat_template_name = siblings
|
||||
.contains("chat_template.json")
|
||||
.then_some("chat_template.json")
|
||||
.or_else(|| siblings.contains("chat_template.jinja").then_some("chat_template.jinja"))
|
||||
.or_else(|| siblings.iter().copied().find(|name| name.ends_with(".jinja")));
|
||||
let chat_template_path = match chat_template_name {
|
||||
Some(name) => Some(download_known_file(&repo, model_id, name).await?),
|
||||
None => None,
|
||||
};
|
||||
let config_path = download_if_present(&repo, model_id, &siblings, "config.json").await?;
|
||||
|
||||
Ok(ResolvedModelFiles {
|
||||
tokenizer,
|
||||
tokenizer_config_path,
|
||||
generation_config_path,
|
||||
preprocessor_config_path,
|
||||
chat_template_path,
|
||||
config_path,
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFiles>> {
|
||||
let cache_repo = Cache::from_env().model(model_id.to_string());
|
||||
|
||||
let tokenizer_config_path = cache_repo.get("tokenizer_config.json");
|
||||
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = match resolve_cached_tokenizer_source(&cache_repo, &tokenizer_config)? {
|
||||
Some(tokenizer) => tokenizer,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let model_dir = tokenizer.path().parent().ok_or_else(|| {
|
||||
Error::Tokenizer("resolved tokenizer file has no parent directory".to_string())
|
||||
})?;
|
||||
let generation_config_path = cache_repo.get("generation_config.json");
|
||||
let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
|
||||
let chat_template_path = discover_chat_template_in_dir(model_dir);
|
||||
let config_path = cache_repo.get("config.json");
|
||||
|
||||
Ok(Some(ResolvedModelFiles {
|
||||
tokenizer,
|
||||
tokenizer_config_path,
|
||||
generation_config_path,
|
||||
preprocessor_config_path,
|
||||
chat_template_path,
|
||||
config_path,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn resolve_remote_tokenizer_source(
|
||||
repo: &ApiRepo,
|
||||
model_id: &str,
|
||||
siblings: &std::collections::BTreeSet<&str>,
|
||||
tokenizer_class: Option<&str>,
|
||||
) -> Result<TokenizerSource> {
|
||||
if let Some(tekken_path) = download_if_present(repo, model_id, siblings, "tekken.json").await? {
|
||||
return Ok(TokenizerSource::Tekken(tekken_path));
|
||||
}
|
||||
|
||||
let tokenizer_path = if siblings.contains("tokenizer.json") {
|
||||
download_known_file(repo, model_id, "tokenizer.json").await?
|
||||
} else if let Some(tiktoken_name) = find_tiktoken_sibling(siblings) {
|
||||
download_known_file(repo, model_id, tiktoken_name).await?
|
||||
} else {
|
||||
return Err(Error::Tokenizer(format!(
|
||||
"model '{model_id}' does not expose a supported tokenizer file \
|
||||
(tokenizer.json, tiktoken.model, or *.tiktoken) on Hugging Face"
|
||||
)));
|
||||
};
|
||||
|
||||
Ok(resolve_tokenizer_source(
|
||||
tokenizer_path,
|
||||
tokenizer_class,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_cached_tokenizer_source(
|
||||
cache_repo: &hf_hub::CacheRepo,
|
||||
tokenizer_config: &HfTokenizerConfig,
|
||||
) -> Result<Option<TokenizerSource>> {
|
||||
let tekken_path = cache_repo.get("tekken.json");
|
||||
|
||||
if let Some(tekken_path) = tekken_path {
|
||||
return Ok(Some(TokenizerSource::Tekken(tekken_path)));
|
||||
}
|
||||
|
||||
let Some(tokenizer_path) = cache_repo.get("tokenizer.json").or_else(|| {
|
||||
// tiktoken.model is the most common name, try it first.
|
||||
cache_repo.get("tiktoken.model").or_else(|| {
|
||||
// Scan for any *.tiktoken file in the cache snapshot directory.
|
||||
let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
|
||||
discover_tiktoken_in_dir(&snapshot_dir)
|
||||
})
|
||||
}) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(Some(resolve_tokenizer_source(
|
||||
tokenizer_path,
|
||||
tokenizer_config.tokenizer_class.as_deref(),
|
||||
None,
|
||||
)))
|
||||
}
|
||||
|
||||
fn resolve_local_tokenizer_source(
|
||||
model_dir: &Path,
|
||||
tokenizer_config: &HfTokenizerConfig,
|
||||
) -> Result<TokenizerSource> {
|
||||
let tekken_path = local_file_if_exists(model_dir, "tekken.json");
|
||||
if let Some(tekken_path) = tekken_path {
|
||||
return Ok(TokenizerSource::Tekken(tekken_path));
|
||||
}
|
||||
|
||||
let tokenizer_path = local_file_if_exists(model_dir, "tokenizer.json")
|
||||
.or_else(|| local_file_if_exists(model_dir, "tiktoken.model"))
|
||||
.or_else(|| discover_tiktoken_in_dir(model_dir))
|
||||
.ok_or_else(|| {
|
||||
Error::Tokenizer(format!(
|
||||
"local model directory '{}' does not contain a supported tokenizer file \
|
||||
(tokenizer.json, tiktoken.model, or *.tiktoken)",
|
||||
model_dir.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(resolve_tokenizer_source(
|
||||
tokenizer_path,
|
||||
tokenizer_config.tokenizer_class.as_deref(),
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
/// Choose the tokenizer.
|
||||
///
|
||||
/// Selection order:
|
||||
/// 1. `tekken.json` — Mistral native tokenizer (preferred over HF `tokenizer.json` because the HF
|
||||
/// version has a known regex bug for Mistral models).
|
||||
/// 2. File extension — `.tiktoken` / `tiktoken.model` files use tiktoken from BPE data.
|
||||
/// 3. `tokenizer_class` in `tokenizer_config.json` — classes containing "Tiktoken" (case-
|
||||
/// insensitive) trigger tiktoken loading from a sibling BPE file.
|
||||
/// 4. Default — `tokenizer.json` in HuggingFace format.
|
||||
fn resolve_tokenizer_source(
|
||||
tokenizer_path: PathBuf,
|
||||
tokenizer_class: Option<&str>,
|
||||
tekken_path: Option<PathBuf>,
|
||||
) -> TokenizerSource {
|
||||
if let Some(tekken_path) = tekken_path {
|
||||
return TokenizerSource::Tekken(tekken_path);
|
||||
}
|
||||
|
||||
if is_tiktoken_file(&tokenizer_path) {
|
||||
return TokenizerSource::Tiktoken(tokenizer_path);
|
||||
}
|
||||
|
||||
if tokenizer_class.is_some_and(|cls| cls.to_ascii_lowercase().contains("tiktoken"))
|
||||
&& let Some(dir) = tokenizer_path.parent()
|
||||
&& let Some(tiktoken_path) = discover_tiktoken_in_dir(dir)
|
||||
{
|
||||
return TokenizerSource::Tiktoken(tiktoken_path);
|
||||
}
|
||||
|
||||
TokenizerSource::HuggingFace(tokenizer_path)
|
||||
}
|
||||
|
||||
/// Download `filename` only if it exists in `siblings`.
|
||||
async fn download_if_present(
|
||||
repo: &ApiRepo,
|
||||
model_id: &str,
|
||||
siblings: &std::collections::BTreeSet<&str>,
|
||||
filename: &str,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
match siblings.contains(filename) {
|
||||
true => download_known_file(repo, model_id, filename).await.map(Some),
|
||||
false => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn download_known_file(repo: &ApiRepo, model_id: &str, filename: &str) -> Result<PathBuf> {
|
||||
repo.get(filename).await.map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to download '{filename}' for model '{model_id}': {}",
|
||||
error.as_report()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn build_api() -> anyhow::Result<Api> {
|
||||
let mut builder = ApiBuilder::from_env().with_progress(true);
|
||||
if let Ok(token) = std::env::var(HF_TOKEN_ENV)
|
||||
&& !token.is_empty()
|
||||
{
|
||||
builder = builder.with_token(Some(token));
|
||||
}
|
||||
Ok(builder.build()?)
|
||||
}
|
||||
|
||||
fn local_file_if_exists(dir: &Path, filename: &str) -> Option<PathBuf> {
|
||||
let path = dir.join(filename);
|
||||
path.is_file().then_some(path)
|
||||
}
|
||||
|
||||
/// Find a tiktoken file name among repo siblings, preferring `tiktoken.model`.
|
||||
fn find_tiktoken_sibling<'a>(siblings: &std::collections::BTreeSet<&'a str>) -> Option<&'a str> {
|
||||
if siblings.contains("tiktoken.model") {
|
||||
return Some("tiktoken.model");
|
||||
}
|
||||
siblings.iter().copied().find(|name| name.ends_with(".tiktoken"))
|
||||
}
|
||||
|
||||
/// Discover a tiktoken model file in a local directory.
|
||||
pub(super) fn discover_tiktoken_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
|
||||
let tiktoken_model = dir.join("tiktoken.model");
|
||||
if tiktoken_model.exists() {
|
||||
return Some(tiktoken_model);
|
||||
}
|
||||
std::fs::read_dir(dir).ok()?.flatten().find_map(|entry| {
|
||||
let path = entry.path();
|
||||
if path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.is_some_and(|n| n.ends_with(".tiktoken"))
|
||||
{
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if `path` points to a tiktoken-format file (by name).
|
||||
pub(super) fn is_tiktoken_file(path: &std::path::Path) -> bool {
|
||||
path.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.is_some_and(|name| name == "tiktoken.model" || name.ends_with(".tiktoken"))
|
||||
}
|
||||
|
||||
/// Chat templates are sometimes stored as dedicated .jinja files rather than as
|
||||
/// a fixed-name config entry, so we scan the cached model dir.
|
||||
fn discover_chat_template_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
|
||||
let json_template_path = dir.join("chat_template.json");
|
||||
if json_template_path.exists() {
|
||||
return Some(json_template_path);
|
||||
}
|
||||
|
||||
let jinja_path = dir.join("chat_template.jinja");
|
||||
if jinja_path.exists() {
|
||||
return Some(jinja_path);
|
||||
}
|
||||
|
||||
std::fs::read_dir(dir).ok()?.flatten().map(|entry| entry.path()).find(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.ends_with(".jinja"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
|
||||
|
||||
use super::{ResolvedModelFiles, TokenizerSource};
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolved_model_files_prefers_absolute_local_model_dir() {
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer");
|
||||
fs::write(
|
||||
dir.path().join("tokenizer_config.json"),
|
||||
r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#,
|
||||
)
|
||||
.expect("write tokenizer config");
|
||||
fs::write(dir.path().join("config.json"), "{}").expect("write config");
|
||||
|
||||
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"))
|
||||
.await
|
||||
.expect("resolve local model files");
|
||||
|
||||
match files.tokenizer {
|
||||
TokenizerSource::HuggingFace(path) => {
|
||||
assert_eq!(path, dir.path().join("tokenizer.json"));
|
||||
}
|
||||
other => panic!("expected HuggingFace tokenizer, got {other:?}"),
|
||||
}
|
||||
assert_eq!(files.config_path, Some(dir.path().join("config.json")));
|
||||
assert_eq!(
|
||||
files.tokenizer_config_path,
|
||||
Some(dir.path().join("tokenizer_config.json"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires network access to Hugging Face and downloads the real Kimi K2.5 tokenizer"]
|
||||
async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
|
||||
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
|
||||
.await
|
||||
.expect("resolve real Kimi K2.5 model files");
|
||||
|
||||
let tokenizer_path = match &files.tokenizer {
|
||||
TokenizerSource::Tiktoken(path) => path.clone(),
|
||||
other => panic!("expected tiktoken tokenizer source, got {other:?}"),
|
||||
};
|
||||
|
||||
for backend in [
|
||||
TiktokenTokenizer::new_riptoken(&tokenizer_path).expect("load riptoken backend"),
|
||||
TiktokenTokenizer::new_tiktoken_rs(&tokenizer_path).expect("load tiktoken-rs backend"),
|
||||
] {
|
||||
let think_id = backend.token_to_id("<think>").expect("resolve <think>");
|
||||
let end_think_id = backend.token_to_id("</think>").expect("resolve </think>");
|
||||
let tool_section_id = backend
|
||||
.token_to_id("<|tool_calls_section_begin|>")
|
||||
.expect("resolve tool call section marker");
|
||||
let contraction_heavy_text =
|
||||
"I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.";
|
||||
let contraction_heavy_ids = backend.encode(contraction_heavy_text, false).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(think_id, end_think_id, tool_section_id),
|
||||
(163606, 163607, 163595)
|
||||
);
|
||||
assert_eq!(backend.decode(&[think_id], true).unwrap(), "<think>");
|
||||
assert_eq!(backend.decode(&[end_think_id], true).unwrap(), "</think>");
|
||||
assert_eq!(
|
||||
backend.decode(&[tool_section_id], true).unwrap(),
|
||||
"<|tool_calls_section_begin|>"
|
||||
);
|
||||
|
||||
// This demonstrates that we're using Kimi's custom BPE pattern.
|
||||
// With CL100K this will be 23 tokens instead.
|
||||
assert_eq!(
|
||||
contraction_heavy_ids,
|
||||
vec![
|
||||
17172, 3287, 4643, 8201, 11, 996, 374, 8971, 3637, 20020, 8173, 473, 4643,
|
||||
1573, 56229, 13922, 13,
|
||||
]
|
||||
);
|
||||
assert_eq!(contraction_heavy_ids.len(), 17);
|
||||
assert_eq!(
|
||||
backend.decode(&contraction_heavy_ids, false).unwrap(),
|
||||
contraction_heavy_text
|
||||
);
|
||||
|
||||
// Special-looking text that is not actually registered should fail gracefully.
|
||||
assert_eq!(backend.token_to_id("◁think▷"), None);
|
||||
assert_eq!(backend.token_to_id("<|definitely_not_registered|>"), None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
pub mod hf;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Tokenizer/model-derived hints used to enrich text-generation requests before
|
||||
/// they are lowered into engine-core.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct SamplingHints {
|
||||
pub primary_eos_token_id: Option<u32>,
|
||||
pub extra_eos_token_ids: std::collections::BTreeSet<u32>,
|
||||
pub default_temperature: Option<f32>,
|
||||
pub default_top_p: Option<f32>,
|
||||
pub default_top_k: Option<u32>,
|
||||
pub default_min_p: Option<f32>,
|
||||
pub default_repetition_penalty: Option<f32>,
|
||||
pub default_max_tokens: Option<u32>,
|
||||
/// Model context window size (`max_position_embeddings` from
|
||||
/// `config.json`).
|
||||
pub max_model_len: Option<u32>,
|
||||
}
|
||||
|
||||
/// Minimal text-processing backend needed by `vllm-text`.
|
||||
pub trait TextBackend: Send + Sync {
|
||||
/// Return the tokenizer used by this backend.
|
||||
fn tokenizer(&self) -> DynTokenizer;
|
||||
|
||||
/// Return whether the loaded model is a mixture-of-experts model.
|
||||
fn is_moe(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Return the backend model ID.
|
||||
fn model_id(&self) -> &str;
|
||||
|
||||
/// Return tokenizer/model-derived hints used to enrich southbound sampling
|
||||
/// parameters.
|
||||
fn sampling_hints(&self) -> Result<SamplingHints> {
|
||||
Ok(SamplingHints::default())
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared trait-object form of [`TextBackend`].
|
||||
pub type DynTextBackend = Arc<dyn TextBackend>;
|
||||
@@ -0,0 +1,30 @@
|
||||
use thiserror::Error;
|
||||
use vllm_engine_core_client::Error as EngineCoreError;
|
||||
use vllm_llm::Error as LlmError;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("tokenizer error: {0}")]
|
||||
Tokenizer(String),
|
||||
#[error("text request `{request_id}` must contain at least one prompt token ID")]
|
||||
EmptyPromptTokenIds { request_id: String },
|
||||
#[error(
|
||||
"this model's maximum context length is {max_model_len} tokens, \
|
||||
but the prompt contains {prompt_len} input tokens"
|
||||
)]
|
||||
PromptTooLong { max_model_len: u32, prompt_len: u32 },
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error(transparent)]
|
||||
Llm(#[from] LlmError),
|
||||
#[error(transparent)]
|
||||
EngineCore(#[from] EngineCoreError),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl From<vllm_tokenizer::TokenizerError> for Error {
|
||||
fn from(error: vllm_tokenizer::TokenizerError) -> Self {
|
||||
Self::Tokenizer(error.0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
//! Shared text-generation support used by chat and future raw completions.
|
||||
//!
|
||||
//! This crate intentionally stays below chat semantics:
|
||||
//! prompt text handling, tokenizer/model loading, incremental detokenization,
|
||||
//! and the thin generate-facing backend interface live here.
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
pub use backend::{DynTextBackend, SamplingHints, TextBackend};
|
||||
pub use error::{Error, Result};
|
||||
use futures::Stream;
|
||||
pub use lower::{
|
||||
PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,
|
||||
};
|
||||
pub use output::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
|
||||
};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest};
|
||||
use trait_set::trait_set;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
pub use vllm_llm::FinishReason;
|
||||
use vllm_llm::{GenerateOutputStream, Llm};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
pub mod backend;
|
||||
mod error;
|
||||
mod lower;
|
||||
pub mod output;
|
||||
mod request;
|
||||
pub use vllm_tokenizer as tokenizer;
|
||||
|
||||
trait_set! {
|
||||
/// Shared streamed text output type used by raw completions and other text-only northbound paths.
|
||||
pub trait TextOutputStream = Stream<Item = Result<DecodedTextEvent>> + Send + 'static;
|
||||
}
|
||||
|
||||
/// Raw text facade above [`Llm`].
|
||||
///
|
||||
/// This layer stays below chat semantics: prompt text or prompt token IDs flow
|
||||
/// in, decoded text deltas and terminal metadata flow out.
|
||||
pub struct TextLlm {
|
||||
/// Generate-only client owned by this text facade.
|
||||
llm: Llm,
|
||||
/// Tokenizer/model metadata backend responsible for prompt encode/decode
|
||||
/// and sampling hints.
|
||||
backend: DynTextBackend,
|
||||
/// Context window size derived by the backend or from engine startup
|
||||
/// handshake, with optional override from config.
|
||||
max_model_len: Option<u32>,
|
||||
}
|
||||
|
||||
impl TextLlm {
|
||||
/// Create a new text-generation facade from a shared LLM client plus a text
|
||||
/// backend.
|
||||
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
|
||||
// Prefer the engine-reported max_model_len because it reflects the
|
||||
// post-profiling, auto-fitted KV cache limit rather than static
|
||||
// frontend metadata.
|
||||
let max_model_len = llm.engine_core_client().max_model_len();
|
||||
|
||||
Self {
|
||||
llm,
|
||||
backend,
|
||||
max_model_len,
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the maximum model context length explicitly.
|
||||
///
|
||||
/// This takes priority over both the engine-reported default and any
|
||||
/// tokenizer/model metadata exposed by the backend.
|
||||
pub fn with_max_model_len(mut self, max_model_len: u32) -> Self {
|
||||
self.max_model_len = Some(max_model_len);
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the backend model ID.
|
||||
pub fn model_id(&self) -> &str {
|
||||
self.backend.model_id()
|
||||
}
|
||||
|
||||
/// Expose the underlying engine-core client for low-level utility/admin
|
||||
/// calls.
|
||||
pub fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
self.llm.engine_core_client()
|
||||
}
|
||||
|
||||
/// Return the tokenizer used by this text backend.
|
||||
pub fn tokenizer(&self) -> DynTokenizer {
|
||||
self.backend.tokenizer()
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and return the raw
|
||||
/// token stream.
|
||||
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {
|
||||
let (_, raw_stream) = self.generate_inner(request).await?;
|
||||
Ok(raw_stream)
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and stream
|
||||
/// incrementally decoded text.
|
||||
pub async fn generate(&self, request: TextRequest) -> Result<impl TextOutputStream> {
|
||||
let (text_request, raw_stream) = self.generate_inner(request).await?;
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let decoded_stream = output::decoded_text_event_stream(
|
||||
text_request.request_id,
|
||||
tokenizer,
|
||||
raw_stream,
|
||||
text_request.decode_options,
|
||||
text_request.intermediate,
|
||||
);
|
||||
|
||||
Ok(decoded_stream)
|
||||
}
|
||||
|
||||
async fn generate_inner(
|
||||
&self,
|
||||
mut request: TextRequest,
|
||||
) -> Result<(TextRequest, GenerateOutputStream)> {
|
||||
request.validate()?;
|
||||
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let prompt_token_ids = match take(&mut request.prompt) {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
|
||||
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
|
||||
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
|
||||
let mut sampling_hints = self.backend.sampling_hints()?;
|
||||
if let Some(max_model_len) = self.max_model_len {
|
||||
sampling_hints.max_model_len = Some(max_model_len);
|
||||
}
|
||||
let PreparedTextRequest {
|
||||
text_request,
|
||||
generate_request,
|
||||
} = lower_text_request(request, prompt_token_ids, sampling_hints, &*tokenizer)?;
|
||||
|
||||
let raw_stream = self.llm.generate(generate_request).await?;
|
||||
Ok((text_request, raw_stream))
|
||||
}
|
||||
|
||||
/// Shut down the underlying LLM client and its background tasks.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
self.llm.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,739 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
use vllm_llm::GenerateRequest;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use crate::backend::SamplingHints;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{SamplingParams, TextRequest};
|
||||
|
||||
/// One text request after it has been lowered into the raw generate boundary.
|
||||
#[derive(Debug)]
|
||||
pub struct PreparedTextRequest {
|
||||
/// The original high-level request, preserved for response-side metadata
|
||||
/// and decoding options.
|
||||
pub text_request: TextRequest,
|
||||
/// The southbound request ready to be sent to `vllm-llm`.
|
||||
pub generate_request: GenerateRequest,
|
||||
}
|
||||
|
||||
/// Convert a high-level [`TextRequest`] into one lower-level
|
||||
/// [`GenerateRequest`] ready for the `llm` crate.
|
||||
pub fn lower_text_request(
|
||||
request: TextRequest,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
sampling_hints: SamplingHints,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<PreparedTextRequest> {
|
||||
let prompt_len = prompt_token_ids.len() as u32;
|
||||
let generate_request = GenerateRequest {
|
||||
request_id: request.request_id.clone(),
|
||||
prompt_token_ids,
|
||||
mm_features: request.mm_features.clone(),
|
||||
sampling_params: lower_sampling_params(
|
||||
request.sampling_params.clone(),
|
||||
sampling_hints,
|
||||
prompt_len,
|
||||
tokenizer,
|
||||
)?,
|
||||
cache_salt: request.cache_salt.clone(),
|
||||
priority: request.priority,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
// Fields below are currently placeholders.
|
||||
arrival_time: None,
|
||||
trace_headers: None,
|
||||
reasoning_ended: None,
|
||||
lora_request: None,
|
||||
};
|
||||
|
||||
Ok(PreparedTextRequest {
|
||||
text_request: request,
|
||||
generate_request,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert [`SamplingParams`] into [`EngineCoreSamplingParams`], enriching
|
||||
/// omitted user values with tokenizer/model-derived hints when available.
|
||||
pub fn lower_sampling_params(
|
||||
sampling_params: SamplingParams,
|
||||
SamplingHints {
|
||||
primary_eos_token_id,
|
||||
extra_eos_token_ids,
|
||||
default_temperature,
|
||||
default_top_p,
|
||||
default_top_k,
|
||||
default_min_p,
|
||||
default_repetition_penalty,
|
||||
default_max_tokens,
|
||||
max_model_len,
|
||||
}: SamplingHints,
|
||||
prompt_len: u32,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<EngineCoreSamplingParams> {
|
||||
let SamplingParams {
|
||||
temperature,
|
||||
top_p,
|
||||
top_k,
|
||||
seed,
|
||||
max_tokens,
|
||||
min_tokens,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
repetition_penalty,
|
||||
stop_token_ids,
|
||||
ignore_eos,
|
||||
logit_bias,
|
||||
allowed_token_ids,
|
||||
bad_words,
|
||||
logprob_token_ids,
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache,
|
||||
vllm_xargs,
|
||||
} = sampling_params;
|
||||
|
||||
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
|
||||
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
|
||||
// If neither the caller nor the model provides a value, fall back to 1.0 — the
|
||||
// default used by the Python vLLM OpenAI-compatible API (via
|
||||
// `_DEFAULT_SAMPLING_PARAMS`).
|
||||
let temperature = temperature.or(default_temperature).unwrap_or(1.0);
|
||||
let top_p = top_p.or(default_top_p).unwrap_or(1.0);
|
||||
let top_k = top_k.or(default_top_k).unwrap_or(0);
|
||||
let min_p = min_p.or(default_min_p).unwrap_or(0.0);
|
||||
let repetition_penalty = repetition_penalty.or(default_repetition_penalty).unwrap_or(1.0);
|
||||
let max_tokens = resolve_max_tokens(max_tokens, default_max_tokens, max_model_len, prompt_len)?;
|
||||
let min_tokens = min_tokens.unwrap_or(0);
|
||||
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
|
||||
let presence_penalty = presence_penalty.unwrap_or(0.0);
|
||||
|
||||
let mut stop_token_ids = stop_token_ids.unwrap_or_default();
|
||||
let mut all_stop_token_ids = BTreeSet::from_iter(stop_token_ids.iter().copied());
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
all_stop_token_ids.insert(primary_eos_token_id);
|
||||
}
|
||||
all_stop_token_ids.extend(extra_eos_token_ids.iter().copied());
|
||||
|
||||
if !ignore_eos {
|
||||
merge_unique_token_ids(&mut stop_token_ids, extra_eos_token_ids.iter().copied());
|
||||
}
|
||||
|
||||
Ok(EngineCoreSamplingParams {
|
||||
temperature,
|
||||
top_p,
|
||||
top_k,
|
||||
seed,
|
||||
max_tokens,
|
||||
min_tokens,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
repetition_penalty,
|
||||
stop_token_ids,
|
||||
eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(),
|
||||
all_stop_token_ids,
|
||||
logit_bias,
|
||||
allowed_token_ids,
|
||||
bad_words_token_ids: tokenize_bad_words(bad_words.as_deref(), tokenizer)?,
|
||||
structured_outputs,
|
||||
logprob_token_ids,
|
||||
skip_reading_prefix_cache,
|
||||
extra_args: vllm_xargs,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
|
||||
/// logic in `SamplingParams.update_from_tokenizer()`.
|
||||
///
|
||||
/// Each word is encoded both with and without a leading space so that the ban
|
||||
/// applies regardless of whether the word appears at the beginning or in the
|
||||
/// middle of generated text (this accounts for tokenizers that use an
|
||||
/// `add_prefix_space` convention).
|
||||
///
|
||||
/// Reference: <https://github.com/vllm-project/vllm/blob/f22d6e026/vllm/sampling_params.py#L555-L594>
|
||||
fn tokenize_bad_words(
|
||||
bad_words: Option<&[String]>,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<Option<Vec<Vec<u32>>>> {
|
||||
let bad_words = bad_words.filter(|w| !w.is_empty());
|
||||
let mut all_token_ids = Vec::new();
|
||||
|
||||
for bad_word in bad_words.into_iter().flatten() {
|
||||
// Without a leading space we always keep the encoding.
|
||||
// With a leading space we only keep it when the prefix-space variant produces a
|
||||
// distinct first token but the same sequence length — this mirrors the Python
|
||||
// dedup condition that avoids redundant entries.
|
||||
let without_space = tokenizer.encode(bad_word, false)?;
|
||||
let with_space = tokenizer.encode(&format!(" {}", bad_word.trim_start()), false)?;
|
||||
|
||||
if !without_space.is_empty() {
|
||||
all_token_ids.push(without_space);
|
||||
}
|
||||
if !with_space.is_empty()
|
||||
&& all_token_ids.last().is_some_and(|prev: &Vec<u32>| {
|
||||
with_space[0] != prev[0] && with_space.len() == prev.len()
|
||||
})
|
||||
{
|
||||
all_token_ids.push(with_space);
|
||||
}
|
||||
}
|
||||
|
||||
Ok((!all_token_ids.is_empty()).then_some(all_token_ids))
|
||||
}
|
||||
|
||||
/// Resolve the effective `max_tokens` for generation, mirroring vLLM Python's
|
||||
/// `get_max_tokens()` in `vllm/entrypoints/utils.py`.
|
||||
///
|
||||
/// Takes the minimum of all available limits (user-specified, generation-config
|
||||
/// default, and `max_model_len - prompt_len`). When nothing is known, falls
|
||||
/// back to `u32::MAX` so the engine-core can apply its own context-window
|
||||
/// limit.
|
||||
pub fn resolve_max_tokens(
|
||||
user_max_tokens: Option<u32>,
|
||||
default_max_tokens: Option<u32>,
|
||||
max_model_len: Option<u32>,
|
||||
prompt_len: u32,
|
||||
) -> Result<u32> {
|
||||
let model_max_tokens = match max_model_len {
|
||||
Some(max_model_len) if prompt_len >= max_model_len => {
|
||||
return Err(Error::PromptTooLong {
|
||||
max_model_len,
|
||||
prompt_len,
|
||||
});
|
||||
}
|
||||
Some(max_model_len) => Some(max_model_len - prompt_len),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let fallback_max_tokens = user_max_tokens.or(default_max_tokens);
|
||||
Ok([fallback_max_tokens, model_max_tokens]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min()
|
||||
.unwrap_or(u32::MAX /* TODO: a reasonable fallback? */))
|
||||
}
|
||||
|
||||
fn merge_unique_token_ids(
|
||||
stop_token_ids: &mut Vec<u32>,
|
||||
extra_token_ids: impl Iterator<Item = u32>,
|
||||
) {
|
||||
// Keep user-provided ordering stable while still folding in backend-derived EOS
|
||||
// aliases.
|
||||
for token_id in extra_token_ids {
|
||||
if !stop_token_ids.contains(&token_id) {
|
||||
stop_token_ids.push(token_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
use crate::backend::hf::HfTextBackend;
|
||||
use crate::backend::{SamplingHints, TextBackend as _};
|
||||
use crate::request::{Prompt, TextRequest};
|
||||
|
||||
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
|
||||
/// don't exercise bad-words tokenization.
|
||||
struct StubTokenizer;
|
||||
|
||||
impl Tokenizer for StubTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn stub_tokenizer() -> StubTokenizer {
|
||||
StubTokenizer
|
||||
}
|
||||
|
||||
fn sample_request() -> TextRequest {
|
||||
TextRequest {
|
||||
prompt: Prompt::TokenIds(vec![1, 2, 3]),
|
||||
request_id: "text-1".to_string(),
|
||||
..TextRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_sampling_hints() -> SamplingHints {
|
||||
SamplingHints {
|
||||
primary_eos_token_id: Some(99),
|
||||
extra_eos_token_ids: BTreeSet::from([77]),
|
||||
default_temperature: None,
|
||||
default_top_p: None,
|
||||
default_top_k: None,
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_applies_python_style_eos_hints() {
|
||||
let prepared = lower_text_request(
|
||||
sample_request(),
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let params = prepared.generate_request.sampling_params;
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: [
|
||||
77,
|
||||
],
|
||||
eos_token_id: Some(
|
||||
99,
|
||||
),
|
||||
all_stop_token_ids: {
|
||||
77,
|
||||
99,
|
||||
},
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_respects_ignore_eos_for_stop_token_ids() {
|
||||
let mut request = sample_request();
|
||||
request.sampling_params.ignore_eos = true;
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let params = prepared.generate_request.sampling_params;
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {
|
||||
77,
|
||||
99,
|
||||
},
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires network access to Hugging Face"]
|
||||
async fn lower_text_request_uses_real_qwen_generation_defaults() {
|
||||
let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")
|
||||
.await
|
||||
.expect("load qwen tokenizer and generation config");
|
||||
let hints = backend.sampling_hints().expect("collect sampling hints");
|
||||
|
||||
expect_test::expect![[r#"
|
||||
SamplingHints {
|
||||
primary_eos_token_id: Some(
|
||||
151645,
|
||||
),
|
||||
extra_eos_token_ids: {
|
||||
151643,
|
||||
},
|
||||
default_temperature: Some(
|
||||
0.6,
|
||||
),
|
||||
default_top_p: Some(
|
||||
0.95,
|
||||
),
|
||||
default_top_k: Some(
|
||||
20,
|
||||
),
|
||||
default_min_p: Some(
|
||||
0.1,
|
||||
),
|
||||
default_repetition_penalty: Some(
|
||||
1.2,
|
||||
),
|
||||
default_max_tokens: None,
|
||||
max_model_len: Some(
|
||||
40960,
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&hints);
|
||||
|
||||
let prepared =
|
||||
lower_text_request(sample_request(), vec![1, 2, 3], hints, &stub_tokenizer())
|
||||
.expect("lower request");
|
||||
let params = prepared.generate_request.sampling_params;
|
||||
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.6,
|
||||
top_p: 0.95,
|
||||
top_k: 20,
|
||||
seed: None,
|
||||
max_tokens: 40957,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.2,
|
||||
stop_token_ids: [
|
||||
151643,
|
||||
],
|
||||
eos_token_id: Some(
|
||||
151645,
|
||||
),
|
||||
all_stop_token_ids: {
|
||||
151643,
|
||||
151645,
|
||||
},
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_preserves_explicit_stop_token_ids_in_all_stop_set() {
|
||||
let sampling_params = SamplingParams {
|
||||
stop_token_ids: Some(vec![11, 77]),
|
||||
..SamplingParams::default()
|
||||
};
|
||||
|
||||
let params = lower_sampling_params(
|
||||
sampling_params,
|
||||
SamplingHints {
|
||||
primary_eos_token_id: Some(99),
|
||||
extra_eos_token_ids: BTreeSet::from([77, 88]),
|
||||
default_temperature: None,
|
||||
default_top_p: None,
|
||||
default_top_k: None,
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
},
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: [
|
||||
11,
|
||||
77,
|
||||
88,
|
||||
],
|
||||
eos_token_id: Some(
|
||||
99,
|
||||
),
|
||||
all_stop_token_ids: {
|
||||
11,
|
||||
77,
|
||||
88,
|
||||
99,
|
||||
},
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_prefers_user_values_over_generation_defaults() {
|
||||
let sampling_params = SamplingParams {
|
||||
temperature: Some(0.2),
|
||||
top_p: Some(0.3),
|
||||
top_k: Some(4),
|
||||
max_tokens: Some(32),
|
||||
min_tokens: Some(2),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let params = lower_sampling_params(
|
||||
sampling_params,
|
||||
SamplingHints {
|
||||
primary_eos_token_id: None,
|
||||
extra_eos_token_ids: BTreeSet::new(),
|
||||
default_temperature: Some(0.8),
|
||||
default_top_p: Some(0.9),
|
||||
default_top_k: Some(12),
|
||||
default_min_p: Some(0.1),
|
||||
default_repetition_penalty: Some(1.2),
|
||||
default_max_tokens: Some(128),
|
||||
max_model_len: None,
|
||||
},
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.2,
|
||||
top_p: 0.3,
|
||||
top_k: 4,
|
||||
seed: None,
|
||||
max_tokens: 32,
|
||||
min_tokens: 2,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.2,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {},
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_passes_logprobs_fields_through() {
|
||||
let sampling_params = SamplingParams {
|
||||
logprobs: Some(3),
|
||||
prompt_logprobs: Some(-1),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let params = lower_sampling_params(
|
||||
sampling_params,
|
||||
SamplingHints {
|
||||
primary_eos_token_id: None,
|
||||
extra_eos_token_ids: BTreeSet::new(),
|
||||
default_temperature: None,
|
||||
default_top_p: None,
|
||||
default_top_k: None,
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
},
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(params.logprobs, Some(3));
|
||||
assert_eq!(params.prompt_logprobs, Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() {
|
||||
let params = lower_sampling_params(
|
||||
SamplingParams::default(),
|
||||
SamplingHints {
|
||||
primary_eos_token_id: None,
|
||||
extra_eos_token_ids: BTreeSet::new(),
|
||||
default_temperature: Some(0.8),
|
||||
default_top_p: Some(0.9),
|
||||
default_top_k: Some(12),
|
||||
default_min_p: Some(0.1),
|
||||
default_repetition_penalty: Some(1.2),
|
||||
default_max_tokens: Some(128),
|
||||
max_model_len: None,
|
||||
},
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect_test::expect![[r#"
|
||||
EngineCoreSamplingParams {
|
||||
temperature: 0.8,
|
||||
top_p: 0.9,
|
||||
top_k: 12,
|
||||
seed: None,
|
||||
max_tokens: 128,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.2,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {},
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_caps_by_model_len() {
|
||||
let result = resolve_max_tokens(Some(150), None, Some(200), 100);
|
||||
assert_eq!(result.unwrap(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_preserves_non_streaming_request_metadata() {
|
||||
let mut request = sample_request();
|
||||
request.intermediate = false;
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(!prepared.text_request.intermediate);
|
||||
assert_eq!(prepared.generate_request.request_id, "text-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_user_smaller_than_model_limit() {
|
||||
let result = resolve_max_tokens(Some(50), None, Some(200), 100);
|
||||
assert_eq!(result.unwrap(), 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_uses_default_when_user_omits() {
|
||||
let result = resolve_max_tokens(None, Some(64), Some(200), 100);
|
||||
assert_eq!(result.unwrap(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_default_capped_by_model_len() {
|
||||
let result = resolve_max_tokens(None, Some(256), Some(200), 100);
|
||||
assert_eq!(result.unwrap(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_no_model_len_falls_back() {
|
||||
let result = resolve_max_tokens(Some(9999), None, None, 100);
|
||||
assert_eq!(result.unwrap(), 9999);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_no_limits_known_falls_back_to_u32_max() {
|
||||
let result = resolve_max_tokens(None, None, None, 100);
|
||||
assert_eq!(result.unwrap(), u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_prompt_too_long() {
|
||||
let result = resolve_max_tokens(Some(10), None, Some(100), 100);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::PromptTooLong {
|
||||
max_model_len: 100,
|
||||
prompt_len: 100,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_prompt_exceeds_model_len() {
|
||||
let result = resolve_max_tokens(Some(10), None, Some(100), 200);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::PromptTooLong {
|
||||
max_model_len: 100,
|
||||
prompt_len: 200,
|
||||
})
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::{Stream, StreamExt};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
use vllm_llm::{FinishReason, GenerateOutput};
|
||||
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
|
||||
|
||||
use super::logprobs::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
|
||||
};
|
||||
use crate::error::Error;
|
||||
|
||||
/// Request-neutral options for incremental text decoding.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TextDecodeOptions {
|
||||
pub skip_special_tokens: bool,
|
||||
pub include_stop_str_in_output: bool,
|
||||
pub stop_strings: Option<Vec<String>>,
|
||||
/// Minimum number of tokens to generate before stop-string checking kicks
|
||||
/// in. Stop strings found within the first `min_tokens` tokens are
|
||||
/// ignored.
|
||||
pub min_tokens: u32,
|
||||
}
|
||||
|
||||
impl Default for TextDecodeOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
skip_special_tokens: true,
|
||||
include_stop_str_in_output: false,
|
||||
stop_strings: None,
|
||||
min_tokens: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Terminal metadata carried on the final [`DecodedTextEvent`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Finished {
|
||||
pub prompt_token_count: usize,
|
||||
pub output_token_count: usize,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Internal decoded-text event emitted before higher-level assistant
|
||||
/// adaptation.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum DecodedTextEvent {
|
||||
/// The request has reached the point where prompt-scoped decoding metadata
|
||||
/// is ready.
|
||||
Start {
|
||||
/// The actual prompt token IDs for this request.
|
||||
prompt_token_ids: Arc<[u32]>,
|
||||
/// Once-only prompt logprobs metadata, when requested.
|
||||
///
|
||||
/// The first prompt token is carried separately because it has no left
|
||||
/// context to score against; `scored_positions` covers the
|
||||
/// remaining prompt positions.
|
||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
},
|
||||
/// A delta of text has been decoded, optionally alongside token-position
|
||||
/// logprobs.
|
||||
///
|
||||
/// `delta` is the newly visible decoded text fragment for this update.
|
||||
///
|
||||
/// `logprobs` covers the newly generated token positions from the same
|
||||
/// update, but is not guaranteed to align with `delta` by character
|
||||
/// span. One update may carry token logprobs but no newly visible text
|
||||
/// yet, and one visible text fragment may reflect multiple token
|
||||
/// positions becoming decodable together.
|
||||
///
|
||||
/// Upper-level may further parse `delta` as reasoning or tool calls.
|
||||
///
|
||||
/// When `finished` is `Some`, this is the terminal event for the request.
|
||||
TextDelta {
|
||||
delta: String,
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Convert the output token stream from the `vllm_llm` layer into incrementally
|
||||
/// decoded text.
|
||||
#[try_stream]
|
||||
pub async fn decoded_text_event_stream(
|
||||
request_id: String,
|
||||
tokenizer: DynTokenizer,
|
||||
mut raw_stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>> + Unpin,
|
||||
mut decode_options: TextDecodeOptions,
|
||||
intermediate: bool,
|
||||
mut y: TryYielder<DecodedTextEvent, Error>,
|
||||
) -> crate::Result<()> {
|
||||
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
|
||||
let mut prompt_token_count = 0_usize;
|
||||
let mut token_ids = Vec::new();
|
||||
let mut output_token_count: usize = 0;
|
||||
let mut logprobs: Option<DecodedLogprobs> = None;
|
||||
|
||||
while let Some(next) = raw_stream.next().await {
|
||||
let output = next?;
|
||||
|
||||
// If it's the first output, init states and yield `Start` event.
|
||||
if decoder.is_none() {
|
||||
let prompt_token_ids =
|
||||
output.prompt_token_ids().expect("first llm output must carry prompt token ids");
|
||||
prompt_token_count = prompt_token_ids.len();
|
||||
|
||||
let dec = tokenizer.create_decode_stream(
|
||||
prompt_token_ids,
|
||||
decode_options.skip_special_tokens,
|
||||
// If we are excluding stop strings from output, we need to buffer
|
||||
// the output so that we don't return the beginning of a stop string
|
||||
// when streaming the outputs.
|
||||
match decode_options.include_stop_str_in_output {
|
||||
true => 0,
|
||||
false => {
|
||||
decode_options
|
||||
.stop_strings
|
||||
.as_ref()
|
||||
.and_then(|stops| stops.iter().map(|ss| ss.len()).max())
|
||||
.unwrap_or(1)
|
||||
- 1
|
||||
}
|
||||
},
|
||||
);
|
||||
decoder = Some(dec);
|
||||
|
||||
y.yield_ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: prompt_token_ids.clone(),
|
||||
prompt_logprobs: output
|
||||
.prompt_logprobs()
|
||||
.map(|logprobs| {
|
||||
decode_prompt_logprobs(
|
||||
tokenizer.as_ref(),
|
||||
prompt_token_ids,
|
||||
logprobs,
|
||||
decode_options.skip_special_tokens,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
})
|
||||
.await;
|
||||
};
|
||||
let decoder = decoder.as_mut().unwrap();
|
||||
|
||||
let kv_transfer_params = output.kv_transfer_params;
|
||||
let mut finish_reason = output.finish_reason;
|
||||
let mut stop_str_matched = false;
|
||||
let suppress_terminal_stop_token = finish_reason.as_ref().is_some_and(|r| r.is_stop())
|
||||
&& !decode_options.include_stop_str_in_output;
|
||||
let decodable_token_ids = if suppress_terminal_stop_token {
|
||||
// Match Python V1 token-stop detokenization by keeping the stop token
|
||||
// in metadata while excluding it from user-visible text.
|
||||
output.token_ids.split_last().map(|(_, rest)| rest).unwrap_or(&[])
|
||||
} else {
|
||||
&output.token_ids
|
||||
};
|
||||
|
||||
let mut delta: Option<String> = None;
|
||||
let mut truncate_output_to = None;
|
||||
let mut truncate_tokens_to = None;
|
||||
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
|
||||
let new_bytes = decoder.push_token(token_id)?;
|
||||
if output_token_count + tok_idx + 1 > decode_options.min_tokens as usize
|
||||
&& let Some(stops) = decode_options.stop_strings.as_mut()
|
||||
&& let Some((idx, off)) = matches_stop_string(stops, decoder.output(), new_bytes)
|
||||
{
|
||||
let stop_str = stops.swap_remove(idx);
|
||||
truncate_output_to = match decode_options.include_stop_str_in_output {
|
||||
true => Some(off + stop_str.len()),
|
||||
false => Some(off),
|
||||
};
|
||||
finish_reason = Some(FinishReason::Stop(Some(StopReason::Text(stop_str))));
|
||||
truncate_tokens_to = Some(tok_idx + 1);
|
||||
stop_str_matched = true;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if intermediate && let Some(chunk) = decoder.next_chunk() {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut new_token_ids = output.token_ids;
|
||||
let mut new_logprobs = output.logprobs;
|
||||
|
||||
// Trim tokens and logprobs if we matched stop string.
|
||||
if let Some(num_tokens) = truncate_tokens_to {
|
||||
new_token_ids.truncate(num_tokens);
|
||||
if let Some(logprobs) = &mut new_logprobs {
|
||||
logprobs.positions.truncate(num_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
output_token_count += new_token_ids.len();
|
||||
|
||||
let decoded_logprobs = new_logprobs
|
||||
.as_ref()
|
||||
.map(|logprobs| {
|
||||
decode_logprobs(
|
||||
tokenizer.as_ref(),
|
||||
logprobs,
|
||||
decode_options.skip_special_tokens,
|
||||
)
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
if !intermediate {
|
||||
token_ids.extend(&new_token_ids);
|
||||
if let Some(dlp) = decoded_logprobs.as_ref() {
|
||||
logprobs
|
||||
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
|
||||
.positions
|
||||
.extend_from_slice(&dlp.positions);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(reason) = finish_reason {
|
||||
// Flush any remaining buffered text.
|
||||
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = text.len();
|
||||
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
|
||||
|
||||
if intermediate {
|
||||
if let Some(chunk) = last_chunk {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
}
|
||||
token_ids = new_token_ids;
|
||||
logprobs = decoded_logprobs;
|
||||
text = delta.unwrap_or_default();
|
||||
}
|
||||
|
||||
debug!(
|
||||
finish_reason = ?reason,
|
||||
text_length_bytes = text_len,
|
||||
output_token_count = output_token_count,
|
||||
"request finished with terminal output"
|
||||
);
|
||||
if let Some(full_text) = full_text {
|
||||
trace!(full_text, "request finished with terminal decoded text");
|
||||
}
|
||||
|
||||
// Intentionally drop the stream with explicit cause, so that the engine core
|
||||
// can distinguish between such normal completion vs an unexpected
|
||||
// early drop.
|
||||
if stop_str_matched {
|
||||
AbortCause::StopStringMatched.drop_as(raw_stream);
|
||||
}
|
||||
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: text,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished: Some(Finished {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason: reason,
|
||||
kv_transfer_params,
|
||||
}),
|
||||
})
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if intermediate {
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: delta.unwrap_or_default(),
|
||||
token_ids: new_token_ids,
|
||||
logprobs: decoded_logprobs,
|
||||
finished: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::StreamClosedBeforeTerminalOutput { request_id })
|
||||
}
|
||||
|
||||
/// If stop string matches, returns tuple
|
||||
/// (index into stop string vec, byte index of first byte of stop string in
|
||||
/// output)
|
||||
fn matches_stop_string(stops: &[String], output: &str, new_bytes: usize) -> Option<(usize, usize)> {
|
||||
// We compare byte subslices to avoid utf8 boundary problem
|
||||
let output = output.as_bytes();
|
||||
let next_off = (output.len() + 1) - new_bytes;
|
||||
stops
|
||||
.iter()
|
||||
.map(|ss| (ss.as_bytes(), ss.len(), next_off.saturating_sub(ss.len())))
|
||||
.enumerate()
|
||||
.find_map(|(ss_idx, (ss, len, start_off))| {
|
||||
output[start_off..]
|
||||
.windows(len)
|
||||
.rposition(|w| w == ss)
|
||||
.map(|pos| (ss_idx, start_off + pos))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::{Stream, stream};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_llm::GenerateOutput;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::*;
|
||||
use crate::output::TextOutputStreamExt as _;
|
||||
|
||||
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
|
||||
struct ByteTokenizer;
|
||||
|
||||
impl Tokenizer for ByteTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper: run `decoded_text_event_stream` to completion and return the
|
||||
/// collected output.
|
||||
async fn run_to_completion(
|
||||
token_ids: Vec<u32>,
|
||||
decode_options: TextDecodeOptions,
|
||||
) -> crate::output::CollectedTextOutput {
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let raw_stream = stream::iter(vec![Ok(GenerateOutput::for_test(
|
||||
Some(prompt),
|
||||
token_ids,
|
||||
Some(FinishReason::Length),
|
||||
))]);
|
||||
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
|
||||
decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)
|
||||
.collect_output()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Convert ASCII string to token IDs (one byte per token).
|
||||
fn ascii_tokens(s: &str) -> Vec<u32> {
|
||||
s.bytes().map(u32::from).collect()
|
||||
}
|
||||
|
||||
fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions {
|
||||
TextDecodeOptions {
|
||||
stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()),
|
||||
min_tokens,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
struct DropRecordingStream {
|
||||
next: Option<vllm_llm::Result<GenerateOutput>>,
|
||||
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
|
||||
}
|
||||
|
||||
impl Stream for DropRecordingStream {
|
||||
type Item = vllm_llm::Result<GenerateOutput>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
Poll::Ready(self.next.take())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DropRecordingStream {
|
||||
fn drop(&mut self) {
|
||||
*self.dropped_cause.lock().unwrap() = Some(AbortCause::current());
|
||||
}
|
||||
}
|
||||
|
||||
// --- stop string stream tests ---
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_sets_task_local_abort_cause_on_raw_stream_drop() {
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let dropped_cause = Arc::new(Mutex::new(None));
|
||||
let raw_stream = DropRecordingStream {
|
||||
next: Some(Ok(GenerateOutput::for_test(
|
||||
Some(prompt),
|
||||
ascii_tokens("hello"),
|
||||
Some(FinishReason::Length),
|
||||
))),
|
||||
dropped_cause: Arc::clone(&dropped_cause),
|
||||
};
|
||||
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
|
||||
|
||||
let output = decoded_text_event_stream(
|
||||
"test".into(),
|
||||
tokenizer,
|
||||
raw_stream,
|
||||
opts(&["ll"], 0),
|
||||
false,
|
||||
)
|
||||
.collect_output()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(output.text, "he");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
assert_eq!(
|
||||
*dropped_cause.lock().unwrap(),
|
||||
Some(AbortCause::StopStringMatched)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_truncates_at_match() {
|
||||
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 0)).await;
|
||||
assert_eq!(output.text, "h");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_at_end() {
|
||||
let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await;
|
||||
assert_eq!(output.text, "abc");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_first_token() {
|
||||
let output = run_to_completion(ascii_tokens("xhello"), opts(&["x"], 0)).await;
|
||||
assert_eq!(output.text, "");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_no_match_runs_to_completion() {
|
||||
let output = run_to_completion(ascii_tokens("hello"), opts(&["z"], 0)).await;
|
||||
assert_eq!(output.text, "hello");
|
||||
assert_eq!(output.finish_reason, FinishReason::Length);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_multi_char() {
|
||||
let output = run_to_completion(ascii_tokens("say hello world"), opts(&["lo"], 0)).await;
|
||||
assert_eq!(output.text, "say hel");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_first_of_multiple_wins() {
|
||||
// Both "ll" and "lo" are present; "ll" appears first in the output.
|
||||
let output = run_to_completion(ascii_tokens("hello"), opts(&["ll", "lo"], 0)).await;
|
||||
assert_eq!(output.text, "he");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_stop_string_include_in_output() {
|
||||
let output = run_to_completion(
|
||||
ascii_tokens("hello"),
|
||||
TextDecodeOptions {
|
||||
stop_strings: Some(vec!["ll".to_string()]),
|
||||
include_stop_str_in_output: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(output.text, "hell");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
// --- min_tokens + stop string interaction ---
|
||||
|
||||
#[tokio::test]
|
||||
async fn min_tokens_suppresses_early_stop_string() {
|
||||
// stop="e", min_tokens=3: the 'e' at token 2 is within the first 3 tokens,
|
||||
// so it should be skipped. No later 'e' exists, so output runs to completion.
|
||||
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 3)).await;
|
||||
assert_eq!(output.text, "hello");
|
||||
assert_eq!(output.finish_reason, FinishReason::Length);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn min_tokens_allows_stop_string_after_threshold() {
|
||||
// stop="e", min_tokens=2: the first 'e' at token 3 is past the threshold.
|
||||
let output = run_to_completion(ascii_tokens("greet"), opts(&["e"], 2)).await;
|
||||
assert_eq!(output.text, "gr");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn min_tokens_zero_behaves_like_absent() {
|
||||
let output = run_to_completion(ascii_tokens("hello"), opts(&["e"], 0)).await;
|
||||
assert_eq!(output.text, "h");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_matches_at_end() {
|
||||
let stops = vec!["wor".to_string()];
|
||||
// Output: "say wor", last byte 'r' was just added (new_bytes=1)
|
||||
let result = matches_stop_string(&stops, "say wor", 1);
|
||||
assert_eq!(result, Some((0, 4)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_no_match() {
|
||||
let stops = vec!["xyz".to_string()];
|
||||
let result = matches_stop_string(&stops, "say wor", 1);
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_matches_first_of_multiple() {
|
||||
let stops = vec!["wor".to_string(), "say".to_string()];
|
||||
// "say" appears earlier but "wor" is checked first (index 0)
|
||||
let result = matches_stop_string(&stops, "say wor", 1);
|
||||
assert_eq!(result, Some((0, 4)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_matches_second_of_multiple() {
|
||||
let stops = vec!["xyz".to_string(), "wor".to_string()];
|
||||
let result = matches_stop_string(&stops, "say wor", 1);
|
||||
assert_eq!(result, Some((1, 4)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_matches_with_multiple_new_bytes() {
|
||||
let stops = vec!["wor".to_string()];
|
||||
// "say wor" where last 3 bytes "wor" were added at once
|
||||
let result = matches_stop_string(&stops, "say wor", 3);
|
||||
assert_eq!(result, Some((0, 4)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_matches_at_beginning() {
|
||||
let stops = vec!["say".to_string()];
|
||||
let result = matches_stop_string(&stops, "say wor", 7);
|
||||
assert_eq!(result, Some((0, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_exact_output() {
|
||||
let stops = vec!["abc".to_string()];
|
||||
let result = matches_stop_string(&stops, "abc", 3);
|
||||
assert_eq!(result, Some((0, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_single_char() {
|
||||
let stops = vec!["!".to_string()];
|
||||
let result = matches_stop_string(&stops, "hello!", 1);
|
||||
assert_eq!(result, Some((0, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_not_in_new_bytes_region() {
|
||||
let stops = vec!["say".to_string()];
|
||||
// "say" is in the output but before the new byte region.
|
||||
// new_bytes=1 means only 'r' was added; "say" ended at byte 3,
|
||||
// but the search window starts at next_off - stop_len = 7+1-1 - 3 = 4.
|
||||
let result = matches_stop_string(&stops, "say wor", 1);
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_empty_list() {
|
||||
let stops: Vec<String> = vec![];
|
||||
let result = matches_stop_string(&stops, "hello", 1);
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_string_multibyte_utf8() {
|
||||
let stops = vec!["世界".to_string()];
|
||||
// "你好世界" is 12 bytes: 你(3) + 好(3) + 世(3) + 界(3)
|
||||
// "世界" starts at byte 6
|
||||
let result = matches_stop_string(&stops, "你好世界", 3);
|
||||
assert_eq!(result, Some((0, 6)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
use itertools::Itertools as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_llm::{Logprobs, PositionLogprobs};
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
/// One decoded token candidate and its logprob metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DecodedTokenLogprob {
|
||||
/// Original vocabulary token ID for this candidate.
|
||||
pub token_id: u32,
|
||||
/// Best-effort decoded token string for this candidate.
|
||||
pub token: String,
|
||||
/// Log probability of this token candidate.
|
||||
pub logprob: f32,
|
||||
/// Vocabulary rank of this token candidate.
|
||||
pub rank: u32,
|
||||
}
|
||||
|
||||
/// One position's decoded token candidates and their logprobs.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DecodedPositionLogprobs {
|
||||
/// Candidate tokens for this position.
|
||||
pub entries: Vec<DecodedTokenLogprob>,
|
||||
}
|
||||
|
||||
/// Decoded sample logprobs for generated token positions.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DecodedLogprobs {
|
||||
/// Generated token positions covered by this payload.
|
||||
pub positions: Vec<DecodedPositionLogprobs>,
|
||||
}
|
||||
|
||||
/// Decoded prompt logprobs for prompt token positions.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DecodedPromptLogprobs {
|
||||
/// Original vocabulary token ID for the first prompt token.
|
||||
pub first_token_id: u32,
|
||||
/// Best-effort decoded string for the first prompt token.
|
||||
///
|
||||
/// The first prompt token has no left context to score against, so it is
|
||||
/// stored separately instead of appearing in `scored_positions`.
|
||||
pub first_token: String,
|
||||
/// Scored prompt positions after the first prompt token.
|
||||
///
|
||||
/// `scored_positions[i]` corresponds to the prompt token at position `i +
|
||||
/// 1`.
|
||||
pub scored_positions: Vec<DecodedPositionLogprobs>,
|
||||
}
|
||||
|
||||
/// Decode generated-token logprobs from the raw `llm` token-ID shape into the
|
||||
/// text-layer decoded-token representation.
|
||||
///
|
||||
/// Each returned position corresponds to one generated token position from the
|
||||
/// same `llm` update.
|
||||
pub(super) fn decode_logprobs<T: Tokenizer + ?Sized>(
|
||||
tokenizer: &T,
|
||||
logprobs: &Logprobs,
|
||||
skip_special_tokens: bool,
|
||||
) -> Result<DecodedLogprobs, Error> {
|
||||
Ok(DecodedLogprobs {
|
||||
positions: logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(|position| decode_position_logprobs(tokenizer, position, skip_special_tokens))
|
||||
.try_collect()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode prompt logprobs from the raw `llm` token-ID shape into the text-layer
|
||||
/// decoded-token representation.
|
||||
///
|
||||
/// The returned payload stores the first prompt token separately and decodes
|
||||
/// the remaining scored prompt positions into `scored_positions`, matching
|
||||
/// vLLM's prompt-logprobs semantics.
|
||||
pub(super) fn decode_prompt_logprobs<T: Tokenizer + ?Sized>(
|
||||
tokenizer: &T,
|
||||
prompt_token_ids: &[u32],
|
||||
logprobs: &Logprobs,
|
||||
skip_special_tokens: bool,
|
||||
) -> Result<DecodedPromptLogprobs, Error> {
|
||||
let first_token_id = prompt_token_ids
|
||||
.first()
|
||||
.copied()
|
||||
.expect("prompt logprobs require at least one prompt token");
|
||||
let first_token = tokenizer.decode(&[first_token_id], skip_special_tokens)?;
|
||||
let scored_positions = logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(|position| decode_position_logprobs(tokenizer, position, skip_special_tokens))
|
||||
.try_collect()?;
|
||||
|
||||
Ok(DecodedPromptLogprobs {
|
||||
first_token_id,
|
||||
first_token,
|
||||
scored_positions,
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode one token position's raw candidate set into decoded token strings
|
||||
/// plus logprob metadata.
|
||||
///
|
||||
/// This decodes every candidate token ID independently through the active text
|
||||
/// backend.
|
||||
fn decode_position_logprobs<T: Tokenizer + ?Sized>(
|
||||
tokenizer: &T,
|
||||
position: &PositionLogprobs,
|
||||
skip_special_tokens: bool,
|
||||
) -> Result<DecodedPositionLogprobs, Error> {
|
||||
Ok(DecodedPositionLogprobs {
|
||||
entries: position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
tokenizer.decode(&[entry.token_id], skip_special_tokens).map(|token| {
|
||||
DecodedTokenLogprob {
|
||||
token_id: entry.token_id,
|
||||
token,
|
||||
logprob: entry.logprob,
|
||||
rank: entry.rank,
|
||||
}
|
||||
})
|
||||
})
|
||||
.try_collect()?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ByteTokenizer;
|
||||
|
||||
impl vllm_tokenizer::Tokenizer for ByteTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::from_utf8_lossy(
|
||||
&token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
|
||||
)
|
||||
.into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_logprobs_decodes_every_candidate_token() {
|
||||
let tokenizer = ByteTokenizer;
|
||||
let logprobs = Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: b'a' as u32,
|
||||
logprob: -0.1,
|
||||
rank: 3,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: b'b' as u32,
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
decode_logprobs(&tokenizer, &logprobs, false).unwrap(),
|
||||
DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
token_id: b'a' as u32,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 3,
|
||||
},
|
||||
DecodedTokenLogprob {
|
||||
token_id: b'b' as u32,
|
||||
token: "b".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_prompt_logprobs_separates_first_prompt_token() {
|
||||
let tokenizer = ByteTokenizer;
|
||||
let logprobs = Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![TokenLogprob {
|
||||
token_id: b'x' as u32,
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
decode_prompt_logprobs(&tokenizer, &[b'p' as u32, b'x' as u32], &logprobs, false)
|
||||
.unwrap(),
|
||||
DecodedPromptLogprobs {
|
||||
first_token_id: b'p' as u32,
|
||||
first_token: "p".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: b'x' as u32,
|
||||
token: "x".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//! Output processing helpers shared by text and chat layers.
|
||||
|
||||
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
|
||||
pub use logprobs::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
|
||||
};
|
||||
|
||||
mod decoded;
|
||||
mod logprobs;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt as _, pin_mut};
|
||||
|
||||
use crate::{Error, FinishReason, Result, TextOutputStream};
|
||||
|
||||
/// Final decoded text plus terminal stream metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CollectedTextOutput {
|
||||
pub text: String,
|
||||
pub prompt_token_ids: Arc<[u32]>,
|
||||
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
|
||||
#[easy_ext::ext(TextOutputStreamExt)]
|
||||
impl<T: TextOutputStream> T {
|
||||
/// Collect the stream to completion and return the final decoded text plus
|
||||
/// terminal metadata.
|
||||
pub fn collect_output(self) -> impl Future<Output = Result<CollectedTextOutput>> + Send {
|
||||
async move {
|
||||
let stream = self;
|
||||
pin_mut!(stream);
|
||||
let mut prompt_logprobs = None;
|
||||
let mut prompt_token_ids: Arc<[u32]> = Arc::from([]);
|
||||
let mut collected: Option<CollectedTextOutput> = None;
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
DecodedTextEvent::Start {
|
||||
prompt_logprobs: start_prompt_logprobs,
|
||||
prompt_token_ids: start_prompt_token_ids,
|
||||
..
|
||||
} => {
|
||||
prompt_logprobs = start_prompt_logprobs;
|
||||
prompt_token_ids = start_prompt_token_ids;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids: delta_token_ids,
|
||||
logprobs: mut delta_logprobs,
|
||||
finished,
|
||||
} => {
|
||||
if let Some(c) = collected.as_mut() {
|
||||
c.text.push_str(&delta);
|
||||
c.token_ids.extend(delta_token_ids);
|
||||
if let Some(dlp) = delta_logprobs.as_mut() {
|
||||
if let Some(lp) = c.logprobs.as_mut() {
|
||||
lp.positions.extend_from_slice(&dlp.positions);
|
||||
} else {
|
||||
c.logprobs = delta_logprobs;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
collected = Some(CollectedTextOutput {
|
||||
text: delta,
|
||||
prompt_token_ids: Arc::clone(&prompt_token_ids),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
logprobs: delta_logprobs,
|
||||
token_ids: delta_token_ids,
|
||||
finish_reason: FinishReason::Error,
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
};
|
||||
|
||||
if let Some(finished) = finished {
|
||||
let mut collected = collected.unwrap();
|
||||
collected.finish_reason = finished.finish_reason;
|
||||
collected.kv_transfer_params = finished.kv_transfer_params;
|
||||
return Ok(collected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Note: this is actually unreachable, as the underlying stream always emit an
|
||||
// error on unexpected close.
|
||||
Err(Error::StreamClosedBeforeTerminalOutput {
|
||||
request_id: "unknown".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures::stream;
|
||||
use vllm_llm::FinishReason;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_output_retains_prompt_and_sample_logprobs() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![10, 11].into(),
|
||||
prompt_logprobs: Some(DecodedPromptLogprobs {
|
||||
first_token_id: 0,
|
||||
first_token: "o".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "p".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "bc".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let collected = stream.collect_output().await.unwrap();
|
||||
assert_eq!(collected.text, "bc");
|
||||
assert_eq!(
|
||||
collected.prompt_logprobs,
|
||||
Some(DecodedPromptLogprobs {
|
||||
first_token_id: 0,
|
||||
first_token: "o".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "p".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
collected.logprobs,
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_output_accumulates_intermediate_deltas() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![10, 11].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "he".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "llo".to_string(),
|
||||
token_ids: vec![3, 4, 5],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 5,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let collected = stream.collect_output().await.unwrap();
|
||||
assert_eq!(collected.text, "hello");
|
||||
assert_eq!(collected.prompt_logprobs, None);
|
||||
assert_eq!(collected.token_ids, vec![1, 2, 3, 4, 5]);
|
||||
assert_eq!(
|
||||
collected.logprobs,
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::StructuredOutputsParams;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::output::TextDecodeOptions;
|
||||
|
||||
/// One raw text-generation prompt.
|
||||
///
|
||||
/// This supports either ordinary text that still needs tokenization or
|
||||
/// already-tokenized prompt IDs that should bypass tokenizer work entirely.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, EnumAsInner)]
|
||||
#[serde(untagged)]
|
||||
pub enum Prompt {
|
||||
/// Untokenized prompt text that still needs tokenizer work before
|
||||
/// generation.
|
||||
Text(String),
|
||||
/// Pre-tokenized prompt IDs that should be forwarded southbound without
|
||||
/// re-encoding.
|
||||
TokenIds(Vec<u32>),
|
||||
}
|
||||
|
||||
impl Default for Prompt {
|
||||
fn default() -> Self {
|
||||
Self::Text(String::new()) // placeholder
|
||||
}
|
||||
}
|
||||
|
||||
/// User-facing sampling parameters accepted by `vllm-text`.
|
||||
///
|
||||
/// This intentionally keeps only the subset that the current Rust text layer
|
||||
/// supports as northbound request semantics. Engine-core-specific normalized
|
||||
/// fields are derived later during lowering.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct SamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling. `None` means no explicit user override.
|
||||
pub temperature: Option<f32>,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
pub top_p: Option<f32>,
|
||||
/// Maximum number of top tokens to consider. `Some(0)` means all tokens.
|
||||
pub top_k: Option<u32>,
|
||||
/// Random seed used by the sampler when present.
|
||||
pub seed: Option<i64>,
|
||||
/// Maximum number of tokens to generate. `None` means no explicit user
|
||||
/// override.
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
pub min_tokens: Option<u32>,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
|
||||
pub logprobs: Option<i32>,
|
||||
/// Number of log probabilities to return per prompt token.
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
/// explicit user override.
|
||||
pub min_p: Option<f32>,
|
||||
/// Frequency penalty applied by the sampler. `None` means no explicit user
|
||||
/// override.
|
||||
pub frequency_penalty: Option<f32>,
|
||||
/// Presence penalty applied by the sampler. `None` means no explicit user
|
||||
/// override.
|
||||
pub presence_penalty: Option<f32>,
|
||||
/// Repetition penalty applied by the sampler. `None` means no explicit user
|
||||
/// override.
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// Explicit stop token IDs provided by the caller. `None` means no explicit
|
||||
/// user override.
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
/// If true, do not stop on the model's primary EOS token.
|
||||
pub ignore_eos: bool,
|
||||
/// Modify the likelihood of specified tokens appearing in the completion.
|
||||
/// Keys are token IDs.
|
||||
pub logit_bias: Option<HashMap<u32, f32>>,
|
||||
/// Restrict output to these token IDs only.
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
/// Words to avoid during generation (tokenized to IDs during lowering).
|
||||
pub bad_words: Option<Vec<String>>,
|
||||
/// Specific token IDs for which log probabilities should be returned at
|
||||
/// each position.
|
||||
///
|
||||
/// When set, the engine returns logprobs for exactly these tokens in
|
||||
/// addition to the sampled/scored token. Mutually exclusive with
|
||||
/// `logprobs` in practice.
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
pub structured_outputs: Option<StructuredOutputsParams>,
|
||||
/// If true, bypass reads from the prefix cache for this request (the prompt
|
||||
/// will not reuse cached KV blocks from earlier requests, though newly
|
||||
/// computed blocks may still populate the cache). `None` defers to
|
||||
/// engine-core defaults.
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions.
|
||||
pub vllm_xargs: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
#[allow(clippy::derivable_impls)] // more explicit
|
||||
impl Default for SamplingParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
top_k: None,
|
||||
seed: None,
|
||||
max_tokens: None,
|
||||
min_tokens: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: None,
|
||||
frequency_penalty: None,
|
||||
presence_penalty: None,
|
||||
repetition_penalty: None,
|
||||
stop_token_ids: None,
|
||||
ignore_eos: false,
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words: None,
|
||||
logprob_token_ids: None,
|
||||
structured_outputs: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One raw text-generation request ready to be tokenized or sent directly to
|
||||
/// the engine.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TextRequest {
|
||||
/// Stable caller-supplied request ID.
|
||||
pub request_id: String,
|
||||
/// Prompt text or prompt token IDs for this request.
|
||||
pub prompt: Prompt,
|
||||
/// Multimodal features prepared by a higher-level frontend. Raw text
|
||||
/// requests keep this empty; multimodal chat uses it with pre-tokenized
|
||||
/// prompt IDs.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
/// User-facing sampling parameters accepted by `vllm-text`.
|
||||
pub sampling_params: SamplingParams,
|
||||
/// Incremental detokenization options for the response path.
|
||||
pub decode_options: TextDecodeOptions,
|
||||
/// Whether to emit intermediate northbound deltas before the terminal
|
||||
/// result.
|
||||
///
|
||||
/// If `false`, callers only observe the terminal accumulated output. If
|
||||
/// `true`, callers may receive zero or more incremental decoded updates
|
||||
/// before the final terminal event.
|
||||
pub intermediate: bool,
|
||||
/// Request scheduling priority (lower means earlier handling; default 0).
|
||||
pub priority: i32,
|
||||
/// Salt for prefix cache isolation in multi-user environments.
|
||||
pub cache_salt: Option<String>,
|
||||
/// Whether to add special tokens (e.g. BOS) during prompt tokenization.
|
||||
pub add_special_tokens: bool,
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
}
|
||||
|
||||
impl TextRequest {
|
||||
/// Return one minimal valid request fixture for tests.
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
request_id: "test-request".to_string(),
|
||||
prompt: Prompt::Text("test".to_string()),
|
||||
mm_features: None,
|
||||
sampling_params: SamplingParams::default(),
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: true,
|
||||
priority: 0,
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the minimum invariants before tokenization or request lowering.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if matches!(&self.prompt, Prompt::TokenIds(ids) if ids.is_empty()) {
|
||||
return Err(Error::EmptyPromptTokenIds {
|
||||
request_id: self.request_id.clone(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user