[Rust Frontend] Support max_logprobs validation (#45674)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -127,6 +127,11 @@ pub struct SharedRuntimeArgs {
|
||||
/// `config.json`.
|
||||
#[arg(long)]
|
||||
pub max_model_len: Option<u32>,
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in sampling parameters. `-1` means no cap.
|
||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||
#[serde(default)]
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
|
||||
/// started.
|
||||
#[arg(long)]
|
||||
@@ -281,6 +286,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
@@ -324,6 +330,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
@@ -467,6 +474,7 @@ impl ServeArgs {
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
self.runtime.max_logprobs,
|
||||
self.runtime.language_model_only,
|
||||
self.runtime.disable_log_stats,
|
||||
self.runtime.shutdown_timeout,
|
||||
|
||||
@@ -38,6 +38,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
max_model_len: Some(
|
||||
512,
|
||||
),
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -134,6 +135,29 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
|
||||
assert_eq!(config.python_args, vec!["--disable-log-stats"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--max-logprobs",
|
||||
"-1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.max_logprobs, Some(-1));
|
||||
|
||||
let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert_eq!(frontend_config.max_logprobs, Some(-1));
|
||||
|
||||
let engine_config = args.to_managed_engine_config(5555);
|
||||
assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
|
||||
@@ -388,6 +412,7 @@ fn frontend_args_accept_json() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_model_len: None,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -431,6 +456,7 @@ fn frontend_args_json_applies_defaults() {
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_model_len, None);
|
||||
assert_eq!(args.runtime.max_logprobs, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
}
|
||||
|
||||
@@ -446,7 +472,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"shutdown_timeout":3}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -465,6 +491,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
assert!(args.runtime.language_model_only);
|
||||
assert_eq!(args.runtime.max_model_len, Some(8192));
|
||||
assert_eq!(args.runtime.max_logprobs, Some(-1));
|
||||
assert_eq!(args.runtime.shutdown_timeout, 3);
|
||||
}
|
||||
|
||||
@@ -792,6 +819,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_model_len: None,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -917,6 +945,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -985,6 +1014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -1068,6 +1098,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
|
||||
@@ -202,14 +202,6 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub tokenizer_revision: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in `SamplingParams`. The default value comes the default for
|
||||
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
|
||||
/// (output_length * vocab_size) logprobs are allowed to be returned and
|
||||
/// it may cause OOM.
|
||||
#[arg(long)]
|
||||
pub max_logprobs: Option<Unsupported>,
|
||||
|
||||
/// Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
/// `prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
/// output will contain token ids.
|
||||
|
||||
@@ -71,6 +71,7 @@ impl ManagedEngineArgs {
|
||||
self,
|
||||
model: String,
|
||||
max_model_len: Option<u32>,
|
||||
max_logprobs: Option<i32>,
|
||||
language_model_only: bool,
|
||||
disable_log_stats: bool,
|
||||
shutdown_timeout: u64,
|
||||
@@ -82,6 +83,10 @@ impl ManagedEngineArgs {
|
||||
python_args.push("--max-model-len".to_string());
|
||||
python_args.push(max_model_len.to_string());
|
||||
}
|
||||
if let Some(max_logprobs) = max_logprobs {
|
||||
python_args.push("--max-logprobs".to_string());
|
||||
python_args.push(max_logprobs.to_string());
|
||||
}
|
||||
if language_model_only {
|
||||
python_args.push("--language-model-only".to_string());
|
||||
}
|
||||
|
||||
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
api_keys: Vec::new(),
|
||||
disable_log_stats: false,
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Result, bail};
|
||||
use educe::Educe;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
@@ -77,6 +77,9 @@ pub struct Config {
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Optional maximum number of top log probabilities accepted by the
|
||||
/// frontend. `None` delegates to the text layer default.
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// HTTP/API-server behavior switches.
|
||||
pub api_server_options: ApiServerOptions,
|
||||
/// API keys accepted as bearer tokens for guarded routes.
|
||||
@@ -98,6 +101,14 @@ impl Config {
|
||||
/// startup.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
|
||||
if let Some(max_logprobs) = self.max_logprobs
|
||||
&& max_logprobs < -1
|
||||
{
|
||||
bail!(
|
||||
"max_logprobs must be non-negative or -1, got {}",
|
||||
max_logprobs
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -74,12 +74,11 @@ impl IntoResponse for ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a text-pipeline submit failure: tokenized-prompt validation
|
||||
/// failures (the prompt is too long for the model, or empty after
|
||||
/// tokenization) are the client's fault and map to HTTP 400, mirroring the
|
||||
/// Python frontend. Everything else stays an internal 500.
|
||||
/// Classify a text-pipeline submit failure: request validation failures are
|
||||
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
|
||||
/// Everything else stays an internal 500.
|
||||
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
|
||||
if is_prompt_validation_error(&error) {
|
||||
if is_request_validation_error(&error) {
|
||||
return invalid_request!("{error}");
|
||||
}
|
||||
server_error!("{}: {}", context, error.to_report_string())
|
||||
@@ -90,18 +89,19 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
|
||||
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
|
||||
match &error {
|
||||
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
|
||||
vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
|
||||
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
|
||||
invalid_request!("{error}")
|
||||
}
|
||||
_ => server_error!("{}: {}", context, error.to_report_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
|
||||
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
vllm_text::Error::PromptTooLong { .. }
|
||||
| vllm_text::Error::EmptyPromptTokenIds { .. }
|
||||
| vllm_text::Error::Logprobs(_)
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
|
||||
@@ -145,6 +145,33 @@ mod tests {
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logprobs_validation_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::Logprobs(vllm_text::LogprobsError::TooManyCount {
|
||||
parameter: "logprobs",
|
||||
requested: 1000,
|
||||
max_allowed: 20,
|
||||
});
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("logprobs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wrapped_logprobs_validation_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::Text(vllm_text::Error::Logprobs(
|
||||
vllm_text::LogprobsError::TooManyCount {
|
||||
parameter: "prompt_logprobs",
|
||||
requested: 1000,
|
||||
max_allowed: 20,
|
||||
},
|
||||
));
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_submit_errors_stay_internal() {
|
||||
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
|
||||
|
||||
@@ -90,7 +90,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
|
||||
let text = TextLlm::new(llm, text_backend);
|
||||
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||
|
||||
@@ -245,10 +245,6 @@ impl ModelConfig {
|
||||
pub(super) fn is_moe(&self) -> bool {
|
||||
self.num_experts() > 0
|
||||
}
|
||||
|
||||
pub(super) fn max_position_embeddings(&self) -> Option<u32> {
|
||||
self.effective_text_config().max_position_embeddings
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the tokenizer-side EOS metadata if a config file is present.
|
||||
@@ -356,7 +352,10 @@ mod tests {
|
||||
|
||||
assert_eq!(config.num_experts(), 8);
|
||||
assert_eq!(config.model_type(), Some("top_level"));
|
||||
assert_eq!(config.max_position_embeddings(), Some(4096));
|
||||
assert_eq!(
|
||||
config.effective_text_config().max_position_embeddings,
|
||||
Some(4096)
|
||||
);
|
||||
assert!(config.is_moe());
|
||||
}
|
||||
|
||||
@@ -367,7 +366,10 @@ mod tests {
|
||||
|
||||
assert_eq!(config.num_experts(), 0);
|
||||
assert!(!config.is_moe());
|
||||
assert_eq!(config.max_position_embeddings(), Some(4096));
|
||||
assert_eq!(
|
||||
config.effective_text_config().max_position_embeddings,
|
||||
Some(4096)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -118,7 +118,6 @@ impl TextBackend for HfTextBackend {
|
||||
default_min_p: self.generation_config.min_p,
|
||||
default_repetition_penalty: self.generation_config.repetition_penalty,
|
||||
default_max_tokens: self.generation_config.max_new_tokens,
|
||||
max_model_len: self.model_config.max_position_embeddings(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,8 +6,8 @@ use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Tokenizer/model-derived hints used to enrich text-generation requests before
|
||||
/// they are lowered into engine-core.
|
||||
/// Tokenizer/model-derived defaults used to enrich text-generation requests
|
||||
/// before they are lowered into engine-core.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct SamplingHints {
|
||||
pub primary_eos_token_id: Option<u32>,
|
||||
@@ -18,9 +18,36 @@ pub struct SamplingHints {
|
||||
pub default_min_p: Option<f32>,
|
||||
pub default_repetition_penalty: Option<f32>,
|
||||
pub default_max_tokens: Option<u32>,
|
||||
/// Model context window size (`max_position_embeddings` from
|
||||
/// `config.json`).
|
||||
pub max_model_len: Option<u32>,
|
||||
}
|
||||
|
||||
/// Effective bounds used to validate and lower sampling requests.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SamplingLimits {
|
||||
/// Runtime context window size reported by the engine startup handshake.
|
||||
pub max_model_len: u32,
|
||||
/// Maximum number of top log probabilities accepted by this frontend.
|
||||
///
|
||||
/// `-1` means allowing requests up to the model vocabulary size.
|
||||
pub max_logprobs: i32,
|
||||
/// Model vocabulary size from the model config.
|
||||
pub model_vocab_size: Option<usize>,
|
||||
/// Tokenizer vocabulary size, used as a fallback when the model config does
|
||||
/// not expose a vocabulary size.
|
||||
pub tokenizer_vocab_size: usize,
|
||||
}
|
||||
|
||||
impl SamplingLimits {
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/b5adb027ad03c29b46181752ba3b1cb84eff1dd4/vllm/config/model.py#L216-L220>
|
||||
pub const DEFAULT_MAX_LOGPROBS: i32 = 20;
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/b5adb027ad03c29b46181752ba3b1cb84eff1dd4/vllm/sampling_params.py#L30-L32>
|
||||
pub const MAX_LOGPROB_TOKEN_IDS: usize = 128;
|
||||
|
||||
/// Return the vocabulary size used to expand `logprobs=-1`.
|
||||
pub fn logprobs_vocab_size(&self) -> usize {
|
||||
self.model_vocab_size.unwrap_or(self.tokenizer_vocab_size)
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal text-processing backend needed by `vllm-text`.
|
||||
|
||||
@@ -2,6 +2,8 @@ use thiserror::Error;
|
||||
use vllm_engine_core_client::Error as EngineCoreError;
|
||||
use vllm_llm::Error as LlmError;
|
||||
|
||||
pub use crate::lower::logprobs::LogprobsError;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("tokenizer error: {0}")]
|
||||
@@ -13,6 +15,8 @@ pub enum Error {
|
||||
but the prompt contains {prompt_len} input tokens"
|
||||
)]
|
||||
PromptTooLong { max_model_len: u32, prompt_len: u32 },
|
||||
#[error(transparent)]
|
||||
Logprobs(#[from] LogprobsError),
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error(transparent)]
|
||||
|
||||
+28
-16
@@ -6,8 +6,8 @@
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
pub use backend::{DynTextBackend, SamplingHints, TextBackend};
|
||||
pub use error::{Error, Result};
|
||||
pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
|
||||
pub use error::{Error, LogprobsError, Result};
|
||||
use futures::Stream;
|
||||
pub use lower::{
|
||||
PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,
|
||||
@@ -45,33 +45,33 @@ pub struct TextLlm {
|
||||
/// Tokenizer/model metadata backend responsible for prompt encode/decode
|
||||
/// and sampling hints.
|
||||
backend: DynTextBackend,
|
||||
/// Context window size reported by the engine startup handshake, with
|
||||
/// optional override from config.
|
||||
/// Runtime context window size reported by the engine startup handshake.
|
||||
max_model_len: u32,
|
||||
/// Maximum number of top log probabilities accepted by this text facade.
|
||||
max_logprobs: i32,
|
||||
}
|
||||
|
||||
impl TextLlm {
|
||||
/// Create a new text-generation facade from a shared LLM client plus a text
|
||||
/// backend.
|
||||
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
|
||||
// Prefer the engine-reported max_model_len because it reflects the
|
||||
// post-profiling, auto-fitted KV cache limit rather than static
|
||||
// frontend metadata.
|
||||
// The engine-reported value reflects the post-profiling, auto-fitted
|
||||
// KV cache limit used at runtime.
|
||||
let max_model_len = llm.engine_core_client().max_model_len();
|
||||
|
||||
Self {
|
||||
llm,
|
||||
backend,
|
||||
max_model_len,
|
||||
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the maximum model context length explicitly.
|
||||
///
|
||||
/// This takes priority over both the engine-reported default and any
|
||||
/// tokenizer/model metadata exposed by the backend.
|
||||
pub fn with_max_model_len(mut self, max_model_len: u32) -> Self {
|
||||
self.max_model_len = max_model_len;
|
||||
/// Override the maximum accepted logprobs count.
|
||||
pub fn with_max_logprobs(mut self, max_logprobs: Option<i32>) -> Self {
|
||||
if let Some(max_logprobs) = max_logprobs {
|
||||
self.max_logprobs = max_logprobs;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -140,12 +140,24 @@ impl TextLlm {
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
|
||||
let mut sampling_hints = self.backend.sampling_hints()?;
|
||||
sampling_hints.max_model_len = Some(self.max_model_len);
|
||||
let sampling_hints = self.backend.sampling_hints()?;
|
||||
let sampling_limits = SamplingLimits {
|
||||
max_model_len: self.max_model_len,
|
||||
max_logprobs: self.max_logprobs,
|
||||
model_vocab_size: self.backend.model_vocab_size(),
|
||||
tokenizer_vocab_size: self.backend.tokenizer_vocab_size(),
|
||||
};
|
||||
|
||||
let PreparedTextRequest {
|
||||
text_request,
|
||||
generate_request,
|
||||
} = lower_text_request(request, prompt_token_ids, sampling_hints, &*tokenizer)?;
|
||||
} = lower_text_request(
|
||||
request,
|
||||
prompt_token_ids,
|
||||
sampling_hints,
|
||||
sampling_limits,
|
||||
&*tokenizer,
|
||||
)?;
|
||||
|
||||
let raw_stream = self.llm.generate(generate_request).await?;
|
||||
Ok((text_request, raw_stream))
|
||||
|
||||
+185
-52
@@ -1,12 +1,15 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
pub(crate) mod logprobs;
|
||||
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
use vllm_llm::GenerateRequest;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use crate::backend::SamplingHints;
|
||||
use crate::backend::{SamplingHints, SamplingLimits};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{SamplingParams, TextRequest};
|
||||
use logprobs::validate_logprobs;
|
||||
|
||||
/// One text request after it has been lowered into the raw generate boundary.
|
||||
#[derive(Debug)]
|
||||
@@ -24,6 +27,7 @@ pub fn lower_text_request(
|
||||
request: TextRequest,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
sampling_hints: SamplingHints,
|
||||
sampling_limits: SamplingLimits,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<PreparedTextRequest> {
|
||||
let prompt_len = prompt_token_ids.len() as u32;
|
||||
@@ -34,6 +38,7 @@ pub fn lower_text_request(
|
||||
sampling_params: lower_sampling_params(
|
||||
request.sampling_params.clone(),
|
||||
sampling_hints,
|
||||
sampling_limits,
|
||||
prompt_len,
|
||||
tokenizer,
|
||||
)?,
|
||||
@@ -66,8 +71,8 @@ pub fn lower_sampling_params(
|
||||
default_min_p,
|
||||
default_repetition_penalty,
|
||||
default_max_tokens,
|
||||
max_model_len,
|
||||
}: SamplingHints,
|
||||
sampling_limits: SamplingLimits,
|
||||
prompt_len: u32,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<EngineCoreSamplingParams> {
|
||||
@@ -95,6 +100,14 @@ pub fn lower_sampling_params(
|
||||
vllm_xargs,
|
||||
} = sampling_params;
|
||||
|
||||
// Validate logprobs-related fields first with runtime sampling limits first.
|
||||
validate_logprobs(
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
logprob_token_ids.as_deref(),
|
||||
sampling_limits,
|
||||
)?;
|
||||
|
||||
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
|
||||
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
|
||||
// If neither the caller nor the model provides a value, fall back to 1.0 — the
|
||||
@@ -105,7 +118,12 @@ pub fn lower_sampling_params(
|
||||
let top_k = top_k.or(default_top_k).unwrap_or(0);
|
||||
let min_p = min_p.or(default_min_p).unwrap_or(0.0);
|
||||
let repetition_penalty = repetition_penalty.or(default_repetition_penalty).unwrap_or(1.0);
|
||||
let max_tokens = resolve_max_tokens(max_tokens, default_max_tokens, max_model_len, prompt_len)?;
|
||||
let max_tokens = resolve_max_tokens(
|
||||
max_tokens,
|
||||
default_max_tokens,
|
||||
sampling_limits.max_model_len,
|
||||
prompt_len,
|
||||
)?;
|
||||
let min_tokens = min_tokens.unwrap_or(0);
|
||||
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
|
||||
let presence_penalty = presence_penalty.unwrap_or(0.0);
|
||||
@@ -189,33 +207,25 @@ fn tokenize_bad_words(
|
||||
/// Resolve the effective `max_tokens` for generation, mirroring vLLM Python's
|
||||
/// `get_max_tokens()` in `vllm/entrypoints/utils.py`.
|
||||
///
|
||||
/// Takes the minimum of all available limits (user-specified, generation-config
|
||||
/// default, and `max_model_len - prompt_len`). When nothing is known, falls
|
||||
/// back to `u32::MAX` so the engine-core can apply its own context-window
|
||||
/// limit.
|
||||
/// Takes the minimum of all available limits: user-specified, generation-config
|
||||
/// default, and `max_model_len - prompt_len`.
|
||||
pub fn resolve_max_tokens(
|
||||
user_max_tokens: Option<u32>,
|
||||
default_max_tokens: Option<u32>,
|
||||
max_model_len: Option<u32>,
|
||||
max_model_len: u32,
|
||||
prompt_len: u32,
|
||||
) -> Result<u32> {
|
||||
let model_max_tokens = match max_model_len {
|
||||
Some(max_model_len) if prompt_len >= max_model_len => {
|
||||
return Err(Error::PromptTooLong {
|
||||
max_model_len,
|
||||
prompt_len,
|
||||
});
|
||||
}
|
||||
Some(max_model_len) => Some(max_model_len - prompt_len),
|
||||
None => None,
|
||||
let model_max_tokens = if prompt_len >= max_model_len {
|
||||
return Err(Error::PromptTooLong {
|
||||
max_model_len,
|
||||
prompt_len,
|
||||
});
|
||||
} else {
|
||||
max_model_len - prompt_len
|
||||
};
|
||||
|
||||
let fallback_max_tokens = user_max_tokens.or(default_max_tokens);
|
||||
Ok([fallback_max_tokens, model_max_tokens]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min()
|
||||
.unwrap_or(u32::MAX /* TODO: a reasonable fallback? */))
|
||||
let request_max_tokens = user_max_tokens.or(default_max_tokens);
|
||||
Ok(request_max_tokens.map_or(model_max_tokens, |n| n.min(model_max_tokens)))
|
||||
}
|
||||
|
||||
fn merge_unique_token_ids(
|
||||
@@ -240,6 +250,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::backend::hf::HfTextBackend;
|
||||
use crate::backend::{SamplingHints, TextBackend as _};
|
||||
use crate::error::LogprobsError;
|
||||
use crate::request::{Prompt, TextRequest};
|
||||
|
||||
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
|
||||
@@ -290,16 +301,47 @@ mod tests {
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_sampling_limits() -> SamplingLimits {
|
||||
SamplingLimits {
|
||||
max_model_len: 1_000_000,
|
||||
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
|
||||
model_vocab_size: Some(1000),
|
||||
tokenizer_vocab_size: 2000,
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_sampling_params_with_limits(
|
||||
sampling_params: SamplingParams,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<EngineCoreSamplingParams> {
|
||||
lower_sampling_params(
|
||||
sampling_params,
|
||||
SamplingHints {
|
||||
primary_eos_token_id: None,
|
||||
extra_eos_token_ids: BTreeSet::new(),
|
||||
default_temperature: None,
|
||||
default_top_p: None,
|
||||
default_top_k: None,
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
},
|
||||
sampling_limits,
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_applies_python_style_eos_hints() {
|
||||
let prepared = lower_text_request(
|
||||
sample_request(),
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -311,7 +353,7 @@ mod tests {
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
@@ -350,6 +392,7 @@ mod tests {
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -361,7 +404,7 @@ mod tests {
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
@@ -415,16 +458,23 @@ mod tests {
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: Some(
|
||||
40960,
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&hints);
|
||||
|
||||
let prepared =
|
||||
lower_text_request(sample_request(), vec![1, 2, 3], hints, &stub_tokenizer())
|
||||
.expect("lower request");
|
||||
let prepared = lower_text_request(
|
||||
sample_request(),
|
||||
vec![1, 2, 3],
|
||||
hints,
|
||||
SamplingLimits {
|
||||
max_model_len: 40960,
|
||||
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
|
||||
model_vocab_size: backend.model_vocab_size(),
|
||||
tokenizer_vocab_size: backend.tokenizer_vocab_size(),
|
||||
},
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.expect("lower request");
|
||||
let params = prepared.generate_request.sampling_params;
|
||||
|
||||
expect_test::expect![[r#"
|
||||
@@ -481,8 +531,8 @@ mod tests {
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
@@ -494,7 +544,7 @@ mod tests {
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 4294967295,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
@@ -550,8 +600,8 @@ mod tests {
|
||||
default_min_p: Some(0.1),
|
||||
default_repetition_penalty: Some(1.2),
|
||||
default_max_tokens: Some(128),
|
||||
max_model_len: None,
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
@@ -605,7 +655,10 @@ mod tests {
|
||||
default_min_p: None,
|
||||
default_repetition_penalty: None,
|
||||
default_max_tokens: None,
|
||||
max_model_len: None,
|
||||
},
|
||||
SamplingLimits {
|
||||
max_logprobs: -1,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
@@ -616,6 +669,91 @@ mod tests {
|
||||
assert_eq!(params.prompt_logprobs, Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_full_vocab_logprobs_over_default_cap() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logprobs: Some(-1),
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::Logprobs(LogprobsError::TooManyCount {
|
||||
parameter: "logprobs",
|
||||
requested: 1000,
|
||||
max_allowed: 20,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_expands_full_vocab_logprobs_from_model_vocab() {
|
||||
let params = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logprobs: Some(-1),
|
||||
..Default::default()
|
||||
},
|
||||
SamplingLimits {
|
||||
max_logprobs: 1500,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(params.logprobs, Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_uses_tokenizer_vocab_when_model_vocab_is_unknown() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logprobs: Some(-1),
|
||||
..Default::default()
|
||||
},
|
||||
SamplingLimits {
|
||||
max_logprobs: 1500,
|
||||
model_vocab_size: None,
|
||||
tokenizer_vocab_size: 2000,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::Logprobs(LogprobsError::TooManyCount {
|
||||
parameter: "logprobs",
|
||||
requested: 2000,
|
||||
max_allowed: 1500,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_invalid_logprob_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logprobs: Some(1),
|
||||
logprob_token_ids: Some(vec![1000]),
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::Logprobs(LogprobsError::InvalidTokenIds {
|
||||
token_ids,
|
||||
vocab_size: 1000,
|
||||
}) if token_ids == vec![1000]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() {
|
||||
let params = lower_sampling_params(
|
||||
@@ -629,8 +767,8 @@ mod tests {
|
||||
default_min_p: Some(0.1),
|
||||
default_repetition_penalty: Some(1.2),
|
||||
default_max_tokens: Some(128),
|
||||
max_model_len: None,
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
3,
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
@@ -667,7 +805,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_caps_by_model_len() {
|
||||
let result = resolve_max_tokens(Some(150), None, Some(200), 100);
|
||||
let result = resolve_max_tokens(Some(150), None, 200, 100);
|
||||
assert_eq!(result.unwrap(), 100);
|
||||
}
|
||||
|
||||
@@ -680,6 +818,7 @@ mod tests {
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -690,37 +829,31 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_user_smaller_than_model_limit() {
|
||||
let result = resolve_max_tokens(Some(50), None, Some(200), 100);
|
||||
let result = resolve_max_tokens(Some(50), None, 200, 100);
|
||||
assert_eq!(result.unwrap(), 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_uses_default_when_user_omits() {
|
||||
let result = resolve_max_tokens(None, Some(64), Some(200), 100);
|
||||
let result = resolve_max_tokens(None, Some(64), 200, 100);
|
||||
assert_eq!(result.unwrap(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_default_capped_by_model_len() {
|
||||
let result = resolve_max_tokens(None, Some(256), Some(200), 100);
|
||||
let result = resolve_max_tokens(None, Some(256), 200, 100);
|
||||
assert_eq!(result.unwrap(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_no_model_len_falls_back() {
|
||||
let result = resolve_max_tokens(Some(9999), None, None, 100);
|
||||
assert_eq!(result.unwrap(), 9999);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_no_limits_known_falls_back_to_u32_max() {
|
||||
let result = resolve_max_tokens(None, None, None, 100);
|
||||
assert_eq!(result.unwrap(), u32::MAX);
|
||||
fn resolve_max_tokens_uses_model_limit_when_user_omits() {
|
||||
let result = resolve_max_tokens(None, None, 200, 100);
|
||||
assert_eq!(result.unwrap(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_prompt_too_long() {
|
||||
let result = resolve_max_tokens(Some(10), None, Some(100), 100);
|
||||
let result = resolve_max_tokens(Some(10), None, 100, 100);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::PromptTooLong {
|
||||
@@ -732,7 +865,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_prompt_exceeds_model_len() {
|
||||
let result = resolve_max_tokens(Some(10), None, Some(100), 200);
|
||||
let result = resolve_max_tokens(Some(10), None, 100, 200);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::PromptTooLong {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
//! Python-compatible validation for logprobs sampling params.
|
||||
//!
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LogprobsError {
|
||||
#[error("{parameter} must be non-negative or -1, got {value}")]
|
||||
InvalidCount { parameter: &'static str, value: i32 },
|
||||
#[error(
|
||||
"requested {parameter} of {requested}, which is greater than max allowed: {max_allowed}"
|
||||
)]
|
||||
TooManyCount {
|
||||
parameter: &'static str,
|
||||
requested: usize,
|
||||
max_allowed: usize,
|
||||
},
|
||||
#[error(
|
||||
"requested logprob_token_ids of length {requested}, \
|
||||
which is greater than max allowed: {max_allowed}"
|
||||
)]
|
||||
TooManyTokenIds {
|
||||
requested: usize,
|
||||
max_allowed: usize,
|
||||
},
|
||||
#[error(
|
||||
"token_id(s) {token_ids:?} in logprob_token_ids contain out-of-vocab token ids. \
|
||||
Vocabulary size: {vocab_size}"
|
||||
)]
|
||||
InvalidTokenIds {
|
||||
token_ids: Vec<u32>,
|
||||
vocab_size: usize,
|
||||
},
|
||||
#[error(
|
||||
"when both logprobs and logprob_token_ids are set, logprobs must equal \
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
}
|
||||
|
||||
/// Validate logprobs-related sampling parameters, returning an error if any
|
||||
/// parameter is out of bounds or if the combination of parameters is invalid.
|
||||
pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let vocab_size = sampling_limits.logprobs_vocab_size();
|
||||
let max_logprobs =
|
||||
normalize_logprobs_count(sampling_limits.max_logprobs, vocab_size, "max_logprobs")?;
|
||||
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids, vocab_size)
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
vocab_size: usize,
|
||||
parameter: &'static str,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let Some(requested) = requested else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let requested = normalize_logprobs_count(requested, vocab_size, parameter)?;
|
||||
if requested > max_logprobs {
|
||||
return Err(LogprobsError::TooManyCount {
|
||||
parameter,
|
||||
requested,
|
||||
max_allowed: max_logprobs,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_logprob_token_ids(
|
||||
logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
vocab_size: usize,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let Some(logprob_token_ids) = logprob_token_ids else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let n = logprob_token_ids.len();
|
||||
if n > SamplingLimits::MAX_LOGPROB_TOKEN_IDS {
|
||||
return Err(LogprobsError::TooManyTokenIds {
|
||||
requested: n,
|
||||
max_allowed: SamplingLimits::MAX_LOGPROB_TOKEN_IDS,
|
||||
});
|
||||
}
|
||||
|
||||
let invalid_token_ids: Vec<_> = logprob_token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&token_id| token_id as usize >= vocab_size)
|
||||
.collect();
|
||||
if !invalid_token_ids.is_empty() {
|
||||
return Err(LogprobsError::InvalidTokenIds {
|
||||
token_ids: invalid_token_ids,
|
||||
vocab_size,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(logprobs) = logprobs
|
||||
&& logprobs != n as i32
|
||||
{
|
||||
return Err(LogprobsError::TokenIdsMismatch {
|
||||
logprobs,
|
||||
num_token_ids: n,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_logprobs_count(
|
||||
value: i32,
|
||||
vocab_size: usize,
|
||||
parameter: &'static str,
|
||||
) -> Result<usize, LogprobsError> {
|
||||
match value {
|
||||
-1 => Ok(vocab_size),
|
||||
value if value < 0 => Err(LogprobsError::InvalidCount { parameter, value }),
|
||||
value => Ok(value as usize),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user