[Bugfix][Rust Frontend] Validate out-of-vocab token ids in request params (#44680)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Ting SUN
2026-06-11 03:14:11 -07:00
committed by GitHub
co-authored by Bugen Zhao
parent 9492362972
commit 1f9dd7900d
16 changed files with 258 additions and 2 deletions
+8
View File
@@ -92,6 +92,7 @@ impl HfSpecialTokens {
pub struct ModelConfig {
model_type: Option<String>,
max_position_embeddings: Option<u32>,
vocab_size: Option<u32>,
num_attention_heads: Option<u32>,
num_experts: Option<OneOrManyExpertCount>,
moe_num_experts: Option<OneOrManyExpertCount>,
@@ -179,6 +180,13 @@ impl ModelConfig {
self.model_type.as_deref().or_else(|| self.text_config.as_deref()?.model_type())
}
/// Return the effective model vocabulary size, following the same simplified
/// text-config selection as `model_type`: the top-level config wins,
/// otherwise a single nested `text_config` may provide it.
pub fn vocab_size(&self) -> Option<u32> {
self.vocab_size.or_else(|| self.text_config.as_deref()?.vocab_size())
}
/// Reject partially nested `text_config` payloads that are unlikely to be
/// valid LLM configs for our current use.
///
+4
View File
@@ -100,6 +100,10 @@ impl TextBackend for HfTextBackend {
self.model_config.is_moe()
}
fn model_vocab_size(&self) -> Option<usize> {
self.model_config.vocab_size().map(|v| v as usize)
}
fn model_id(&self) -> &str {
&self.model_id
}
+12
View File
@@ -41,6 +41,18 @@ pub trait TextBackend: Send + Sync {
fn sampling_hints(&self) -> Result<SamplingHints> {
Ok(SamplingHints::default())
}
/// Return the model vocabulary size from the model config, if known. Used to
/// range-check request token ids against the engine embedding table.
fn model_vocab_size(&self) -> Option<usize> {
None
}
/// Return the full tokenizer vocabulary size (Python `len(tokenizer)`).
/// Used to range-check `allowed_token_ids` and token-id prompts.
fn tokenizer_vocab_size(&self) -> usize {
self.tokenizer().vocab_size()
}
}
/// Shared trait-object form of [`TextBackend`].
+12
View File
@@ -91,6 +91,18 @@ impl TextLlm {
self.backend.tokenizer()
}
/// Tokenizer vocabulary size (the number of tokens the tokenizer knows),
/// used to bound `allowed_token_ids` like the Python frontend `len(tokenizer)`.
pub fn tokenizer_vocab_size(&self) -> usize {
self.backend.tokenizer_vocab_size()
}
/// Model vocabulary size from the model config, used to bound `logit_bias`
/// keys and token-id prompts against the engine embedding table.
pub fn model_vocab_size(&self) -> Option<usize> {
self.backend.model_vocab_size()
}
/// Tokenize if needed, lower to a generate request, and return the raw
/// token stream.
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {