[Bugfix][Rust Frontend] Validate out-of-vocab token ids in request params (#44680)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -92,6 +92,7 @@ impl HfSpecialTokens {
|
||||
pub struct ModelConfig {
|
||||
model_type: Option<String>,
|
||||
max_position_embeddings: Option<u32>,
|
||||
vocab_size: Option<u32>,
|
||||
num_attention_heads: Option<u32>,
|
||||
num_experts: Option<OneOrManyExpertCount>,
|
||||
moe_num_experts: Option<OneOrManyExpertCount>,
|
||||
@@ -179,6 +180,13 @@ impl ModelConfig {
|
||||
self.model_type.as_deref().or_else(|| self.text_config.as_deref()?.model_type())
|
||||
}
|
||||
|
||||
/// Return the effective model vocabulary size, following the same simplified
|
||||
/// text-config selection as `model_type`: the top-level config wins,
|
||||
/// otherwise a single nested `text_config` may provide it.
|
||||
pub fn vocab_size(&self) -> Option<u32> {
|
||||
self.vocab_size.or_else(|| self.text_config.as_deref()?.vocab_size())
|
||||
}
|
||||
|
||||
/// Reject partially nested `text_config` payloads that are unlikely to be
|
||||
/// valid LLM configs for our current use.
|
||||
///
|
||||
|
||||
@@ -100,6 +100,10 @@ impl TextBackend for HfTextBackend {
|
||||
self.model_config.is_moe()
|
||||
}
|
||||
|
||||
fn model_vocab_size(&self) -> Option<usize> {
|
||||
self.model_config.vocab_size().map(|v| v as usize)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
&self.model_id
|
||||
}
|
||||
|
||||
@@ -41,6 +41,18 @@ pub trait TextBackend: Send + Sync {
|
||||
fn sampling_hints(&self) -> Result<SamplingHints> {
|
||||
Ok(SamplingHints::default())
|
||||
}
|
||||
|
||||
/// Return the model vocabulary size from the model config, if known. Used to
|
||||
/// range-check request token ids against the engine embedding table.
|
||||
fn model_vocab_size(&self) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the full tokenizer vocabulary size (Python `len(tokenizer)`).
|
||||
/// Used to range-check `allowed_token_ids` and token-id prompts.
|
||||
fn tokenizer_vocab_size(&self) -> usize {
|
||||
self.tokenizer().vocab_size()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared trait-object form of [`TextBackend`].
|
||||
|
||||
@@ -91,6 +91,18 @@ impl TextLlm {
|
||||
self.backend.tokenizer()
|
||||
}
|
||||
|
||||
/// Tokenizer vocabulary size (the number of tokens the tokenizer knows),
|
||||
/// used to bound `allowed_token_ids` like the Python frontend `len(tokenizer)`.
|
||||
pub fn tokenizer_vocab_size(&self) -> usize {
|
||||
self.backend.tokenizer_vocab_size()
|
||||
}
|
||||
|
||||
/// Model vocabulary size from the model config, used to bound `logit_bias`
|
||||
/// keys and token-id prompts against the engine embedding table.
|
||||
pub fn model_vocab_size(&self) -> Option<usize> {
|
||||
self.backend.model_vocab_size()
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and return the raw
|
||||
/// token stream.
|
||||
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {
|
||||
|
||||
Reference in New Issue
Block a user