[Rust Frontend] Pass vision preprocessing context for Nemotron-H (#57634)
Pass the remaining engine context-length budget to model-owned vision processors through VisionPreprocessingContext. Preserve Nemotron batched engine fields and recognize llm_config as a text_config alias. Use the merged upstream llm-multimodal revision f0985ef65967615db2c79279aa07818499301bfd. Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Generated
+1
-1
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
||||
[[package]]
|
||||
name = "llm-multimodal"
|
||||
version = "1.7.1"
|
||||
source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8"
|
||||
source = "git+https://github.com/smg-project/llm-multimodal?rev=f0985ef65967615db2c79279aa07818499301bfd#f0985ef65967615db2c79279aa07818499301bfd"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ indexmap = "2.13.0"
|
||||
indicatif = "0.18.4"
|
||||
itertools = "0.14.0"
|
||||
libc = "0.2.177"
|
||||
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] }
|
||||
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] }
|
||||
mimalloc = "0.1.52"
|
||||
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
|
||||
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
|
||||
|
||||
@@ -72,6 +72,9 @@ pub struct ChatRequestProcessor {
|
||||
/// Effective model dtype reported by the engine.
|
||||
/// Absent for text-only frontends without an engine handshake.
|
||||
model_dtype: Option<ModelDtype>,
|
||||
/// Effective runtime context length reported by the engine.
|
||||
/// Absent for render-only frontends without an engine handshake.
|
||||
max_model_len: Option<usize>,
|
||||
/// Tool-call parser selection used when preparing generation requests.
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection used when preparing generation requests.
|
||||
@@ -83,10 +86,11 @@ pub struct ChatRequestProcessor {
|
||||
impl ChatRequestProcessor {
|
||||
/// Create a processor with multimodal support using the effective model
|
||||
/// dtype reported by the engine.
|
||||
fn new(backend: DynChatBackend, model_dtype: ModelDtype) -> Self {
|
||||
fn new(backend: DynChatBackend, model_dtype: ModelDtype, max_model_len: u32) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
model_dtype: Some(model_dtype),
|
||||
max_model_len: Some(max_model_len as usize),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
@@ -98,6 +102,7 @@ impl ChatRequestProcessor {
|
||||
Self {
|
||||
backend,
|
||||
model_dtype: None,
|
||||
max_model_len: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
@@ -137,6 +142,7 @@ impl ChatRequestProcessor {
|
||||
rendered,
|
||||
self.backend.multimodal_model_info(),
|
||||
model_dtype,
|
||||
self.max_model_len,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -161,7 +167,8 @@ impl ChatRequestProcessor {
|
||||
let features = match media {
|
||||
multimodal::MultimodalInput::Raw(parts) => {
|
||||
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
info.prepare_multimodal(parts, token_ids, model_dtype).await?
|
||||
info.prepare_multimodal(parts, token_ids, model_dtype, self.max_model_len)
|
||||
.await?
|
||||
}
|
||||
multimodal::MultimodalInput::Preprocessed(features) => {
|
||||
info.prepare_preprocessed(features, token_ids.len())?
|
||||
@@ -256,10 +263,11 @@ impl ChatLlm {
|
||||
/// backend.
|
||||
pub fn new(text: TextLlm, backend: DynChatBackend) -> Self {
|
||||
let model_dtype = text.engine_core_client().model_dtype();
|
||||
let max_model_len = text.engine_core_client().max_model_len();
|
||||
|
||||
Self {
|
||||
text,
|
||||
processor: ChatRequestProcessor::new(backend, model_dtype),
|
||||
processor: ChatRequestProcessor::new(backend, model_dtype, max_model_len),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ use llm_multimodal::{
|
||||
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
|
||||
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
|
||||
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
|
||||
VisionPreprocessingContext,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -55,6 +56,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer};
|
||||
#[derive(Clone)]
|
||||
pub struct MultimodalModelInfo {
|
||||
context: MultimodalModelContext,
|
||||
/// Rendered placeholder marker IDs for all resolved modalities.
|
||||
placeholder_token_ids: Vec<u32>,
|
||||
image: Option<VisionModalitySupport>,
|
||||
video: Option<VisionModalitySupport>,
|
||||
audio: Option<AudioModalitySupport>,
|
||||
@@ -420,8 +423,18 @@ impl MultimodalModelInfo {
|
||||
MediaConnectorConfig::default(),
|
||||
)?);
|
||||
|
||||
let placeholder_token_ids = [
|
||||
image.as_ref().map(|support| support.placeholder.marker_token_id),
|
||||
video.as_ref().map(|support| support.placeholder.marker_token_id),
|
||||
audio.as_ref().map(|support| support.placeholder.marker_token_id),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
|
||||
Ok(Some(Self {
|
||||
context,
|
||||
placeholder_token_ids,
|
||||
image,
|
||||
video,
|
||||
audio,
|
||||
@@ -547,6 +560,20 @@ impl MultimodalModelInfo {
|
||||
Modality::ImageEmbeds => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn vision_preprocessing_context(
|
||||
&self,
|
||||
prompt_token_ids: &[u32],
|
||||
max_model_len: Option<usize>,
|
||||
) -> VisionPreprocessingContext {
|
||||
let text_prompt_length = prompt_token_ids
|
||||
.iter()
|
||||
.filter(|token_id| !self.placeholder_token_ids.contains(token_id))
|
||||
.count();
|
||||
VisionPreprocessingContext {
|
||||
token_budget: max_model_len.map(|limit| limit.saturating_sub(text_prompt_length)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Finalize a rendered chat prompt into text-generation input.
|
||||
@@ -559,6 +586,7 @@ pub(crate) async fn finalize_rendered_prompt(
|
||||
rendered: RenderedPrompt,
|
||||
info: Option<&MultimodalModelInfo>,
|
||||
model_dtype: ModelDtype,
|
||||
max_model_len: Option<usize>,
|
||||
) -> Result<(Prompt, Option<MmFeatures>)> {
|
||||
let media_parts = extract_media_parts(request, rendered.media_order.as_deref())?;
|
||||
if media_parts.is_empty() {
|
||||
@@ -574,7 +602,12 @@ pub(crate) async fn finalize_rendered_prompt(
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
let prepared = info
|
||||
.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype)
|
||||
.prepare_multimodal(
|
||||
media_parts,
|
||||
&mut prompt_token_ids,
|
||||
model_dtype,
|
||||
max_model_len,
|
||||
)
|
||||
.instrument(mm_request_span(&request.request_id))
|
||||
.await?;
|
||||
|
||||
@@ -768,11 +801,13 @@ impl MultimodalModelInfo {
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
max_model_len: Option<usize>,
|
||||
) -> Result<MmFeatures> {
|
||||
if media_parts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await
|
||||
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype, max_model_len)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
|
||||
@@ -788,15 +823,24 @@ impl MultimodalModelInfo {
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
max_model_len: Option<usize>,
|
||||
) -> Result<MmFeatures> {
|
||||
let media_parts_len = media_parts.len();
|
||||
self.validate_mm_limits(&media_parts)?;
|
||||
let vision_context = self.vision_preprocessing_context(prompt_token_ids, max_model_len);
|
||||
let fetched = self.fetch_media(media_parts).await?;
|
||||
|
||||
let mut prepared = Vec::new();
|
||||
if !fetched.images.is_empty() {
|
||||
prepared
|
||||
.push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
|
||||
prepared.push(
|
||||
self.prepare_images(
|
||||
fetched.images,
|
||||
fetched.image_uuids,
|
||||
model_dtype,
|
||||
vision_context,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
if !fetched.videos.is_empty() {
|
||||
prepared
|
||||
@@ -1044,6 +1088,100 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vision_preprocessing_context_uses_runtime_budget_and_text_tokens() {
|
||||
let info = qwen3_vl_info();
|
||||
let context = info.vision_preprocessing_context(
|
||||
&[11, QWEN3_IMAGE_PAD_ID, 12, QWEN3_VIDEO_PAD_ID, 13],
|
||||
Some(4096),
|
||||
);
|
||||
|
||||
assert_eq!(context.token_budget, Some(4096 - 3));
|
||||
assert_eq!(
|
||||
info.vision_preprocessing_context(&[11, 12, 13], Some(2)).token_budget,
|
||||
Some(0)
|
||||
);
|
||||
assert_eq!(
|
||||
info.vision_preprocessing_context(&[11], None).token_budget,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nemotron_budget_controls_preprocessing_and_engine_features() {
|
||||
use vllm_engine_core_client::protocol::multimodal::MmKwargValue;
|
||||
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_regular_token("<image>", 1018)
|
||||
.with_regular_token("<img>", 1019)
|
||||
.with_regular_token("</img>", 1020);
|
||||
let config = serde_json::json!({
|
||||
"model_type": "nemotron_h_omni",
|
||||
"img_context_token_id": 1018,
|
||||
"max_position_embeddings": 512,
|
||||
"patch_size": 16,
|
||||
"downsample_ratio": 0.5,
|
||||
"norm_mean": [0.0, 0.0, 0.0],
|
||||
"norm_std": [1.0, 1.0, 1.0],
|
||||
"vision_config": {
|
||||
"args": {"min_num_patches": 1024, "max_num_patches": 13312}
|
||||
}
|
||||
});
|
||||
let info = test_info("nemotron_h_omni", config, tokenizer);
|
||||
let media = MediaContentPart::ImageUrl {
|
||||
url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=".to_string(),
|
||||
detail: None,
|
||||
uuid: None,
|
||||
};
|
||||
let mut prompt = vec![11; 100];
|
||||
prompt.push(1018);
|
||||
|
||||
let mut tokens = prompt.clone();
|
||||
let features = info
|
||||
.prepare_multimodal(
|
||||
vec![media.clone()],
|
||||
&mut tokens,
|
||||
ModelDtype::Float32,
|
||||
Some(512),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(features.len(), 1);
|
||||
let feature = &features[0];
|
||||
assert_eq!(
|
||||
(feature.mm_position.offset, feature.mm_position.length),
|
||||
(100, 258)
|
||||
);
|
||||
assert_eq!(tokens.len(), 358);
|
||||
assert_eq!(tokens[100], 1019);
|
||||
assert!(tokens[101..357].iter().all(|&token| token == 1018));
|
||||
assert_eq!(tokens[357], 1020);
|
||||
let data = feature.data.as_ref().unwrap();
|
||||
assert!(matches!(
|
||||
data["pixel_values_flat"].data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor)) if tensor.shape == [3, 512, 512]
|
||||
));
|
||||
assert_eq!(
|
||||
data["num_tokens_per_image"].data,
|
||||
Some(MmKwargValue::Int(256))
|
||||
);
|
||||
assert_eq!(
|
||||
data["imgs_sizes"].data,
|
||||
Some(MmKwargValue::List(vec![
|
||||
MmKwargValue::Int(512),
|
||||
MmKwargValue::Int(512),
|
||||
]))
|
||||
);
|
||||
|
||||
let error = info
|
||||
.prepare_multimodal(vec![media], &mut prompt, ModelDtype::Float32, Some(100))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, Error::Multimodal(message) if message.contains("exceeding token_budget"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_vl_without_video_token_id_disables_video_support_only() {
|
||||
let config = serde_json::json!({
|
||||
|
||||
@@ -259,7 +259,7 @@ mod tests {
|
||||
uuid: Some("audio-1".to_string()),
|
||||
}];
|
||||
async {
|
||||
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32)
|
||||
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32, None)
|
||||
.instrument(mm_request_span("request-1"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
|
||||
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs, VisionPreprocessingContext};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
|
||||
use super::timing::MM_STAGE_TARGET;
|
||||
@@ -30,11 +30,12 @@ impl MultimodalModelInfo {
|
||||
frames: Vec<Arc<ImageFrame>>,
|
||||
uuids: Vec<Option<String>>,
|
||||
model_dtype: ModelDtype,
|
||||
preprocessing_context: VisionPreprocessingContext,
|
||||
) -> Result<PreparedMedia> {
|
||||
let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality {
|
||||
modality: Modality::Image.to_string(),
|
||||
})?;
|
||||
let preprocessed = self.preprocess_images(support, &frames).await?;
|
||||
let preprocessed = self.preprocess_images(support, &frames, preprocessing_context).await?;
|
||||
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
|
||||
if replacements.len() != frames.len() {
|
||||
bail_multimodal!(
|
||||
@@ -65,13 +66,16 @@ impl MultimodalModelInfo {
|
||||
&self,
|
||||
support: &VisionModalitySupport,
|
||||
image_frames: &[Arc<ImageFrame>],
|
||||
preprocessing_context: VisionPreprocessingContext,
|
||||
) -> Result<PreprocessedEncoderInputs> {
|
||||
let processor = Arc::clone(&support.processor);
|
||||
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
|
||||
|
||||
// TODO: is it still necessary given that we've already in a dedicated runtime?
|
||||
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?))
|
||||
.await
|
||||
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
||||
tokio::task::spawn_blocking(move || {
|
||||
Ok(processor.preprocess_with_context(&images, &preprocessing_context)?)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
pub(super) enum KwargValue {
|
||||
/// Tensor with row-major flat data and shape.
|
||||
Tensor { wire: WireTensor },
|
||||
/// Non-tensor kwarg value that is shared or copied as-is.
|
||||
/// Non-tensor kwarg value preserved in protocol form.
|
||||
Passthrough(ProtocolKwargValue),
|
||||
}
|
||||
|
||||
@@ -150,7 +150,17 @@ impl KwargValue {
|
||||
/// Batched fields use their first axis as media-item index and drop that
|
||||
/// axis in the per-feature value, matching vLLM's batched-field semantics.
|
||||
pub(super) fn batched_wire_value_at(&self, index: usize) -> Result<ProtocolKwargValue> {
|
||||
self.wire_value_range(index, index + 1, true)
|
||||
match self {
|
||||
Self::Passthrough(ProtocolKwargValue::List(values)) => values
|
||||
.get(index)
|
||||
.cloned()
|
||||
.ok_or_else(|| multimodal!("batched passthrough has no item at index {index}")),
|
||||
Self::Passthrough(value) if index == 0 => Ok(value.clone()),
|
||||
Self::Passthrough(_) => Err(multimodal!(
|
||||
"scalar batched passthrough has no item at index {index}"
|
||||
)),
|
||||
Self::Tensor { .. } => self.wire_value_range(index, index + 1, true),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one media item's flat tensor range directly to wire bytes.
|
||||
@@ -336,6 +346,41 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batched_passthrough_value_at_selects_media_item() {
|
||||
let image_sizes = KwargValue::from_model_specific(
|
||||
ModelSpecificValue::TupleVec(vec![(384, 736), (512, 512)]),
|
||||
ModelDtype::BFloat16,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
image_sizes.batched_wire_value_at(1).unwrap(),
|
||||
ProtocolKwargValue::List(vec![
|
||||
ProtocolKwargValue::Int(512),
|
||||
ProtocolKwargValue::Int(512),
|
||||
])
|
||||
);
|
||||
assert!(image_sizes.batched_wire_value_at(2).is_err());
|
||||
|
||||
let token_counts = KwargValue::from_model_specific(
|
||||
ModelSpecificValue::IntVec(vec![276, 400]),
|
||||
ModelDtype::BFloat16,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
token_counts.batched_wire_value_at(1).unwrap(),
|
||||
ProtocolKwargValue::Int(400)
|
||||
);
|
||||
|
||||
let scalar = KwargValue::Passthrough(ProtocolKwargValue::Int(7));
|
||||
assert_eq!(
|
||||
scalar.batched_wire_value_at(0).unwrap(),
|
||||
ProtocolKwargValue::Int(7)
|
||||
);
|
||||
assert!(scalar.batched_wire_value_at(1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_wire_value_range_keeps_first_axis() {
|
||||
let value = KwargValue::Tensor {
|
||||
|
||||
@@ -84,10 +84,11 @@ impl HfSpecialTokens {
|
||||
|
||||
/// Minimal subset of `config.json` (the model's main HF config).
|
||||
///
|
||||
/// This intentionally supports only the two layouts we currently care about in
|
||||
/// This intentionally supports only the layouts we currently care about in
|
||||
/// the Rust frontend:
|
||||
/// - pure text models that keep text metadata at the top level
|
||||
/// - composite models that expose a single nested `text_config`
|
||||
/// - composite models that expose a single nested `text_config` or its
|
||||
/// `llm_config` alias (used by Nemotron)
|
||||
///
|
||||
/// We do not support additional entry points such as `decoder`, `generator`, or
|
||||
/// `text_encoder`.
|
||||
@@ -102,6 +103,7 @@ pub struct ModelConfig {
|
||||
n_routed_experts: Option<OneOrManyExpertCount>,
|
||||
num_local_experts: Option<OneOrManyExpertCount>,
|
||||
block_configs: Vec<BlockConfig>,
|
||||
#[serde(alias = "llm_config")]
|
||||
text_config: Option<Box<ModelConfig>>,
|
||||
}
|
||||
|
||||
@@ -188,7 +190,7 @@ impl ModelConfig {
|
||||
/// Return the config that the Rust frontend treats as the text/LLM config.
|
||||
///
|
||||
/// This is deliberately narrower than Python/transformers: we only support
|
||||
/// either the top-level config itself or a single nested `text_config`.
|
||||
/// either the top-level config itself or a single nested text config.
|
||||
fn effective_text_config(&self) -> &Self {
|
||||
self.text_config.as_deref().unwrap_or(self)
|
||||
}
|
||||
@@ -233,7 +235,8 @@ impl ModelConfig {
|
||||
/// config.
|
||||
///
|
||||
/// The only intentional simplification here is how we pick the text config:
|
||||
/// Rust only looks at the top level or `text_config`, not the broader
|
||||
/// Rust only looks at the top level or `text_config` (including its
|
||||
/// `llm_config` alias), not the broader
|
||||
/// transformers composite-config surface.
|
||||
fn num_experts_from_block_configs(&self) -> u32 {
|
||||
self.effective_text_config()
|
||||
@@ -405,6 +408,27 @@ mod tests {
|
||||
assert_eq!(config.vocab_size().unwrap(), 151936);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_uses_llm_config_for_nemotron_composite_models() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"model_type": "nemotron_h_omni",
|
||||
"llm_config": {
|
||||
"model_type": "nemotron_h",
|
||||
"vocab_size": 131072,
|
||||
"n_routed_experts": 256,
|
||||
"eos_token_id": 2
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.model_type(), Some("nemotron_h_omni"));
|
||||
assert_eq!(config.vocab_size().unwrap(), 131072);
|
||||
assert_eq!(config.eos_token_ids(), &[2]);
|
||||
assert_eq!(config.num_experts(), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_reads_top_level_eos_token_ids() {
|
||||
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user