diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index bd4b908dc6..f52587ae81 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -216,7 +216,7 @@ class NemotronHMultiTokenPredictor(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_config.get_text_config() self.config = config self.vocab_size = config.vocab_size @@ -322,7 +322,7 @@ class NemotronHMTP(nn.Module, SupportsPP): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_config.get_text_config() self.vllm_config = vllm_config self.config = config self.quant_config = vllm_config.quant_config @@ -414,6 +414,10 @@ class NemotronHMTP(nn.Module, SupportsPP): loaded_params: set[str] = set() for name, loaded_weight in weights: + # MTP weights are nested in "language_model." + # in Multimodal Nemotron-H checkpoints. + name = name.removeprefix("language_model.") + # Only process MTP weights - skip all non-MTP weights if not name.startswith("mtp.") and "embeddings" not in name: continue diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 7a67a00084..0eaba01294 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1403,6 +1403,10 @@ class SpecDecodeBaseProposer: self.model.config.image_token_index = ( target_model.config.media_placeholder_token_id ) + elif self.get_model_name(target_model) == "NemotronH_Nano_VL_V2": + self.model.config.image_token_index = ( + target_model.config.img_context_token_id + ) else: self.model.config.image_token_index = ( target_model.config.image_token_index