diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 7bee4615aa..a4c58c1215 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -922,6 +922,22 @@ def _internvl_generate( return outputs +def _restore_resampler_pos_cache(hf_model: HfRunner) -> None: + """Recompute the resampler's 2D sin/cos position cache after loading. + + `from_pretrained` materializes non-persistent buffers as uninitialized + memory, leaving `pos_embed` as zeros or NaN instead of the values computed + in `__init__`. + """ + restored = 0 + for module in hf_model.model.modules(): + if not hasattr(module, "_set_2d_pos_cache"): + continue + module._set_2d_pos_cache(module.max_size, module.pos_embed.device) + restored += 1 + assert restored, "no resampler pos cache found to restore" + + def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner: orig_generate = hf_model.model.generate @@ -965,6 +981,8 @@ def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner: def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner: + _restore_resampler_pos_cache(hf_model) + orig_generate = hf_model.model.generate def _generate(self, *args, image_sizes=None, **kwargs):