[Bugfix][multi_modal] Fix pos_ids being unitialized for minicpmv2.6 in hf runner (#51432)
Signed-off-by: Dino Music <Dino.Music@amd.com>
This commit is contained in:
@@ -922,6 +922,22 @@ def _internvl_generate(
|
||||
return outputs
|
||||
|
||||
|
||||
def _restore_resampler_pos_cache(hf_model: HfRunner) -> None:
|
||||
"""Recompute the resampler's 2D sin/cos position cache after loading.
|
||||
|
||||
`from_pretrained` materializes non-persistent buffers as uninitialized
|
||||
memory, leaving `pos_embed` as zeros or NaN instead of the values computed
|
||||
in `__init__`.
|
||||
"""
|
||||
restored = 0
|
||||
for module in hf_model.model.modules():
|
||||
if not hasattr(module, "_set_2d_pos_cache"):
|
||||
continue
|
||||
module._set_2d_pos_cache(module.max_size, module.pos_embed.device)
|
||||
restored += 1
|
||||
assert restored, "no resampler pos cache found to restore"
|
||||
|
||||
|
||||
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
orig_generate = hf_model.model.generate
|
||||
|
||||
@@ -965,6 +981,8 @@ def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
|
||||
def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
_restore_resampler_pos_cache(hf_model)
|
||||
|
||||
orig_generate = hf_model.model.generate
|
||||
|
||||
def _generate(self, *args, image_sizes=None, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user