[Bugfix][multi_modal] Fix pos_ids being unitialized for minicpmv2.6 in hf runner (#51432)
Signed-off-by: Dino Music <Dino.Music@amd.com>
This commit is contained in:
@@ -922,6 +922,22 @@ def _internvl_generate(
|
|||||||
return outputs
|
return outputs
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_resampler_pos_cache(hf_model: HfRunner) -> None:
|
||||||
|
"""Recompute the resampler's 2D sin/cos position cache after loading.
|
||||||
|
|
||||||
|
`from_pretrained` materializes non-persistent buffers as uninitialized
|
||||||
|
memory, leaving `pos_embed` as zeros or NaN instead of the values computed
|
||||||
|
in `__init__`.
|
||||||
|
"""
|
||||||
|
restored = 0
|
||||||
|
for module in hf_model.model.modules():
|
||||||
|
if not hasattr(module, "_set_2d_pos_cache"):
|
||||||
|
continue
|
||||||
|
module._set_2d_pos_cache(module.max_size, module.pos_embed.device)
|
||||||
|
restored += 1
|
||||||
|
assert restored, "no resampler pos cache found to restore"
|
||||||
|
|
||||||
|
|
||||||
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||||
orig_generate = hf_model.model.generate
|
orig_generate = hf_model.model.generate
|
||||||
|
|
||||||
@@ -965,6 +981,8 @@ def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
|||||||
|
|
||||||
|
|
||||||
def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||||
|
_restore_resampler_pos_cache(hf_model)
|
||||||
|
|
||||||
orig_generate = hf_model.model.generate
|
orig_generate = hf_model.model.generate
|
||||||
|
|
||||||
def _generate(self, *args, image_sizes=None, **kwargs):
|
def _generate(self, *args, image_sizes=None, **kwargs):
|
||||||
|
|||||||
Reference in New Issue
Block a user