[Bugfix] unskip InternViT test for transformers v5 compatibility (#55767)
Signed-off-by: sahil <sahil@example.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
874a6b5a06
commit
4e61ad8ee1
@@ -3,7 +3,7 @@
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
|
from transformers import AutoConfig, AutoModel, CLIPImageProcessor, PreTrainedModel
|
||||||
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
from vllm.distributed import cleanup_dist_env_and_memory
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -12,11 +12,6 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
|||||||
|
|
||||||
from ....conftest import ImageTestAssets
|
from ....conftest import ImageTestAssets
|
||||||
|
|
||||||
pytestmark = pytest.mark.skip(
|
|
||||||
reason="InternVisionModel's custom code is incompatible with "
|
|
||||||
"transformers v5 (missing all_tied_weights_keys)"
|
|
||||||
)
|
|
||||||
|
|
||||||
# we use snapshot_download to prevent conflicts between
|
# we use snapshot_download to prevent conflicts between
|
||||||
# dynamic_module and trust_remote_code for hf_runner
|
# dynamic_module and trust_remote_code for hf_runner
|
||||||
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
||||||
@@ -45,9 +40,23 @@ def run_intern_vit_test(
|
|||||||
if not getattr(config, "norm_type", None):
|
if not getattr(config, "norm_type", None):
|
||||||
config.norm_type = "rms_norm"
|
config.norm_type = "rms_norm"
|
||||||
|
|
||||||
hf_model = AutoModel.from_pretrained(
|
# Monkey-patch for transformers v5 compatibility: InternVisionModel's
|
||||||
model, dtype=torch_dtype, trust_remote_code=True
|
# custom code doesn't call post_init(), so all_tied_weights_keys is
|
||||||
).to(DEVICE_TYPE)
|
# never set. Remove once https://github.com/OpenGVLab/InternVL fixes it.
|
||||||
|
_orig_init = PreTrainedModel.__init__
|
||||||
|
|
||||||
|
def _patched_init(self, *args, **kwargs):
|
||||||
|
_orig_init(self, *args, **kwargs)
|
||||||
|
self.all_tied_weights_keys = {}
|
||||||
|
|
||||||
|
PreTrainedModel.__init__ = _patched_init
|
||||||
|
|
||||||
|
try:
|
||||||
|
hf_model = AutoModel.from_pretrained(
|
||||||
|
model, dtype=torch_dtype, trust_remote_code=True
|
||||||
|
).to(DEVICE_TYPE)
|
||||||
|
finally:
|
||||||
|
PreTrainedModel.__init__ = _orig_init
|
||||||
hf_outputs_per_image = [
|
hf_outputs_per_image = [
|
||||||
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
|
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
|
||||||
for pixel_value in pixel_values
|
for pixel_value in pixel_values
|
||||||
|
|||||||
Reference in New Issue
Block a user