[Test] Refactor hard coded device string in test files under compile/quantization/models/model_executor folders (#38901)
Signed-off-by: Liao, Wei <wei.liao@intel.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from huggingface_hub import snapshot_download
|
||||
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
|
||||
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
@@ -15,6 +16,8 @@ from ....conftest import ImageTestAssets
|
||||
# dynamic_module and trust_remote_code for hf_runner
|
||||
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_intern_vit_test(
|
||||
@@ -39,9 +42,9 @@ def run_intern_vit_test(
|
||||
|
||||
hf_model = AutoModel.from_pretrained(
|
||||
model, dtype=torch_dtype, trust_remote_code=True
|
||||
).to("cuda")
|
||||
).to(DEVICE_TYPE)
|
||||
hf_outputs_per_image = [
|
||||
hf_model(pixel_value.to("cuda")).last_hidden_state
|
||||
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
|
||||
@@ -53,9 +56,10 @@ def run_intern_vit_test(
|
||||
del hf_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
vllm_model = vllm_model.to("cuda", torch_dtype)
|
||||
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
|
||||
vllm_outputs_per_image = [
|
||||
vllm_model(pixel_values=pixel_value.to("cuda")) for pixel_value in pixel_values
|
||||
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
del vllm_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
@@ -8,6 +8,7 @@ from transformers import AutoConfig, AutoModel, CLIPImageProcessor
|
||||
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.models.radio import RadioModel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.radio import RadioConfig
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
@@ -17,6 +18,8 @@ from ....conftest import ImageTestAssets
|
||||
# dynamic_module and trust_remote_code for hf_runner
|
||||
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_radio_test(
|
||||
@@ -51,7 +54,7 @@ def run_radio_test(
|
||||
config=hf_config,
|
||||
dtype=torch_dtype,
|
||||
trust_remote_code=True,
|
||||
).to("cuda")
|
||||
).to(DEVICE_TYPE)
|
||||
hf_model.eval()
|
||||
|
||||
# A HF model has image normalization as a part of model's forward
|
||||
@@ -62,7 +65,7 @@ def run_radio_test(
|
||||
hf_model.make_preprocessor_external()
|
||||
|
||||
hf_outputs_per_image = [
|
||||
hf_model(pixel_value.to("cuda")) for pixel_value in pixel_values
|
||||
hf_model(pixel_value.to(DEVICE_TYPE)) for pixel_value in pixel_values
|
||||
]
|
||||
|
||||
vllm_config = RadioConfig(
|
||||
@@ -71,10 +74,11 @@ def run_radio_test(
|
||||
)
|
||||
vllm_model = RadioModel(vllm_config)
|
||||
vllm_model.load_weights(hf_model.state_dict())
|
||||
vllm_model = vllm_model.to("cuda", torch_dtype)
|
||||
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
|
||||
|
||||
vllm_outputs_per_image = [
|
||||
vllm_model(pixel_values=pixel_value.to("cuda")) for pixel_value in pixel_values
|
||||
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
del vllm_model, hf_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
@@ -10,6 +10,8 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
class ModuleWithBatchNorm(torch.nn.Module):
|
||||
def __init__(self):
|
||||
@@ -174,8 +176,12 @@ class raise_if_cuda_sync:
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
def test_merge_multimodal_embeddings_no_sync():
|
||||
inputs_embeds = torch.zeros([5, 10], dtype=torch.bfloat16, device="cuda:0")
|
||||
multimodal_embeddings = [torch.ones([3, 10], dtype=torch.bfloat16, device="cuda:0")]
|
||||
inputs_embeds = torch.zeros(
|
||||
[5, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0"
|
||||
)
|
||||
multimodal_embeddings = [
|
||||
torch.ones([3, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0")
|
||||
]
|
||||
is_multimodal = torch.tensor([True, False, True, True, False], device="cpu")
|
||||
with raise_if_cuda_sync():
|
||||
_merge_multimodal_embeddings(
|
||||
|
||||
Reference in New Issue
Block a user