Add pt_load_map_location to allow loading to cuda (#16869)
Signed-off-by: Jerry Zhang <jerryzh168@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import importlib.metadata
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
@@ -21,5 +22,30 @@ def test_pre_quantized_model(vllm_runner):
|
||||
print(output)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.parametrize(
|
||||
"pt_load_map_location",
|
||||
[
|
||||
"cuda:0",
|
||||
# {"": "cuda"},
|
||||
])
|
||||
def test_opt_125m_int4wo_model_loading_with_params(vllm_runner,
|
||||
pt_load_map_location):
|
||||
"""
|
||||
Test loading roberta-base model with no lm_head.
|
||||
"""
|
||||
torch._dynamo.reset()
|
||||
model_name = "jerryzh168/opt-125m-int4wo"
|
||||
with vllm_runner(model_name=model_name,
|
||||
quantization="torchao",
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=pt_load_map_location) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"],
|
||||
max_tokens=32)
|
||||
|
||||
assert output
|
||||
print(output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
|
||||
+15
-1
@@ -5,7 +5,8 @@ from typing import Literal, Union
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig, config, get_field
|
||||
from vllm.config import (LoadConfig, ModelConfig, PoolerConfig, VllmConfig,
|
||||
config, get_field)
|
||||
from vllm.model_executor.layers.pooler import PoolingType
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -410,3 +411,16 @@ def test_generation_config_loading():
|
||||
override_generation_config=override_generation_config)
|
||||
|
||||
assert model_config.get_diff_sampling_param() == override_generation_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pt_load_map_location", [
|
||||
"cuda",
|
||||
{
|
||||
"": "cuda"
|
||||
},
|
||||
])
|
||||
def test_load_config_pt_load_map_location(pt_load_map_location):
|
||||
load_config = LoadConfig(pt_load_map_location=pt_load_map_location)
|
||||
config = VllmConfig(load_config=load_config)
|
||||
|
||||
assert config.load_config.pt_load_map_location == pt_load_map_location
|
||||
|
||||
Reference in New Issue
Block a user