[Misc][LoRA] Replace hardcoded cuda device with configurable argument (#10223)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
+36
-20
@@ -51,6 +51,7 @@ TOLERANCES = {
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
# We will launch different triton kernels between the prefill and decode
|
||||
# stages, so we need to verify this. prefill stage(True) or decode stage(False)
|
||||
STAGES = [True, False]
|
||||
@@ -120,11 +121,12 @@ def populate_loras(
|
||||
subloras: List[LoRALayerWeights] = []
|
||||
sublora_len = layer_weights.shape[0] // repeats
|
||||
for i in range(repeats):
|
||||
sublora = DummyLoRAManager().init_random_lora(
|
||||
module_name=f"fake_{i}",
|
||||
weight=layer_weights,
|
||||
generate_embeddings_tensor=generate_embeddings_tensor,
|
||||
)
|
||||
sublora = DummyLoRAManager(
|
||||
layer_weights.device).init_random_lora(
|
||||
module_name=f"fake_{i}",
|
||||
weight=layer_weights,
|
||||
generate_embeddings_tensor=generate_embeddings_tensor,
|
||||
)
|
||||
sublora.lora_b = sublora.lora_b[:, (sublora_len *
|
||||
i):(sublora_len * (i + 1))]
|
||||
sublora.optimize()
|
||||
@@ -152,6 +154,7 @@ def create_random_inputs(
|
||||
input_size: Tuple[int, ...],
|
||||
input_range: Tuple[float, float],
|
||||
input_type: torch.dtype = torch.int,
|
||||
device: torch.device = "cuda"
|
||||
) -> Tuple[List[torch.Tensor], List[int], List[int]]:
|
||||
"""Creates random inputs.
|
||||
|
||||
@@ -173,10 +176,14 @@ def create_random_inputs(
|
||||
for _ in range(num_inputs):
|
||||
if input_type == torch.int:
|
||||
inputs.append(
|
||||
torch.randint(low=int(low), high=int(high), size=input_size))
|
||||
torch.randint(low=int(low),
|
||||
high=int(high),
|
||||
size=input_size,
|
||||
device=device))
|
||||
else:
|
||||
inputs.append(
|
||||
torch.rand(size=input_size, dtype=input_type) * high + low)
|
||||
torch.rand(size=input_size, dtype=input_type, device=device) *
|
||||
high + low)
|
||||
|
||||
lora_id = random.choice(active_lora_ids)
|
||||
index_mapping += [lora_id] * input_size[0]
|
||||
@@ -191,6 +198,10 @@ def create_random_inputs(
|
||||
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
|
||||
@pytest.mark.parametrize("stage", STAGES)
|
||||
def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
|
||||
# For multi-GPU testing of Triton kernel, we must explicitly set the CUDA
|
||||
# device, see: https://github.com/triton-lang/triton/issues/2925
|
||||
# Same below.
|
||||
torch.cuda.set_device(device)
|
||||
|
||||
torch.set_default_device(device)
|
||||
max_loras = 8
|
||||
@@ -225,7 +236,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
|
||||
num_inputs=num_loras * 3,
|
||||
input_size=(200, ),
|
||||
input_range=(1, vocab_size),
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -263,7 +274,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
|
||||
num_inputs=num_loras * 3,
|
||||
input_size=(200, ),
|
||||
input_range=(1, vocab_size),
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -291,6 +302,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
|
||||
def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
|
||||
vocab_size, stage) -> None:
|
||||
|
||||
torch.cuda.set_device(device)
|
||||
torch.set_default_device(device)
|
||||
max_loras = 8
|
||||
punica_wrapper = PunicaWrapper(8192, 256, device)
|
||||
@@ -345,7 +357,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
|
||||
num_inputs=num_loras * 3,
|
||||
input_size=(200, ),
|
||||
input_range=(1, vocab_size),
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -400,7 +412,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
|
||||
num_inputs=num_loras * 3,
|
||||
input_size=(200, ),
|
||||
input_range=(1, vocab_size),
|
||||
)
|
||||
device=device)
|
||||
original_inputs = deepcopy(inputs)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
@@ -426,6 +438,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
|
||||
def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
|
||||
stage) -> None:
|
||||
|
||||
torch.cuda.set_device(device)
|
||||
torch.set_default_device(device)
|
||||
max_loras = 8
|
||||
punica_wrapper = PunicaWrapper(8192, 256, device)
|
||||
@@ -471,7 +484,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
|
||||
input_size=(1, 1024),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -520,7 +533,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
|
||||
input_size=(1, 1024),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -554,6 +567,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
|
||||
@pytest.mark.parametrize("stage", STAGES)
|
||||
def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
|
||||
|
||||
torch.cuda.set_device(device)
|
||||
torch.set_default_device(device)
|
||||
punica_wrapper = PunicaWrapper(8192, 256, device)
|
||||
max_loras = 8
|
||||
@@ -592,7 +606,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -631,7 +645,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -658,6 +672,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
|
||||
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
|
||||
device, stage) -> None:
|
||||
|
||||
torch.cuda.set_device(device)
|
||||
torch.set_default_device(device)
|
||||
punica_wrapper = PunicaWrapper(8192, 256, device)
|
||||
max_loras = 8
|
||||
@@ -706,7 +721,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -745,7 +760,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -772,6 +787,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
|
||||
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
|
||||
device, stage) -> None:
|
||||
|
||||
torch.cuda.set_device(device)
|
||||
torch.set_default_device(device)
|
||||
punica_wrapper = PunicaWrapper(8192, 256, device)
|
||||
max_loras = 8
|
||||
@@ -842,7 +858,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -883,7 +899,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
|
||||
input_size=(1, 4096),
|
||||
input_range=(0, 1),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
lora_mapping = LoRAMapping(index_mapping,
|
||||
prompt_mapping,
|
||||
is_prefill=stage)
|
||||
@@ -962,7 +978,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
|
||||
input_size=(1, max_position),
|
||||
input_range=(0, lora_config.lora_extra_vocab_size),
|
||||
input_type=torch.float16,
|
||||
)
|
||||
device=device)
|
||||
|
||||
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
|
||||
long_lora_context = LongContextLoRAContext(list(scaling_factors),
|
||||
|
||||
+113
-40
@@ -25,8 +25,13 @@ EMBEDDING_MODULES = {
|
||||
|
||||
EMBEDDING_PADDING_MODULES = ["lm_head"]
|
||||
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
def test_from_lora_tensors(sql_lora_files):
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_from_lora_tensors(sql_lora_files, device):
|
||||
tensors = load_file(
|
||||
os.path.join(sql_lora_files, "adapter_model.safetensors"))
|
||||
new_embeddings = load_file(
|
||||
@@ -36,7 +41,7 @@ def test_from_lora_tensors(sql_lora_files):
|
||||
8,
|
||||
16,
|
||||
tensors,
|
||||
"cuda",
|
||||
device,
|
||||
embeddings=new_embeddings,
|
||||
embedding_modules=EMBEDDING_MODULES,
|
||||
embedding_padding_modules=EMBEDDING_PADDING_MODULES)
|
||||
@@ -46,6 +51,8 @@ def test_from_lora_tensors(sql_lora_files):
|
||||
assert lora.lora_alpha == 16
|
||||
assert lora.lora_a is not None
|
||||
assert lora.lora_b is not None
|
||||
assert lora.lora_a.device == torch.device(device)
|
||||
assert lora.lora_b.device == torch.device(device)
|
||||
assert (lora.lora_a.shape[1] == lora.lora_b.shape[0]
|
||||
), f"{lora.lora_a.shape=}, {lora.lora_b.shape=}"
|
||||
assert lora.lora_a.shape[1] == 8
|
||||
@@ -60,8 +67,8 @@ def test_from_lora_tensors(sql_lora_files):
|
||||
assert lora.embeddings_tensor is None
|
||||
|
||||
|
||||
def create_lora(lora_id: int, model: nn.Module,
|
||||
sub_modules: List[str]) -> LoRAModel:
|
||||
def create_lora(lora_id: int, model: nn.Module, sub_modules: List[str],
|
||||
device: torch.device) -> LoRAModel:
|
||||
loras: Dict[str, LoRALayerWeights] = {}
|
||||
for name in sub_modules:
|
||||
w = model.get_submodule(name).weight
|
||||
@@ -69,8 +76,8 @@ def create_lora(lora_id: int, model: nn.Module,
|
||||
name,
|
||||
8,
|
||||
16,
|
||||
torch.rand([w.shape[1], 8], device="cuda"),
|
||||
torch.rand([8, w.shape[0]], device="cuda"),
|
||||
torch.rand([w.shape[1], 8], device=device),
|
||||
torch.rand([8, w.shape[0]], device=device),
|
||||
)
|
||||
return LoRAModel(lora_id, 8, loras)
|
||||
|
||||
@@ -80,6 +87,7 @@ def create_packed_lora(
|
||||
model: nn.Module,
|
||||
module_name,
|
||||
replaced_module_names,
|
||||
device: torch.device,
|
||||
empty_replaced_module_name=None,
|
||||
) -> LoRAModel:
|
||||
w = model.get_submodule(module_name).weight
|
||||
@@ -91,9 +99,9 @@ def create_packed_lora(
|
||||
replaced_module_name,
|
||||
8,
|
||||
16,
|
||||
torch.rand([w.shape[1], 8], device="cuda"),
|
||||
torch.rand([w.shape[1], 8], device=device),
|
||||
torch.rand([8, w.shape[0] // len(replaced_module_names)],
|
||||
device="cuda"),
|
||||
device=device),
|
||||
)
|
||||
return LoRAModel(lora_id, 8, loras)
|
||||
|
||||
@@ -104,7 +112,8 @@ def test_replace_submodules(dist_init, dummy_model):
|
||||
model.packed_modules_mapping = {}
|
||||
manager = LoRAModelManager(
|
||||
model, 1, 1, 1,
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8))
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
|
||||
torch.device("cuda"))
|
||||
model = manager.model
|
||||
|
||||
assert isinstance(model.get_submodule("dense1"),
|
||||
@@ -116,16 +125,28 @@ def test_replace_submodules(dist_init, dummy_model):
|
||||
RowParallelLinearWithLoRA)
|
||||
|
||||
|
||||
def test_lora_model_manager(dist_init, dummy_model):
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_lora_model_manager(dist_init, dummy_model, device):
|
||||
model = dummy_model
|
||||
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
|
||||
model.packed_modules_mapping = {}
|
||||
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
|
||||
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
|
||||
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
|
||||
manager = LoRAModelManager(
|
||||
model, 2, 2, 2,
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
|
||||
model_lora1 = create_lora(1,
|
||||
model, ["layer1.dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora2 = create_lora(2,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora3 = create_lora(3,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
manager = LoRAModelManager(model,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
LoRAConfig(max_lora_rank=8,
|
||||
max_cpu_loras=3,
|
||||
max_loras=2),
|
||||
device=device)
|
||||
assert all(x is None for x in manager.lora_index_to_id)
|
||||
assert manager.add_adapter(model_lora1)
|
||||
assert manager.activate_adapter(1)
|
||||
@@ -161,17 +182,32 @@ def test_lora_model_manager(dist_init, dummy_model):
|
||||
assert manager.lora_index_to_id[0] == 3
|
||||
assert manager.lora_index_to_id[1] == 2
|
||||
|
||||
assert manager.device == device
|
||||
assert manager.punica_wrapper.device == device
|
||||
|
||||
def test_lora_lru_cache_model_manager(dist_init, dummy_model):
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
|
||||
model = dummy_model
|
||||
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
|
||||
model.packed_modules_mapping = {}
|
||||
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
|
||||
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
|
||||
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
|
||||
manager = LRUCacheLoRAModelManager(
|
||||
model, 2, 2, 2,
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
|
||||
model_lora1 = create_lora(1,
|
||||
model, ["layer1.dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora2 = create_lora(2,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora3 = create_lora(3,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
manager = LRUCacheLoRAModelManager(model,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
LoRAConfig(max_lora_rank=8,
|
||||
max_cpu_loras=3,
|
||||
max_loras=2),
|
||||
device=device)
|
||||
assert all(x is None for x in manager.lora_index_to_id)
|
||||
assert manager.add_adapter(model_lora1)
|
||||
assert manager.activate_adapter(1)
|
||||
@@ -238,20 +274,37 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
|
||||
with pytest.raises(ValueError):
|
||||
assert manager.pin_adapter(3)
|
||||
|
||||
assert manager.punica_wrapper.device == device
|
||||
assert manager.device == device
|
||||
|
||||
def test_lru_lora_model_manager(dist_init, dummy_model):
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_lru_lora_model_manager(dist_init, dummy_model, device):
|
||||
# This tests just the LRU cache functionality, everything else is
|
||||
# tested in test_lora_model_manager
|
||||
model = dummy_model
|
||||
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
|
||||
model.packed_modules_mapping = {}
|
||||
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
|
||||
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
|
||||
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
|
||||
model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
|
||||
manager = LRUCacheLoRAModelManager(
|
||||
model, 2, 2, 2,
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
|
||||
model_lora1 = create_lora(1,
|
||||
model, ["layer1.dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora2 = create_lora(2,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora3 = create_lora(3,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
model_lora4 = create_lora(4,
|
||||
model, ["dense1", "dense2", "lm_head"],
|
||||
device=device)
|
||||
manager = LRUCacheLoRAModelManager(model,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
LoRAConfig(max_lora_rank=8,
|
||||
max_cpu_loras=2,
|
||||
max_loras=2),
|
||||
device=device)
|
||||
|
||||
assert all(x is None for x in manager.lora_index_to_id)
|
||||
|
||||
@@ -351,14 +404,17 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
|
||||
assert manager.remove_oldest_adapter()
|
||||
|
||||
assert set(manager.list_adapters()) == {1}
|
||||
assert manager.punica_wrapper.device == device
|
||||
assert manager.device == device
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
|
||||
sql_lora_files):
|
||||
sql_lora_files, device):
|
||||
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
|
||||
worker_adapter_manager = LRUCacheWorkerLoRAManager(
|
||||
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
|
||||
lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
|
||||
lora_config.lora_extra_vocab_size, lora_config, device,
|
||||
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
|
||||
worker_adapter_manager.create_lora_manager(
|
||||
llama_2_7b_model_extra_embeddings)
|
||||
@@ -426,14 +482,19 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
|
||||
LoRARequest("14", 14, sql_lora_files)
|
||||
], mapping)
|
||||
|
||||
assert worker_adapter_manager.device == device
|
||||
assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
|
||||
device)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
|
||||
sql_lora_files):
|
||||
sql_lora_files, device):
|
||||
# Should remove every LoRA not specified in the request.
|
||||
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
|
||||
worker_adapter_manager = WorkerLoRAManager(
|
||||
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
|
||||
lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
|
||||
lora_config.lora_extra_vocab_size, lora_config, device,
|
||||
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
|
||||
worker_adapter_manager.create_lora_manager(
|
||||
llama_2_7b_model_extra_embeddings)
|
||||
@@ -497,8 +558,13 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
|
||||
LoRARequest("14", 14, sql_lora_files)
|
||||
], mapping)
|
||||
|
||||
assert worker_adapter_manager.device == device
|
||||
assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
|
||||
device)
|
||||
|
||||
def test_packed_loras(dist_init, dummy_model_gate_up):
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_packed_loras(dist_init, dummy_model_gate_up, device):
|
||||
model = dummy_model_gate_up
|
||||
model.supported_lora_modules = ["gate_up_proj"]
|
||||
model.packed_modules_mapping = {
|
||||
@@ -511,18 +577,25 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
|
||||
1,
|
||||
model,
|
||||
module_name="gate_up_proj",
|
||||
replaced_module_names=["gate_proj", "up_proj"])
|
||||
replaced_module_names=["gate_proj", "up_proj"],
|
||||
device=device)
|
||||
model_lora1 = create_packed_lora(
|
||||
2,
|
||||
model,
|
||||
module_name="gate_up_proj",
|
||||
replaced_module_names=["gate_proj", "up_proj"],
|
||||
device=device,
|
||||
empty_replaced_module_name="gate_proj",
|
||||
)
|
||||
|
||||
manager = LoRAModelManager(
|
||||
model, 2, 2, 2,
|
||||
LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
|
||||
manager = LoRAModelManager(model,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
LoRAConfig(max_lora_rank=8,
|
||||
max_cpu_loras=2,
|
||||
max_loras=2),
|
||||
device=device)
|
||||
model = manager.model
|
||||
|
||||
assert isinstance(model.get_submodule("gate_up_proj"),
|
||||
|
||||
+5
-4
@@ -7,9 +7,10 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
|
||||
|
||||
class DummyLoRAManager:
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, device: torch.device = "cuda:0"):
|
||||
super().__init__()
|
||||
self._loras: Dict[str, LoRALayerWeights] = {}
|
||||
self._device = device
|
||||
|
||||
def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
|
||||
self._loras[module_name] = lora
|
||||
@@ -28,16 +29,16 @@ class DummyLoRAManager:
|
||||
lora_alpha=1,
|
||||
lora_a=torch.rand([weight.shape[1], rank],
|
||||
dtype=weight.dtype,
|
||||
device="cuda"),
|
||||
device=self._device),
|
||||
lora_b=torch.rand([rank, weight.shape[0]],
|
||||
dtype=weight.dtype,
|
||||
device="cuda"),
|
||||
device=self._device),
|
||||
)
|
||||
if generate_embeddings_tensor:
|
||||
lora.embeddings_tensor = torch.rand(5,
|
||||
generate_embeddings_tensor,
|
||||
dtype=weight.dtype,
|
||||
device="cuda")
|
||||
device=self._device)
|
||||
self.set_module_lora(module_name, lora)
|
||||
|
||||
return lora
|
||||
|
||||
Reference in New Issue
Block a user