Remove hardcoded device="cuda" to support more devices (#2503)
Co-authored-by: Jiang Li <jiang1.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -126,8 +126,8 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
|
||||
cleanup()
|
||||
get_model_old = get_model
|
||||
|
||||
def get_model_patched(model_config, lora_config=None):
|
||||
return get_model_old(model_config,
|
||||
def get_model_patched(model_config, device_config, lora_config=None):
|
||||
return get_model_old(model_config, device_config,
|
||||
LoRAConfig(max_loras=4, max_lora_rank=8))
|
||||
|
||||
with patch("vllm.worker.model_runner.get_model", get_model_patched):
|
||||
|
||||
Reference in New Issue
Block a user