Remove hardcoded device="cuda" to support more devices (#2503)

Co-authored-by: Jiang Li <jiang1.li@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Kunshang Ji
2024-02-01 15:46:39 -08:00
committed by GitHub
co-authored by Jiang Li
parent c410f5d020
commit 96b6f475dd
32 changed files with 343 additions and 292 deletions
+2 -1
View File
@@ -228,7 +228,8 @@ def create_kv_caches_with_random(
device: Optional[str] = "cuda",
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
torch.random.manual_seed(seed)
torch.cuda.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
if isinstance(cache_dtype, str):
if cache_dtype == "auto":