[Core][Optimization] change python dict to pytorch tensor (#4607)

This commit is contained in:
youkaichao
2024-05-06 21:30:27 -07:00
committed by GitHub
parent a98187cf72
commit 63575bc2e1
19 changed files with 77 additions and 81 deletions
+5 -2
View File
@@ -248,9 +248,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
def cache_copy(
self,
blocks_to_copy: Dict[int, List[int]],
blocks_to_copy: torch.Tensor,
) -> None:
if blocks_to_copy:
if blocks_to_copy.numel() > 0:
self.cache_engine.copy(blocks_to_copy)
@torch.inference_mode()
@@ -269,6 +269,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
num_seq_groups: int = len(seq_group_metadata_list)
assert execute_model_req is not None
blocks_to_copy = execute_model_req.blocks_to_copy
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
device="cpu",
dtype=torch.int64).view(-1, 2)
assert len(execute_model_req.blocks_to_swap_in) == 0
assert len(execute_model_req.blocks_to_swap_out) == 0
data: Dict[str, Any] = {