[Core][Optimization] change python dict to pytorch tensor (#4607)
This commit is contained in:
@@ -248,9 +248,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
|
||||
|
||||
def cache_copy(
|
||||
self,
|
||||
blocks_to_copy: Dict[int, List[int]],
|
||||
blocks_to_copy: torch.Tensor,
|
||||
) -> None:
|
||||
if blocks_to_copy:
|
||||
if blocks_to_copy.numel() > 0:
|
||||
self.cache_engine.copy(blocks_to_copy)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -269,6 +269,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
|
||||
num_seq_groups: int = len(seq_group_metadata_list)
|
||||
assert execute_model_req is not None
|
||||
blocks_to_copy = execute_model_req.blocks_to_copy
|
||||
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
|
||||
device="cpu",
|
||||
dtype=torch.int64).view(-1, 2)
|
||||
assert len(execute_model_req.blocks_to_swap_in) == 0
|
||||
assert len(execute_model_req.blocks_to_swap_out) == 0
|
||||
data: Dict[str, Any] = {
|
||||
|
||||
Reference in New Issue
Block a user