[torch.compile] use empty tensor instead of None for profiling (#8875)

This commit is contained in:
youkaichao
2024-09-27 08:11:32 -07:00
committed by GitHub
parent 8df2dc3c88
commit a9b15c606f
15 changed files with 84 additions and 32 deletions
+2 -2
View File
@@ -714,7 +714,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
t: torch.Tensor,
p: torch.Tensor,
num_samples: int,
kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
) -> torch.Tensor:
"""Executes the forward pass of the model and samples the next token.
@@ -745,7 +745,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
)
# Skip this in memory profiling at initialization.
if kv_caches[0][0] is not None:
if kv_caches[0][0].numel() > 0:
# index_copy_(slot_mapping) only works when the inserted dimension
# is 0. However, the KV cache in the Pallas backend has the shape
# [num_kv_heads, num_blocks, block_size, head_size]. To make it