[torch.compile] use empty tensor instead of None for profiling (#8875)

This commit is contained in:
youkaichao
2024-09-27 08:11:32 -07:00
committed by GitHub
parent 8df2dc3c88
commit a9b15c606f
15 changed files with 84 additions and 32 deletions
+7 -1
View File
@@ -97,7 +97,13 @@ class EmbeddingModelRunner(
model_executable = self.model
num_layers = self.model_config.get_num_layers(self.parallel_config)
kv_caches = [None] * num_layers
# use an empty tensor instead of `None`` to force Dynamo to pass
# it by reference, rather by specializing on the value ``None``.
# the `dtype` argument does not matter, and we use `float32` as
# a placeholder (it has wide hardware support).
kv_caches = [
torch.tensor([], dtype=torch.float32, device=self.device)
] * num_layers
execute_model_kwargs = {
"input_ids":