[v1] fix use compile sizes (#11000)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
@@ -582,6 +582,9 @@ class GPUModelRunner:
|
||||
# can reuse the memory pool allocated for the large shapes.
|
||||
with graph_capture():
|
||||
for num_tokens in reversed(self.cudagraph_batch_sizes):
|
||||
for _ in range(self.vllm_config.compilation_config.
|
||||
cudagraph_num_of_warmups):
|
||||
self._dummy_run(self.model, num_tokens, self.kv_caches)
|
||||
self._dummy_run(self.model, num_tokens, self.kv_caches)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user