Allow users to specify kv cache memory size (#21489)
Signed-off-by: Boyuan Feng <boyuan@meta.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
fd1ce98cdd
commit
94e6b2d55f
@@ -1337,8 +1337,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
return self.lora_manager.list_adapters()
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
|
||||
"""Cuda graph capture a model.
|
||||
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> int:
|
||||
"""Cuda graph capture a model and return cudagraph memory
|
||||
consumption in bytes.
|
||||
|
||||
Note that CUDA graph's performance gain is negligible if number
|
||||
of batched tokens are larger than 200. And since CUDA graph
|
||||
@@ -1505,6 +1506,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
# This usually takes < 10 seconds.
|
||||
logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
|
||||
elapsed_time, cuda_graph_size / GiB_bytes)
|
||||
return cuda_graph_size
|
||||
|
||||
def _update_inputs_to_capture_for_enc_dec_model(self,
|
||||
capture_inputs: Dict[str,
|
||||
|
||||
Reference in New Issue
Block a user