Allow users to specify kv cache memory size (#21489)
Signed-off-by: Boyuan Feng <boyuan@meta.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
fd1ce98cdd
commit
94e6b2d55f
@@ -2791,7 +2791,10 @@ def memory_profiling(
|
||||
result.torch_peak_increase = diff_profile.torch_peak
|
||||
result.non_torch_increase = diff_from_create.non_torch_memory
|
||||
result.profile_time = diff_profile.timestamp
|
||||
result.non_kv_cache_memory = result.non_torch_increase + result.torch_peak_increase + result.weights_memory # noqa
|
||||
|
||||
non_torch_memory = result.non_torch_increase
|
||||
peak_activation_memory = result.torch_peak_increase
|
||||
result.non_kv_cache_memory = non_torch_memory + peak_activation_memory + result.weights_memory # noqa
|
||||
|
||||
|
||||
# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 # noqa: E501
|
||||
|
||||
Reference in New Issue
Block a user