🐛 fix torch memory profiling (#9516)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
This commit is contained in:
@@ -54,16 +54,17 @@ def test_gpu_memory_profiling():
|
||||
gpu_blocks, _ = worker.determine_num_available_blocks()
|
||||
|
||||
# Peak vram usage by torch should be 0.7077 GiB
|
||||
# Non-torch allocations should be 0.0079 GiB
|
||||
# No memory should be allocated outside of torch
|
||||
# 9.0 GiB should be the utilization target
|
||||
# 8.2843 GiB should be available for the KV cache
|
||||
# 8.2923 GiB should be available for the KV cache
|
||||
block_size = CacheEngine.get_cache_block_size(
|
||||
engine_config.cache_config, engine_config.model_config,
|
||||
engine_config.parallel_config)
|
||||
|
||||
expected_blocks = (8.2843 * 1024**3) // block_size
|
||||
expected_blocks = (8.2923 * 1024**3) // block_size
|
||||
|
||||
# Check within a small tolerance for portability
|
||||
# Hardware, kernel, or dependency changes could all affect memory
|
||||
# utilization
|
||||
assert abs(gpu_blocks - expected_blocks) < 5
|
||||
# utilization.
|
||||
# A 10 block tolerance here should be about 6MB of wiggle room.
|
||||
assert abs(gpu_blocks - expected_blocks) < 10
|
||||
|
||||
Reference in New Issue
Block a user