[ROCm][CI] Add missing quantization methods and fix online quant test failures (#39801)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-04-27 15:08:57 -05:00
committed by GitHub
parent c8bbe05189
commit 5e2c37facd
9 changed files with 50 additions and 23 deletions
+4 -5
View File
@@ -80,14 +80,13 @@ def _test_online_quant_peak_mem_impl(
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
# model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant
# uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)
expected_model_memory_gib = 6.7
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
# GiB, which is 1.36x above model_memory_gib. A slightly higher number is
# expected as when we load and quantize weights in a streaming fashion we
# need to have individual weights in bf16 + fp8 alive at the same time.
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
# number is expected as when we load and quantize weights in a streaming
# fashion we need to have individual weights in bf16 + fp8 alive at the
# same time.
expected_peak_memory_gib = expected_model_memory_gib * 1.4
assert model_memory_gib < expected_model_memory_gib, (