[ROCm][CI] Add missing quantization methods and fix online quant test failures (#39801)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -54,13 +54,13 @@ MODEL_ARG_EXPTYPES = [
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
None,
|
||||
"awq_marlin" if current_platform.is_cuda() else "awq",
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "awq",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
"marlin",
|
||||
"awq_marlin" if current_platform.is_cuda() else "ERROR",
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
|
||||
]
|
||||
|
||||
@@ -43,7 +43,7 @@ TEST_CONFIGS = {
|
||||
"amd/Qwen3-8B-WMXFP4FP8-AMXFP4FP8-AMP-KVFP8": {"arc_challenge": 0.52, "mmlu": 0.72},
|
||||
# Non-mixed-precision (PTQ) model
|
||||
# - Reference for pipeline compatibility verification -> No conflicts or breakings
|
||||
"amd/Llama-2-70b-chat-hf-FP8-MLPerf-fp8_attn_quark_format": {
|
||||
"amd/Llama-2-70b-chat-hf_FP8_MLPerf_V2": {
|
||||
"arc_challenge": 0.53,
|
||||
"mmlu": 0.61,
|
||||
},
|
||||
|
||||
@@ -352,7 +352,10 @@ def generate_rotation_matrix(d: int, seed: int, device: str = "cpu") -> torch.Te
|
||||
gen = torch.Generator(device="cpu")
|
||||
gen.manual_seed(seed)
|
||||
G = torch.randn(d, d, generator=gen, device="cpu", dtype=torch.float32)
|
||||
Q, R = torch.linalg.qr(G)
|
||||
# torch.linalg.qr on CPU requires LAPACK, which some torch wheels
|
||||
# (ROCm) ship without. Run QR on accelerator instead
|
||||
qr_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
Q, R = torch.linalg.qr(G.to(qr_device))
|
||||
diag_sign = torch.sign(torch.diag(R))
|
||||
diag_sign[diag_sign == 0] = 1.0
|
||||
Q = Q * diag_sign.unsqueeze(0)
|
||||
|
||||
@@ -80,14 +80,13 @@ def _test_online_quant_peak_mem_impl(
|
||||
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
|
||||
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
|
||||
|
||||
# model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant
|
||||
# uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)
|
||||
expected_model_memory_gib = 6.7
|
||||
|
||||
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
|
||||
# GiB, which is 1.36x above model_memory_gib. A slightly higher number is
|
||||
# expected as when we load and quantize weights in a streaming fashion we
|
||||
# need to have individual weights in bf16 + fp8 alive at the same time.
|
||||
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
|
||||
# number is expected as when we load and quantize weights in a streaming
|
||||
# fashion we need to have individual weights in bf16 + fp8 alive at the
|
||||
# same time.
|
||||
expected_peak_memory_gib = expected_model_memory_gib * 1.4
|
||||
|
||||
assert model_memory_gib < expected_model_memory_gib, (
|
||||
|
||||
Reference in New Issue
Block a user