[ROCm] [Bugfix] Enable Load and Inference of GLM-5.3-Flash Quark MXFP4 Checkpoint (#56176)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Colin Z
2026-09-16 01:25:29 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 1fd119def5
commit c8d1cf077a
8 changed files with 164 additions and 4 deletions
+145
View File
@@ -1668,3 +1668,148 @@ def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected):
)
is expected
)
_GLM5_MXFP4_WEIGHT = {
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
}
_GLM5_BLOCK_FP8_WEIGHT = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
_GLM5_BLOCK_FP8_INPUT = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
_GLM5_GATE_UP = "language_model.model.layers.0.mlp.gate_up_proj"
def _glm5_mixed_precision_config() -> QuarkConfig:
"""Global MXFP4 with the dense-MLP gate/up shards marked block-FP8."""
fp8 = {"weight": _GLM5_BLOCK_FP8_WEIGHT, "input_tensors": _GLM5_BLOCK_FP8_INPUT}
return QuarkConfig(
{
"global_quant_config": {
"weight": _GLM5_MXFP4_WEIGHT,
"input_tensors": None,
},
"layer_type_quant_config": {},
"layer_quant_config": {"*mlp.gate_proj": fp8, "*mlp.up_proj": fp8},
"exclude": [],
}
)
class _GLM5RecordingParam:
"""Stand-in for a BF16 target param that records what gets loaded."""
def __init__(self):
self.loaded_weight: torch.Tensor | None = None
self.loaded_shard: object = "unset"
def weight_loader(self, param, loaded_weight, shard_id=None):
self.loaded_weight = loaded_weight
self.loaded_shard = shard_id
def _glm5_block_fp8(out_dim: int, in_dim: int, block: int = 128):
"""Return a (fp8 weight, f32 per-block scale) pair."""
weight = (torch.randn(out_dim, in_dim) * 0.1).to(torch.float8_e4m3fn)
scale = torch.rand(out_dim // block, in_dim // block, dtype=torch.float32) + 0.5
return weight, scale
def test_glm5next_gate_up_proj_mapping_is_genuine_fusion():
from vllm.models.glm5next.common.model import Glm5NextForConditionalGeneration
mapping = Glm5NextForConditionalGeneration.packed_modules_mapping
assert mapping["gate_up_proj"] == ["gate_proj", "up_proj"]
def test_glm5next_genuine_fusion_resolves_gate_up_to_block_fp8():
# The fused module must expand to its real shard names so each resolves to
# the per-layer block-FP8 entry (rather than the global MXFP4 scheme).
config = _glm5_mixed_precision_config()
config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
_, _, scheme_cls = config.get_scheme_cls(LinearBase, _GLM5_GATE_UP)
assert scheme_cls is QuarkW8A8Fp8PerBlock
def test_glm5next_attn_loader_accepts_quark_weight_scale():
# Regression for KeyError on '...kv_a_proj_with_mqa.weight_scale': the fused
# q_a/kv_a projection is kept BF16 and dequantized on load; the Quark scale
# name must be recognized and routed to the correct fused shard.
from vllm.models.glm5next.common.model import (
_dequant_fp8_block,
_try_load_fp8_attn_proj,
)
prefix = "layers.3.self_attn"
target = _GLM5RecordingParam()
params_dict = {f"{prefix}.fused_qkv_a_proj.weight": target}
buf: dict = {}
loaded: set = set()
weight, scale = _glm5_block_fp8(256, 256)
# fp8 weight arrives first -> buffered, nothing loaded yet.
assert (
_try_load_fp8_attn_proj(
f"{prefix}.kv_a_proj_with_mqa.weight", weight, buf, params_dict, loaded, 0
)
is True
)
assert target.loaded_weight is None
# Quark-named scale completes the pair -> dequantize + load.
assert (
_try_load_fp8_attn_proj(
f"{prefix}.kv_a_proj_with_mqa.weight_scale",
scale,
buf,
params_dict,
loaded,
0,
)
is True
)
assert target.loaded_weight is not None
assert target.loaded_weight.dtype == torch.bfloat16
assert target.loaded_shard == 1 # kv_a is shard 1 of fused_qkv_a_proj
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
assert f"{prefix}.fused_qkv_a_proj.weight" in loaded
def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
# DeepSeek scale name keeps working unchanged.
from vllm.models.glm5next.common.model import (
_dequant_fp8_block,
_try_load_fp8_attn_proj,
)
prefix = "layers.7.self_attn"
target = _GLM5RecordingParam()
params_dict = {f"{prefix}.o_proj.weight": target}
buf: dict = {}
loaded: set = set()
weight, scale = _glm5_block_fp8(128, 256)
_try_load_fp8_attn_proj(
f"{prefix}.o_proj.weight", weight, buf, params_dict, loaded, 0
)
assert (
_try_load_fp8_attn_proj(
f"{prefix}.o_proj.weight_scale_inv", scale, buf, params_dict, loaded, 0
)
is True
)
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))