[ROCm] [Bugfix] Enable Load and Inference of GLM-5.3-Flash Quark MXFP4 Checkpoint (#56176)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Colin Z
2026-09-16 01:25:29 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 1fd119def5
commit c8d1cf077a
8 changed files with 164 additions and 4 deletions
+2 -2
View File
@@ -6,8 +6,8 @@ from vllm.platforms import current_platform
if current_platform.is_xpu():
raise NotImplementedError("GLM-5.3-Flash does not currently support XPU.")
from .nvidia.model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
from .nvidia.mtp import Glm5NextMTP
from .common.model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
from .common.mtp import Glm5NextMTP
__all__ = [
"Glm5NextForCausalLM",
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1036,6 +1036,17 @@ class Glm5NextForConditionalGeneration(
has_inner_state: ClassVar[Literal[True]] = True
is_hybrid: ClassVar[Literal[True]] = True
# GLM-5.3-Flash stores the dense-MLP gate/up as separate tensors (like
# ``Glm4vMoeForConditionalGeneration``, ``glm4_moe`` and ``deepseek_v2``),
# so the fused ``gate_up_proj`` must expand to its real shard names for
# per-layer quant-scheme resolution. The identity ``gate_up_proj`` entry
# inherited from ``Glm4vForConditionalGeneration`` (pre-fused gate_up_proj)
# would otherwise route the module to ``global_quant_config`` and mismatch
# at load for mixed-precision Quark checkpoints.
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
}
# NOTE: weight-prefix mapping is inherited from Glm4vForConditionalGeneration
# (``model.visual.`` -> ``visual.``, ``model.language_model.`` ->
# ``language_model.model.``, ``lm_head.`` -> ``language_model.lm_head.``),
@@ -1259,7 +1270,9 @@ def _try_load_fp8_attn_proj(
return False
suffix, (key, target_base, shard_id, is_kva) = matched
is_weight = name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn
is_scale = "weight_scale_inv" in name
# Need to accept both the DeepSeek-native ``weight_scale_inv`` and the Quark
# ``weight_scale`` names before feeding the shared block dequant below.
is_scale = "weight_scale_inv" in name or name.endswith(".weight_scale")
if not is_weight and not is_scale:
return False
@@ -22,6 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.glm5next.nvidia.ops.fused_eh_norm import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -33,7 +34,6 @@ from .model import (
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from .ops.fused_eh_norm import fused_eh_norm
class Glm5NextMultiTokenPredictorLayer(nn.Module):