[ROCm] [Bugfix] Enable Load and Inference of GLM-5.3-Flash Quark MXFP4 Checkpoint (#56176)
Signed-off-by: Colin Zeng <Colin.Zeng@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
1fd119def5
commit
c8d1cf077a
@@ -6,8 +6,8 @@ from vllm.platforms import current_platform
|
||||
if current_platform.is_xpu():
|
||||
raise NotImplementedError("GLM-5.3-Flash does not currently support XPU.")
|
||||
|
||||
from .nvidia.model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
|
||||
from .nvidia.mtp import Glm5NextMTP
|
||||
from .common.model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
|
||||
from .common.mtp import Glm5NextMTP
|
||||
|
||||
__all__ = [
|
||||
"Glm5NextForCausalLM",
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1036,6 +1036,17 @@ class Glm5NextForConditionalGeneration(
|
||||
has_inner_state: ClassVar[Literal[True]] = True
|
||||
is_hybrid: ClassVar[Literal[True]] = True
|
||||
|
||||
# GLM-5.3-Flash stores the dense-MLP gate/up as separate tensors (like
|
||||
# ``Glm4vMoeForConditionalGeneration``, ``glm4_moe`` and ``deepseek_v2``),
|
||||
# so the fused ``gate_up_proj`` must expand to its real shard names for
|
||||
# per-layer quant-scheme resolution. The identity ``gate_up_proj`` entry
|
||||
# inherited from ``Glm4vForConditionalGeneration`` (pre-fused gate_up_proj)
|
||||
# would otherwise route the module to ``global_quant_config`` and mismatch
|
||||
# at load for mixed-precision Quark checkpoints.
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
# NOTE: weight-prefix mapping is inherited from Glm4vForConditionalGeneration
|
||||
# (``model.visual.`` -> ``visual.``, ``model.language_model.`` ->
|
||||
# ``language_model.model.``, ``lm_head.`` -> ``language_model.lm_head.``),
|
||||
@@ -1259,7 +1270,9 @@ def _try_load_fp8_attn_proj(
|
||||
return False
|
||||
suffix, (key, target_base, shard_id, is_kva) = matched
|
||||
is_weight = name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn
|
||||
is_scale = "weight_scale_inv" in name
|
||||
# Need to accept both the DeepSeek-native ``weight_scale_inv`` and the Quark
|
||||
# ``weight_scale`` names before feeding the shared block dequant below.
|
||||
is_scale = "weight_scale_inv" in name or name.endswith(".weight_scale")
|
||||
if not is_weight and not is_scale:
|
||||
return False
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.glm5next.nvidia.ops.fused_eh_norm import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -33,7 +34,6 @@ from .model import (
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from .ops.fused_eh_norm import fused_eh_norm
|
||||
|
||||
|
||||
class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
||||
Reference in New Issue
Block a user