[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -4,6 +4,7 @@
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
@@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
@@ -221,7 +221,7 @@ class Indexer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
hidden_size: int,
|
||||
q_lora_rank: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
@@ -236,7 +236,7 @@ class Indexer(nn.Module):
|
||||
# self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
|
||||
# Indexer is only constructed for v32 configs, where these sparse-indexer
|
||||
# fields are guaranteed populated; narrow away the `int | None` declared
|
||||
# on Glm5NextConfig for the optional-indexer case.
|
||||
# on Glm5NextTextConfig for the optional-indexer case.
|
||||
assert config.index_topk is not None
|
||||
assert config.index_n_heads is not None
|
||||
assert config.index_head_dim is not None
|
||||
@@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
@@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
# `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`;
|
||||
# a checkpoint may still ship one.
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
|
||||
if not skip_rope:
|
||||
assert config.rope_parameters is not None
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
assert rope_parameters is not None
|
||||
if rope_parameters["rope_type"] != "default":
|
||||
rope_parameters["rope_type"] = (
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
if rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
|
||||
if (
|
||||
config.rope_parameters["rope_type"] != "default"
|
||||
and config.rope_parameters["rope_type"] == "deepseek_yarn"
|
||||
rope_parameters["rope_type"] != "default"
|
||||
and rope_parameters["rope_type"] == "deepseek_yarn"
|
||||
):
|
||||
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = config.rope_parameters["factor"]
|
||||
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
self.scaling = self.scaling * mscale * mscale
|
||||
else:
|
||||
@@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=not config.indexer_rope_interleave,
|
||||
)
|
||||
# The sparse indexer projects from the MLA q-lora rank, which is
|
||||
|
||||
Reference in New Issue
Block a user