[Chore] Use Transformers v5 names and drop redundant processor use_fast (#58550)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
484c211fe8
commit
d881812500
@@ -9,7 +9,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from torch.nn import LayerNorm
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
|
||||
from vllm.third_party.deep_gemm import per_block_cast_to_fp8
|
||||
@@ -27,7 +27,7 @@ from .vision_attention import (
|
||||
from .vision_moe import note_vision_fused_moe_fp8
|
||||
|
||||
|
||||
class DotsMoEVitConfig(PretrainedConfig):
|
||||
class DotsMoEVitConfig(PreTrainedConfig):
|
||||
model_type: str = "dots_moe_vit"
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -30,7 +30,7 @@ from itertools import islice
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -75,7 +75,7 @@ from ..nvidia.moe import HYV4FeedForward, HYV4MoEFused
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig:
|
||||
def _normalize_hyv4_config(config: PreTrainedConfig) -> PreTrainedConfig:
|
||||
"""Populate the aliases consumed by the shared MoE implementation."""
|
||||
config.router_scaling_factor = config.routed_scaling_factor
|
||||
config.num_experts = config.n_routed_experts
|
||||
@@ -95,7 +95,7 @@ class HYV4DecoderLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import cast
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config, PretrainedConfig
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config, PreTrainedConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -51,7 +51,7 @@ _SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention"
|
||||
_WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
|
||||
|
||||
|
||||
def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]:
|
||||
def compute_skip_topk_layers(config: PreTrainedConfig) -> set[int]:
|
||||
"""Return the backbone layers that reuse a previous layer's top-k indices.
|
||||
|
||||
A "shared" indexer layer performs sparse attention with the indices computed
|
||||
@@ -280,7 +280,7 @@ class HYV4MLAAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
|
||||
@@ -17,7 +17,7 @@ requires restructuring the decoder-layer forward scheduling.
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.hpc import HpcIHCHead, HpcIHCPost, HpcIHCPre
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
@@ -40,7 +40,7 @@ class HYV4HCPreLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
hidden_dim: int,
|
||||
hc_mult: int = 4,
|
||||
magnitude: float = 2.0,
|
||||
@@ -163,7 +163,7 @@ class HYV4HCPostLayer(nn.Module):
|
||||
y[n, i, d] = post[n, i] * x[n, d] + residual[n, i, d]
|
||||
"""
|
||||
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
def __init__(self, config: PreTrainedConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
@@ -213,7 +213,7 @@ class HYV4HCHeadLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
hidden_size: int,
|
||||
hc_mult: int = 4,
|
||||
hc_eps: float = 1e-6,
|
||||
@@ -301,7 +301,7 @@ class HYV4HCLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_idx: int,
|
||||
init_std: float = 6e-3,
|
||||
base_noise_std: float = 0.0,
|
||||
|
||||
@@ -30,7 +30,7 @@ from itertools import islice
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -74,7 +74,7 @@ from .moe import HYV4FeedForward, HYV4MoEFused
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig:
|
||||
def _normalize_hyv4_config(config: PreTrainedConfig) -> PreTrainedConfig:
|
||||
"""Populate the aliases consumed by the shared MoE implementation."""
|
||||
config.router_scaling_factor = config.routed_scaling_factor
|
||||
config.num_experts = config.n_routed_experts
|
||||
@@ -94,7 +94,7 @@ class HYV4DecoderLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
|
||||
@@ -87,7 +87,7 @@ class HYV4MoEFused(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
enable_eplb: bool = False,
|
||||
|
||||
@@ -8,7 +8,7 @@ from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -59,7 +59,7 @@ _MTP_QUANT_EXCLUSION_ATTRS = ("ignored_layers", "exclude_modules")
|
||||
|
||||
|
||||
def _get_spec_layer_idx_from_weight_name(
|
||||
config: PretrainedConfig, weight_name: str
|
||||
config: PreTrainedConfig, weight_name: str
|
||||
) -> int | None:
|
||||
"""Return the MTP layer index a checkpoint weight belongs to, or None.
|
||||
|
||||
@@ -147,7 +147,7 @@ def _prepare_mtp_fp8_expert_scale(
|
||||
|
||||
|
||||
def _create_mtp_quant_config(
|
||||
hf_config: PretrainedConfig,
|
||||
hf_config: PreTrainedConfig,
|
||||
backbone_quant_config: QuantizationConfig | None = None,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Create the quantization config for the MTP layers.
|
||||
@@ -303,7 +303,7 @@ class HYV4SharedHead(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -329,7 +329,7 @@ class HYV4MultiTokenPredictorLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
prefix: str,
|
||||
vllm_config: VllmConfig,
|
||||
model_config: ModelConfig,
|
||||
|
||||
@@ -8,10 +8,10 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, ClassVar, Literal, cast
|
||||
|
||||
import torch
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class InklingModelConfig(PretrainedConfig):
|
||||
class InklingModelConfig(PreTrainedConfig):
|
||||
model_type = "inkling_model"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
@@ -215,7 +215,7 @@ class InklingModelConfig(PretrainedConfig):
|
||||
return TMLConvCacheParams(shape=shape, layers=self.conv_layer_ids, dtype=dtype)
|
||||
|
||||
|
||||
class InklingAudioConfig(PretrainedConfig):
|
||||
class InklingAudioConfig(PreTrainedConfig):
|
||||
model_type = "inkling_audio_model"
|
||||
|
||||
def __init__(
|
||||
@@ -255,7 +255,7 @@ class InklingAudioConfig(PretrainedConfig):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class InklingVisionConfig(PretrainedConfig):
|
||||
class InklingVisionConfig(PreTrainedConfig):
|
||||
model_type = "inkling_vision_model"
|
||||
|
||||
def __init__(
|
||||
@@ -295,10 +295,10 @@ class InklingVisionConfig(PretrainedConfig):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class InklingMMConfig(PretrainedConfig):
|
||||
class InklingMMConfig(PreTrainedConfig):
|
||||
model_type = "inkling_mm_model"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = {
|
||||
sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
|
||||
"text_config": InklingModelConfig,
|
||||
"audio_config": InklingAudioConfig,
|
||||
"vision_config": InklingVisionConfig,
|
||||
|
||||
@@ -21,7 +21,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
@@ -134,7 +134,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
def _sparse_attention_layer_ids(config: PreTrainedConfig) -> set[int]:
|
||||
"""Layer ids whose attention runs the extra sparse "index" branch."""
|
||||
cfg = getattr(config, "sparse_attention_config", None)
|
||||
if not cfg:
|
||||
@@ -145,7 +145,7 @@ def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
return {i for i, f in enumerate(freq) if f != 0}
|
||||
|
||||
|
||||
def _sparse_attention_layer_ordinals(config: PretrainedConfig) -> dict[int, int]:
|
||||
def _sparse_attention_layer_ordinals(config: PreTrainedConfig) -> dict[int, int]:
|
||||
"""Map each sparse-attention layer id to its ordinal among sparse layers."""
|
||||
return {
|
||||
lid: ordinal
|
||||
@@ -153,7 +153,7 @@ def _sparse_attention_layer_ordinals(config: PretrainedConfig) -> dict[int, int]
|
||||
}
|
||||
|
||||
|
||||
def _should_skip_index_topk(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
def _should_skip_index_topk(config: PreTrainedConfig, layer_id: int) -> bool:
|
||||
"""ATOM ``index_topk_freq`` (cross-layer index sharing).
|
||||
|
||||
Only 1 of every ``index_topk_freq`` sparse-attention layers recomputes the
|
||||
@@ -176,7 +176,7 @@ def _should_skip_index_topk(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
return max(ordinal - offset, 0) % freq != 0
|
||||
|
||||
|
||||
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
def _is_moe_layer(config: PreTrainedConfig, layer_id: int) -> bool:
|
||||
"""Whether this layer's MLP is a sparse MoE block (vs a dense MLP)."""
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
if moe_layer_freq is None:
|
||||
@@ -184,7 +184,7 @@ def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
return moe_layer_freq[layer_id] != 0
|
||||
|
||||
|
||||
def _build_rotary_emb(config: PretrainedConfig, head_dim: int):
|
||||
def _build_rotary_emb(config: PreTrainedConfig, head_dim: int):
|
||||
"""Build the (partial NeoX) RoPE, honoring an optional ``rope_scaling`` config.
|
||||
|
||||
Without scaling the cos/sin cache is sized to ``max_position_embeddings``
|
||||
@@ -262,7 +262,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
intermediate_size: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
@@ -338,7 +338,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -488,7 +488,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -623,7 +623,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -1264,7 +1264,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
class MiniMaxM3DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
prefix: str,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
@@ -1738,7 +1738,7 @@ class MiniMaxM3SparseForConditionalGeneration(
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
vision_config = config.vision_config
|
||||
self.vision_tower = MiniMaxVLVisionModel(
|
||||
config=PretrainedConfig.from_dict(vision_config),
|
||||
config=PreTrainedConfig.from_dict(vision_config),
|
||||
text_hidden_size=text_hidden_size,
|
||||
projector_hidden_size=projector_hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.distributed import utils as dist_utils
|
||||
@@ -44,7 +44,7 @@ class MiniMaxVLPatchEmbed(nn.Module):
|
||||
and projects each to a hidden-size embedding.
|
||||
"""
|
||||
|
||||
def __init__(self, config: PretrainedConfig) -> None:
|
||||
def __init__(self, config: PreTrainedConfig) -> None:
|
||||
super().__init__()
|
||||
compression = config.img_token_compression_config
|
||||
temporal_patch_size = compression.get("temporal_patch_size", 2)
|
||||
@@ -232,7 +232,7 @@ class MiniMaxVLEncoderLayer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
@@ -295,7 +295,7 @@ class MiniMaxVLEncoderLayer(nn.Module):
|
||||
class MiniMaxVLEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
num_hidden_layers_override: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -349,7 +349,7 @@ class MiniMaxVLVisionTransformer(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
num_hidden_layers_override: int | None = None,
|
||||
require_post_norm: bool | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
@@ -679,7 +679,7 @@ class MiniMaxVLVisionModel(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
text_hidden_size: int,
|
||||
projector_hidden_size: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
|
||||
@@ -16,7 +16,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
@@ -92,7 +92,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
|
||||
|
||||
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
def _sparse_attention_layer_ids(config: PreTrainedConfig) -> set[int]:
|
||||
"""Layer ids whose attention runs the extra sparse "index" branch."""
|
||||
cfg = getattr(config, "sparse_attention_config", None)
|
||||
if not cfg:
|
||||
@@ -103,7 +103,7 @@ def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
return {i for i, f in enumerate(freq) if f != 0}
|
||||
|
||||
|
||||
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
def _is_moe_layer(config: PreTrainedConfig, layer_id: int) -> bool:
|
||||
"""Whether this layer's MLP is a sparse MoE block (vs a dense MLP)."""
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
if moe_layer_freq is None:
|
||||
@@ -147,7 +147,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
intermediate_size: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
@@ -194,7 +194,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
@@ -290,7 +290,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -403,7 +403,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
config: PreTrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -1130,7 +1130,7 @@ class MiniMaxM3SparseForConditionalGeneration(
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
vision_config = config.vision_config
|
||||
self.vision_tower = MiniMaxVLVisionModel(
|
||||
config=PretrainedConfig.from_dict(vision_config),
|
||||
config=PreTrainedConfig.from_dict(vision_config),
|
||||
text_hidden_size=text_hidden_size,
|
||||
projector_hidden_size=projector_hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
|
||||
Qwen3VLVisionConfig,
|
||||
)
|
||||
@@ -201,9 +201,9 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
return max(int(self.ngram_size) - 1, 0)
|
||||
|
||||
|
||||
class Qwen4ExpConfig(PretrainedConfig):
|
||||
class Qwen4ExpConfig(PreTrainedConfig):
|
||||
model_type = "qwen4_exp"
|
||||
sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = {
|
||||
sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
|
||||
"vision_config": Qwen4ExpVisionConfig,
|
||||
"text_config": Qwen4ExpTextConfig,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user