[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

requirements/common.txt

@@ -7,7 +7,7 @@ requests >= 2.33.0
 tqdm
 blake3
 py-cpuinfo
-transformers >= 5.10.4
+transformers >= 5.16.1
 huggingface_hub >= 1.31.0  # Required for the `huggingface_hub.utils.httpx` re-export
 tokenizers >= 0.21.1  # Required for fast incremental detokenization.
 safetensors >= 0.6.2  # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611

tests/models/glm5next/test_sequence_parallel.py

@@ -15,11 +15,11 @@ from types import SimpleNamespace
 import pytest
 import torch
 from torch import nn
+from transformers import Glm5NextTextConfig
 
 from vllm.model_executor import parameter
 from vllm.model_executor.layers import linear
 from vllm.models.glm5next.common import model as glm_model
-from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
 
 HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
 

tests/models/multimodal/processing/test_glm5next.py

@@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU.
 import pytest
 import torch
 from PIL import Image
+from transformers.models.glm5_next.video_processing_glm5_next import (
+    Glm5NextVideoProcessor,
+    smart_resize,
+)
+from transformers.video_utils import VideoMetadata
 
 from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.platforms import current_platform
-from vllm.transformers_utils.processors.glm5next import (
-    Glm5NextVideoProcessor,
-    _pixel_budget,
-    glm_sample_frame_indices,
-    smart_resize,
-)
 
 from ...utils import build_model_context
 
@@ -48,27 +47,16 @@ def _pixel_path_grid(
     width: int,
 ) -> tuple[int, int, int]:
     """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
-    min_pixels, max_pixels = _pixel_budget(
-        video_processor.min_image_tokens,
-        video_processor.max_image_tokens,
-        video_processor.patch_size,
-        video_processor.merge_size,
-        video_processor.temporal_patch_size,
-    )
-    factor = (
-        video_processor.patch_size
-        * video_processor.merge_size
-        * video_processor.patch_expand_factor
-    )
     resized_height, resized_width = smart_resize(
-        t=num_frames,
+        num_frames=num_frames,
-        h=height,
+        height=height,
-        w=width,
+        width=width,
-        t_factor=video_processor.temporal_patch_size,
+        temporal_factor=video_processor.temporal_patch_size,
-        h_factor=factor,
+        factor=video_processor.patch_size
-        w_factor=factor,
+        * video_processor.merge_size
-        min_pixels=min_pixels,
+        * video_processor.patch_expand_factor,
-        max_pixels=max_pixels,
+        min_pixels=video_processor.min_image_tokens,
+        max_pixels=video_processor.max_image_tokens,
     )
     padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
     return (
@@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows(
     info = processor.info
     video_processor = info.get_video_processor()
 
-    frame_indices = glm_sample_frame_indices(
+    frame_indices = video_processor.sample_frames(
-        total_num_frames,
+        VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
-        fps,
-        duration,
-        target_fps=video_processor.fps_interval,
-        max_frame_count=video_processor.max_frame_count_dynamic,
-        temporal_patch_size=video_processor.temporal_patch_size,
     )
     grid_t, grid_h, grid_w = _pixel_path_grid(
         video_processor, len(frame_indices), height, width

tests/models/qwen4_exp/test_config.py

@@ -7,16 +7,13 @@ from unittest.mock import patch
 
 import pytest
 import torch
+from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
 
 from vllm.config.speculative import SpeculativeConfig
 from vllm.model_executor.models.config import (
     Qwen3_5ForConditionalGenerationConfig,
     Qwen4ExpForConditionalGenerationConfig,
 )
-from vllm.models.qwen4_exp.config import (
-    Qwen4ExpConfig,
-    Qwen4ExpTextConfig,
-)
 from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
 from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
 
@@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
         "linear_num_value_heads": 2,
         "linear_key_head_dim": 8,
         "linear_value_head_dim": 8,
-        "num_experts": 0,
+        "num_experts": 4,
+        "num_experts_per_tok": 2,
+        # `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
+        "eos_token_id": 1,
         "hc_count": 2,
         "hc_lowrank": 4,
         "ple_layer_ids": [1],

tests/models/qwen4_exp/test_ple.py

@@ -10,6 +10,7 @@ import pytest
 import torch
 from torch import nn
 from torch.nn import functional as F
+from transformers import Qwen4ExpTextConfig
 
 import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
 import vllm.model_executor.parameter as parameter_module
@@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import (
     compute_ple_shard_overlap,
     copy_ple_embedding_shard_,
 )
-from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
 from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
     Qwen4ExpNGramEmbedding,
     Qwen4ExpPLEDeviceEmbedding,

tests/multimodal/test_video.py

@@ -1591,10 +1591,6 @@ def test_glm5next_backend_indices_match_sampler(
 ):
     """The loader must select exactly the frames the processor's sampler
     would, with target.fps mapping onto the raw-fps override."""
-    from vllm.transformers_utils.processors.glm5next import (
-        glm_sample_frame_indices,
-    )
-
     source = VideoSourceMetadata(
         total_frames_num=total_frames, original_fps=original_fps, duration=duration
     )
@@ -1604,7 +1600,7 @@ def test_glm5next_backend_indices_match_sampler(
         source, target, max_frames=max_frames
     )
 
-    assert indices == glm_sample_frame_indices(
+    assert indices == Glm5NextVideoBackend._sample_frame_indices(
         total_frames,
         original_fps,
         duration,
@@ -1692,10 +1688,6 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
     if backend == "torchcodec":
         pytest.importorskip("torchcodec")
 
-    from vllm.transformers_utils.processors.glm5next import (
-        glm_sample_frame_indices,
-    )
-
     total_frames, fps = 120, 10
     path = _write_gray_video(tmp_path, total_frames, fps)
     # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
@@ -1703,7 +1695,7 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
         kwargs: dict[str, Any] = (
             {} if max_frames is None else {"max_frames": max_frames}
         )
-        expected = glm_sample_frame_indices(
+        expected = Glm5NextVideoBackend._sample_frame_indices(
             total_frames, float(fps), 12.0, max_frame_count=max_frames
         )
 
@@ -1726,11 +1718,7 @@ def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
     total_frames, fps = 60, 10
     path = _write_gray_video(tmp_path, total_frames, fps)
 
-    from vllm.transformers_utils.processors.glm5next import (
+    expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0)
-        glm_sample_frame_indices,
-    )
-
-    expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
 
     frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
 

tests/test_config.py

@@ -4114,3 +4114,47 @@ def test_revision_resolved_for_model(mock_resolve):
     assert isinstance(config.revision, ResolvedRevision)
     assert config.revision.resolved == REVISION
     mock_resolve.assert_any_call(model, None, config.hf_token)
+
+
+@pytest.mark.parametrize(
+    ("layer_types", "expected_attention"),
+    [
+        # Qwen3-Next / Qwen3.5 spell their attention layers "full_attention".
+        (["linear_attention", "full_attention"], 1),
+        # GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches
+        # every token and so must count as attention.
+        (["linear_attention", "deepseek_sparse_attention"], 1),
+        (["linear_attention", "qwen_sparse_attention"], 1),
+        (["linear_attention", "linear_attention"], 0),
+    ],
+)
+def test_hybrid_layer_counts_sparse_attention_as_attention(
+    layer_types, expected_attention
+):
+    """Sparse-attention layer types consume a full attention KV cache."""
+    model_config = object.__new__(ModelConfig)
+    model_config.hf_config = SimpleNamespace()
+    model_config.hf_text_config = SimpleNamespace(layer_types=layer_types)
+    model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next")
+
+    parallel_config = SimpleNamespace()
+    with (
+        patch.object(ModelConfig, "is_hybrid", True),
+        patch.object(ModelConfig, "has_noops", False),
+        patch.object(ModelConfig, "is_attention_free", False),
+        patch.object(
+            ModelConfig,
+            "get_layers_start_end_indices",
+            return_value=(0, len(layer_types)),
+        ),
+    ):
+        assert (
+            model_config.get_num_layers_by_block_type(parallel_config, "attention")
+            == expected_attention
+        )
+        assert (
+            model_config.get_num_layers_by_block_type(
+                parallel_config, "linear_attention"
+            )
+            == len(layer_types) - expected_attention
+        )

tests/transformers_utils/processors/test_glm5next.py

@@ -1,389 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor."""
-
-import math
-
-import pytest
-import torch
-from PIL import Image
-from transformers.image_utils import PILImageResampling
-from transformers.video_utils import VideoMetadata
-
-from vllm.transformers_utils.processors.glm5next import (
-    Glm5NextImageProcessor,
-    Glm5NextVideoProcessor,
-    _get_pad_content_size,
-    _resize_or_pad,
-    glm_sample_frame_indices,
-    smart_resize,
-)
-
-PATCH_SIZE = 14
-MERGE_SIZE = 2
-PATCH_EXPAND_FACTOR = 1  # checkpoint processor_config.json
-FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR  # 28
-PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2  # 1568
-MIN_TOKENS = 16
-MAX_TOKENS = 800  # serving-cap equivalent budget, keeps test canvases small
-MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN
-MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN
-
-
-def resize(height: int, width: int, t: int = 2) -> tuple[int, int]:
-    return smart_resize(
-        t,
-        height,
-        width,
-        t_factor=2,
-        h_factor=FACTOR,
-        w_factor=FACTOR,
-        min_pixels=MIN_PIXELS,
-        max_pixels=MAX_PIXELS,
-    )
-
-
-@pytest.mark.parametrize(
-    ("height", "width", "t", "expected"),
-    [
-        (300, 400, 2, (308, 420)),  # ceil alignment, no budget pressure
-        (50, 80, 2, (112, 168)),  # below min budget: proportional upscale
-        (57, 3, 2, (504, 28)),  # slim side ceils to one factor
-        (2160, 3840, 2, (588, 1064)),  # shrink: max canvas under the cap
-        (112, 11200, 2, (84, 7420)),  # extreme aspect stays proportional
-        (300, 400, 16, (252, 308)),  # frame count eats into the budget
-        (112, 11200, 16, (28, 2800)),  # slim side stays positive, budget holds
-    ],
-)
-def test_smart_resize_reference_values(height, width, t, expected):
-    got = smart_resize(
-        t,
-        height,
-        width,
-        t_factor=2,
-        h_factor=FACTOR,
-        w_factor=FACTOR,
-        min_pixels=MIN_PIXELS,
-        max_pixels=MAX_PIXELS,
-    )
-    assert got == expected
-    # The aligned canvas never exceeds the budget it was fitted against.
-    t_bar = max(2, round(t / 2) * 2)
-    assert t_bar * got[0] * got[1] <= MAX_PIXELS
-
-
-def test_smart_resize_stays_snapped_and_positive():
-    heights = (1, 27, 57, 111, 113, 300, 720, 2160)
-    widths = (1, 27, 57, 111, 113, 400, 1280, 11200)
-    for h in heights:
-        for w in widths:
-            for t in (2, 4, 16, 64):
-                resized_h, resized_w = resize(h, w, t)
-                assert resized_h > 0 and resized_w > 0
-                assert resized_h % FACTOR == 0
-                assert resized_w % FACTOR == 0
-                t_bar = max(2, round(t / 2) * 2)
-                assert t_bar * resized_h * resized_w <= MAX_PIXELS
-
-
-@pytest.mark.parametrize(
-    ("height", "width", "match"),
-    [
-        (0, 100, "must be positive"),
-        (100, 0, "must be positive"),
-    ],
-)
-def test_smart_resize_rejects_degenerate_inputs(height, width, match):
-    with pytest.raises(ValueError, match=match):
-        resize(height, width, 2)
-
-
-def test_smart_resize_rejects_inverted_budget():
-    with pytest.raises(ValueError, match="min_pixels must be less than or equal"):
-        smart_resize(
-            2,
-            100,
-            100,
-            t_factor=2,
-            h_factor=FACTOR,
-            w_factor=FACTOR,
-            min_pixels=MAX_PIXELS,
-            max_pixels=MIN_PIXELS,
-        )
-
-
-def test_get_pad_content_size():
-    # Oversized content shrinks proportionally, never upscales by default.
-    assert _get_pad_content_size(300, 400, 308, 420) == (300, 400)
-    assert _get_pad_content_size(600, 800, 308, 420) == (308, 410)
-    # allow_upscale enlarges small content toward the canvas.
-    assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112)
-
-
-def test_resize_or_pad_pads_right_and_bottom():
-    stacked = torch.rand(1, 3, 300, 400)
-
-    def identity_resize(x, size, resample=None):
-        assert (size.height, size.width) == (300, 400)
-        return x
-
-    padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize)
-    assert padded.shape == (1, 3, 308, 420)
-    torch.testing.assert_close(padded[..., :300, :400], stacked)
-    assert padded[..., 300:, :].eq(0).all()
-    assert padded[..., :, 400:].eq(0).all()
-
-    def force_resize(x, size, resample=None):
-        return torch.zeros(x.shape[0], x.shape[1], size.height, size.width)
-
-    assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == (
-        1,
-        3,
-        308,
-        420,
-    )
-    with pytest.raises(ValueError, match="resize_mode"):
-        _resize_or_pad(stacked, 308, 420, "crop", None, identity_resize)
-
-
-@pytest.fixture(scope="module")
-def image_processor():
-    return Glm5NextImageProcessor(
-        min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
-    )
-
-
-@pytest.mark.parametrize(
-    ("height", "width"),
-    [(50, 80), (300, 400), (2160, 3840), (112, 11200)],
-)
-def test_image_grid_matches_smart_resize(image_processor, height, width):
-    out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
-    resized_h, resized_w = resize(height, width)
-    grid = out["image_grid_thw"][0].tolist()
-    assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE]
-    assert out["pixel_values"].shape == (
-        grid[1] * grid[2],
-        3 * 2 * PATCH_SIZE * PATCH_SIZE,
-    )
-
-
-@pytest.mark.parametrize(
-    ("height", "width"),
-    [(50, 80), (300, 400), (2160, 3840), (112, 11200)],
-)
-def test_image_patch_count_matches_preprocess(image_processor, height, width):
-    out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
-    grid = out["image_grid_thw"][0].tolist()
-    expected = grid[1] * grid[2]
-    assert image_processor.get_number_of_image_patches(height, width) == expected
-
-
-@pytest.fixture(scope="module")
-def video_processor():
-    return Glm5NextVideoProcessor(
-        min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
-    )
-
-
-def run_video_preprocess(video_processor, frames, **overrides):
-    kwargs = dict(
-        resample=PILImageResampling.BICUBIC,
-        image_mean=(0.48145466, 0.4578275, 0.40821073),
-        image_std=(0.26862954, 0.26130258, 0.27577711),
-        patch_size=PATCH_SIZE,
-        temporal_patch_size=2,
-        merge_size=MERGE_SIZE,
-        patch_expand_factor=PATCH_EXPAND_FACTOR,
-        return_tensors="pt",
-    )
-    kwargs.update(overrides)
-    return video_processor._preprocess([frames], **kwargs)
-
-
-def test_video_preprocess_pads_odd_frame_count(video_processor):
-    out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400))
-    grid = out["video_grid_thw"][0].tolist()
-    # 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420).
-    assert grid == [4, 22, 30]
-    assert out["pixel_values_videos"].shape == (
-        4 * 22 * 30,
-        3 * 2 * PATCH_SIZE * PATCH_SIZE,
-    )
-
-
-def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor):
-    out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200))
-    # Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap.
-    assert out["video_grid_thw"][0].tolist() == [8, 2, 200]  # height 28, not 0
-
-
-def test_video_preprocess_pads_content_not_distort(video_processor):
-    frames = torch.zeros(4, 3, 300, 400)
-    frames[..., 50, 60] = 1.0  # marker inside the content area
-    # Bypass rescale/normalize so zero padding stays exactly zero.
-    out = run_video_preprocess(
-        video_processor, frames, do_rescale=False, do_normalize=False
-    )
-    # Pad mode keeps the 300x400 content aspect on the (308, 420) canvas
-    # with zero padding on the right/bottom.
-    grid = out["video_grid_thw"][0].tolist()
-    assert grid == [2, 22, 30]
-    patches = out["pixel_values_videos"].view(
-        grid[0],
-        grid[1] // MERGE_SIZE,
-        grid[2] // MERGE_SIZE,
-        MERGE_SIZE,
-        MERGE_SIZE,
-        3 * 2 * PATCH_SIZE * PATCH_SIZE,
-    )
-    frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1)
-    # Patch columns fully right of the 400px content are pure padding.
-    assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0
-
-
-@pytest.mark.parametrize(
-    ("total_frames", "fps", "duration", "expected_len", "first", "last"),
-    [
-        # Dense source: the tp-scaled greedy overshoots extract_t, so the
-        # fixup spreads picks uniformly across the whole video (linspace).
-        (900, 30.0, 30.0, 60, 0, 899),
-        (3000, 30.0, 100.0, 200, 0, 2999),
-        # extract_t capped at 2048 frames.
-        (72000, 30.0, 2400.0, 2048, 0, 71999),
-        # Low container fps: the greedy picks every frame.
-        (48, 2.0, 24.0, 48, 0, 47),
-        # Duration derived from the frame count when metadata lacks it.
-        (300, 25.0, 0, 26, 0, 299),
-    ],
-)
-def test_glm_sample_frame_indices_behaviour(
-    total_frames, fps, duration, expected_len, first, last
-):
-    indices = glm_sample_frame_indices(total_frames, fps, duration)
-    assert len(indices) == expected_len
-    assert indices[0] == first
-    assert indices[-1] == last
-    assert len(indices) % 2 == 0
-    assert indices == sorted(indices)
-    assert all(0 <= idx < total_frames for idx in indices)
-
-
-def test_glm_sample_frame_indices_short_clip_floor_spread():
-    # extract_t (20) > total (7): evenly spaced timestamps, deduplicated,
-    # then pair-padded to an even count.
-    assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6]
-
-
-@pytest.mark.parametrize(
-    ("kwargs", "expected_len"),
-    [
-        ({"target_fps": 0.5}, 16),
-        ({"max_frame_count": 16}, 16),
-        ({"target_fps": 8}, 240),  # 30s * 8 = 240, under the 2048 cap
-    ],
-)
-def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len):
-    indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs)
-    assert len(indices) == expected_len
-    assert len(indices) % 2 == 0
-
-
-def test_sample_frames_fps_interval(video_processor):
-    indices = video_processor.sample_frames(
-        VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
-    )
-    assert len(indices) == 60
-    assert indices[0] == 0
-    assert indices[-1] == 899  # uniform spread reaches the final frame
-
-    # Request overrides reach the sampler through both kwarg spellings. The
-    # 15-frame spread is odd, so pair-padding duplicates the last frame.
-    assert (
-        len(
-            video_processor.sample_frames(
-                VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
-                fps=0.5,
-            )
-        )
-        == 16
-    )
-    assert (
-        len(
-            video_processor.sample_frames(
-                VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
-                target_fps=0.5,
-            )
-        )
-        == 16
-    )
-    assert (
-        len(
-            video_processor.sample_frames(
-                VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
-                max_frames=16,
-            )
-        )
-        == 16
-    )
-
-
-def test_defaults_mirror_checkpoint_config():
-    """Bare instantiation matches the checkpoint's ``processor_config.json``
-    token-budget style defaults; ``size`` carries no budget."""
-    image_processor = Glm5NextImageProcessor()
-    assert image_processor.patch_expand_factor == 1
-    assert image_processor.min_image_tokens == 16
-    assert image_processor.max_image_tokens == 8000
-    assert image_processor.resize_mode == "pad"
-
-    video_processor = Glm5NextVideoProcessor()
-    assert video_processor.patch_expand_factor == 1
-    assert video_processor.min_image_tokens == 16
-    assert video_processor.max_image_tokens == 240000
-    assert video_processor.fps_interval == 2.0
-    assert video_processor.max_frame_count_dynamic == 2048
-
-
-def test_token_budgets_drive_geometry():
-    """The token bounds fully determine the pixel budget and the grid."""
-    token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512)
-    img = Image.new("RGB", (400, 300))
-    grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist()
-    # Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420)
-    # without hitting either bound -> 22x30 patches.
-    assert grid == [1, 22, 30]
-    assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30
-
-
-def test_missing_token_budgets_rejected():
-    proc = Glm5NextImageProcessor(min_image_tokens=None)
-    with pytest.raises(ValueError, match="min_image_tokens"):
-        proc(Image.new("RGB", (400, 300)), return_tensors="pt")
-
-
-def test_video_config_fields_land():
-    """fps_interval / max_frame_count_dynamic from the dedicated config
-    shape sampling without any request overrides."""
-    proc = Glm5NextVideoProcessor(
-        fps_interval=4,
-        max_frame_count_dynamic=32,
-    )
-    indices = proc.sample_frames(
-        VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
-    )
-    # 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32.
-    assert len(indices) == 32
-    assert indices[0] == 0
-    assert indices[-1] == 899
-
-    # Request overrides still win over the config values.
-    assert (
-        len(
-            proc.sample_frames(
-                VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
-                fps=0.5,
-            )
-        )
-        == 16
-    )

tests/transformers_utils/test_config.py

@@ -24,11 +24,6 @@ from vllm.transformers_utils.config import (
     try_get_generation_config,
     uses_mrope,
 )
-from vllm.transformers_utils.configs.glm5_next import (
-    Glm5NextConfig,
-    Glm5NextTextConfig,
-    Glm5NextVisionConfig,
-)
 from vllm.transformers_utils.configs.mistral import adapt_config_dict
 
 
@@ -174,54 +169,6 @@ def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling():
     assert config.rope_parameters["attention_factor"] == 1.0
 
 
-def test_glm5_next_accepts_deepseek_sparse_attention_layers():
-    layer_types = ["linear_attention", "deepseek_sparse_attention"]
-
-    config = Glm5NextTextConfig(
-        num_hidden_layers=len(layer_types), layer_types=layer_types
-    )
-
-    assert config.layer_types == layer_types
-    assert config.layers_block_type == ["linear_attention", "attention"]
-
-
-def test_glm5_next_accepts_prebuilt_subconfigs():
-    text_config = Glm5NextTextConfig(hidden_size=1024)
-    vision_config = Glm5NextVisionConfig(hidden_size=768)
-
-    config = Glm5NextConfig(
-        text_config=text_config,
-        vision_config=vision_config,
-    )
-
-    assert config.text_config is text_config
-    assert config.vision_config is vision_config
-
-
-@pytest.mark.parametrize(
-    ("kwargs", "option"),
-    [
-        (
-            {"index_topk": 2048, "index_dsa_use_layernorm": False},
-            "index_dsa_use_layernorm",
-        ),
-        (
-            {"index_topk": 2048, "index_kpool_compress": False},
-            "index_kpool_compress",
-        ),
-        (
-            {"index_topk": 2048, "index_kpool_always_select_tail": False},
-            "index_kpool_always_select_tail",
-        ),
-        ({"hres_vwnstyle": False}, "hres_vwnstyle"),
-        ({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"),
-    ],
-)
-def test_glm5_next_rejects_unimplemented_config_options(kwargs, option):
-    with pytest.raises(NotImplementedError, match=option):
-        Glm5NextTextConfig(**kwargs)
-
-
 def test_get_llama3_eos_token():
     model_name = "meta-llama/Llama-3.2-1B-Instruct"
 

tests/v1/spec_decode/test_mtp.py

@@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config):
         index_kpool=4,
     )
     vllm_config = mock.MagicMock()
-    vllm_config.speculative_config.draft_model_config.hf_config = config
+    vllm_config.speculative_config.draft_model_config.hf_text_config = config
     vllm_config.scheduler_config.max_num_batched_tokens = 4
 
     with (

vllm/config/model.py

@@ -108,6 +108,16 @@ HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig]
 ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"]
 LayerBlockType = Literal["attention", "linear_attention", "mamba"]
 
+_ATTENTION_LAYER_TYPES = frozenset(
+    {
+        "full_attention",
+        "deepseek_sparse_attention",
+        "qwen_sparse_attention",
+    }
+)
+"""`layer_types` spellings that consume a full attention KV cache. Sparse
+attention still caches every token, so it counts as attention here."""
+
 _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
     "generate": [],
     "pooling": ["embed", "classify"],
@@ -1718,7 +1728,8 @@ class ModelConfig:
             if layer_types_value is not None:
                 if block_type == "attention":
                     return sum(
-                        t == "full_attention" for t in layer_types_value[start:end]
+                        t in _ATTENTION_LAYER_TYPES
+                        for t in layer_types_value[start:end]
                     )
                 elif block_type == "linear_attention":
                     return sum(

vllm/config/speculative.py

@@ -1059,7 +1059,7 @@ class SpeculativeConfig:
             )
         if hf_config.model_type == "glm5_next":
             hf_config.model_type = "glm5_next_mtp"
-            n_predict = hf_config.num_nextn_predict_layers
+            n_predict = hf_config.get_text_config().num_nextn_predict_layers
             hf_config.update(
                 {"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]}
             )

vllm/models/glm5next/common/attention.py

@@ -4,6 +4,7 @@
 import torch
 import torch.nn.functional as F
 from torch import nn
+from transformers import Glm5NextTextConfig
 
 from vllm.config import (
     CacheConfig,
@@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
 from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
 from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
 from vllm.platforms import current_platform
-from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
 from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
 from vllm.utils.math_utils import cdiv, next_power_of_2
 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
@@ -221,7 +221,7 @@ class Indexer(nn.Module):
     def __init__(
         self,
         vllm_config: VllmConfig,
-        config: Glm5NextConfig,
+        config: Glm5NextTextConfig,
         hidden_size: int,
         q_lora_rank: int,
         quant_config: QuantizationConfig | None,
@@ -236,7 +236,7 @@ class Indexer(nn.Module):
         # self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
         # Indexer is only constructed for v32 configs, where these sparse-indexer
         # fields are guaranteed populated; narrow away the `int | None` declared
-        # on Glm5NextConfig for the optional-indexer case.
+        # on Glm5NextTextConfig for the optional-indexer case.
         assert config.index_topk is not None
         assert config.index_n_heads is not None
         assert config.index_head_dim is not None
@@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module):
     def __init__(
         self,
         vllm_config: VllmConfig,
-        config: Glm5NextConfig,
+        config: Glm5NextTextConfig,
         hidden_size: int,
         num_heads: int,
         qk_nope_head_dim: int,
@@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module):
             prefix=f"{prefix}.o_proj",
         )
 
+        # `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`;
+        # a checkpoint may still ship one.
+        rope_parameters = getattr(config, "rope_parameters", None)
+
         if not skip_rope:
-            assert config.rope_parameters is not None
+            assert rope_parameters is not None
-            if config.rope_parameters["rope_type"] != "default":
+            if rope_parameters["rope_type"] != "default":
-                config.rope_parameters["rope_type"] = (
+                rope_parameters["rope_type"] = (
                     "deepseek_llama_scaling"
-                    if config.rope_parameters.get("attention_factor") == 1.0
+                    if rope_parameters.get("attention_factor") == 1.0
                     else "deepseek_yarn"
                 )
 
             self.rotary_emb: RotaryEmbedding | None = get_rope(
                 qk_rope_head_dim,
                 max_position=max_position_embeddings,
-                rope_parameters=config.rope_parameters,
+                rope_parameters=rope_parameters,
                 is_neox_style=False,
             )
 
             if (
-                config.rope_parameters["rope_type"] != "default"
+                rope_parameters["rope_type"] != "default"
-                and config.rope_parameters["rope_type"] == "deepseek_yarn"
+                and rope_parameters["rope_type"] == "deepseek_yarn"
             ):
-                mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
+                mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
-                scaling_factor = config.rope_parameters["factor"]
+                scaling_factor = rope_parameters["factor"]
                 mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
                 self.scaling = self.scaling * mscale * mscale
         else:
@@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module):
             self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
                 qk_rope_head_dim,
                 max_position=max_position_embeddings,
-                rope_parameters=config.rope_parameters,
+                rope_parameters=rope_parameters,
                 is_neox_style=not config.indexer_rope_interleave,
             )
             # The sparse indexer projects from the MLA q-lora rank, which is

vllm/models/glm5next/common/kda.py

@@ -6,6 +6,7 @@ from dataclasses import replace
 
 import torch
 from torch import nn
+from transformers import Glm5NextTextConfig
 
 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
 from vllm.config import VllmConfig, get_current_vllm_config
@@ -44,7 +45,6 @@ from vllm.model_executor.utils import (
 )
 from vllm.platforms import current_platform
 from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
-from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
 from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
 from vllm.v1.kv_cache_interface import MambaSpec
 from vllm.v1.worker.workspace import current_workspace_manager
@@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
 
     def __init__(
         self,
-        config: Glm5NextConfig,
+        config: Glm5NextTextConfig,
         vllm_config: VllmConfig,
         prefix: str = "",
     ) -> None:

vllm/models/glm5next/common/model.py

@@ -6,6 +6,7 @@ from typing import ClassVar, Literal
 
 import torch
 from torch import nn
+from transformers import Glm5NextTextConfig
 
 from vllm.config import ParallelConfig, VllmConfig
 from vllm.distributed import (
@@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import (
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.platforms import current_platform
 from vllm.sequence import IntermediateTensors
-from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
 
 from .attention import Glm5NextMLAAttention
 from .kda import Glm5NextLinearAttention
@@ -95,6 +95,57 @@ from .multimodal import (
 
 logger = init_logger(__name__)
 
+_MHC_TAU = 0.05
+"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the
+checkpoint nor `Glm5NextTextConfig` carries."""
+
+_MHC_POST_MULT_VALUE = 2.0
+"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the
+checkpoint nor `Glm5NextTextConfig` carries."""
+
+_LOGIT_SCALE = 1.0
+"""Output logit scale. A GLM-5.3-Flash trained value that neither the
+checkpoint nor `Glm5NextTextConfig` carries."""
+
+_VISION_RMS_NORM_EPS = 1e-6
+"""Vision tower RMSNorm epsilon.
+
+GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the
+vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and
+produces repetitive/degraded image descriptions, so force the trained value
+regardless of the checkpoint field.
+"""
+
+
+def _is_moe(config: Glm5NextTextConfig) -> bool:
+    return config.n_routed_experts is not None
+
+
+def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool:
+    layer_types = config.layer_types
+    return (
+        layer_types is not None
+        and layer_idx < len(layer_types)
+        and layer_types[layer_idx] == "linear_attention"
+    )
+
+
+def _is_linear_attn(config: Glm5NextTextConfig) -> bool:
+    layer_types = config.layer_types
+    return layer_types is not None and "linear_attention" in layer_types
+
+
+def _validate_supported_config(config: Glm5NextTextConfig) -> None:
+    """Reject checkpoints using config options this implementation lacks.
+
+    The kpool indexer kernels always keep the incomplete trailing pool, so a
+    checkpoint asking otherwise would be served silently wrong.
+    """
+    if config.index_topk is not None and not config.index_kpool_always_select_tail:
+        raise NotImplementedError(
+            "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
+        )
+
 
 class Glm5NextMLP(nn.Module):
     def __init__(
@@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module):
 class Glm5NextMoE(nn.Module):
     def __init__(
         self,
-        config: Glm5NextConfig,
+        config: Glm5NextTextConfig,
         parallel_config: ParallelConfig,
         quant_config: QuantizationConfig | None = None,
         prefix: str = "",
@@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module):
             shared_experts=self.shared_experts,
             gate=self.gate,
             num_experts=config.n_routed_experts,
-            top_k=config.num_experts_per_token,
+            top_k=config.num_experts_per_tok,
             hidden_size=config.hidden_size,
             intermediate_size=config.moe_intermediate_size,
-            renormalize=config.moe_renormalize,
+            renormalize=config.norm_topk_prob,
             quant_config=quant_config,
             use_grouped_topk=True,
             num_expert_group=config.n_group,
@@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module):
     def __init__(
         self,
         vllm_config: VllmConfig,
-        config: Glm5NextConfig,
+        config: Glm5NextTextConfig,
         layer_idx: int,
         prefix: str = "",
         topk_indices_buffer: torch.Tensor | None = None,
@@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module):
 
         self.hidden_size = config.hidden_size
         self.layer_idx = layer_idx
-        self.is_moe = config.is_moe
+        self.is_moe = _is_moe(config)
         self.num_hidden_layers = config.num_hidden_layers
         self.rms_norm_eps = config.rms_norm_eps
         self.num_experts = config.n_routed_experts
         self.is_mtp_layer = is_mtp_layer
         self.mhc = config.mhc
-        is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx)
+        is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx)
         self.layer_kind = "kda" if is_kda_layer else "mla"
         self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
 
@@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module):
                 quant_config=None,  # MLA projections are BF16 in checkpoint
                 prefix=f"{prefix}.self_attn",
                 topk_indices_buffer=topk_indices_buffer,
-                skip_rope=config.mla_nope,
+                skip_rope=config.mla_use_nope,
             )
 
         # MTP layers sit past the base model's hidden layers (layer_idx >=
@@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module):
 
         if self.mhc and not is_mtp_layer:
             # mhc config
-            self.mhc_num_residual_streams = config.mhc_num_residual_streams
+            self.mhc_num_residual_streams = config.hc_mult
-            self.mhc_tau = config.mhc_tau
+            self.mhc_tau = _MHC_TAU
             self.hc_eps = config.hc_eps
-            self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations
+            self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters
-            self.mhc_post_mult_value = config.mhc_post_mult_value
+            self.mhc_post_mult_value = _MHC_POST_MULT_VALUE
 
-            n = config.mhc_num_residual_streams
+            n = config.hc_mult
             d_model = n * self.hidden_size
             mix_hc = (2 + n) * n
 
@@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module):
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__()
 
-        config = vllm_config.model_config.hf_config
+        config = vllm_config.model_config.hf_text_config
+        _validate_supported_config(config)
         self.config = config
 
         self.vocab_size = config.vocab_size
@@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module):
             (".in_proj_qkvbfg_a", ".f_a_proj", 4),
             (".in_proj_qkvbfg_a", ".g_a_proj", 5),
         ]
-        if self.config.is_moe:
+        if _is_moe(self.config):
             # Params for weights, fp8 weight scales, fp8 activation scales
             # (param_name, weight_name, expert_id, shard_id)
             # EPLB: the mapping enumerates physical experts, so it must cover
@@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module):
         # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
         # ``kv_a_proj_with_mqa``; pad them with zeros for the model shape.
         kv_a_pad_size = 0
-        if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
+        if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
             kv_a_pad_size = self.config.qk_rope_head_dim
 
         _pending_wk_fp8: dict = {}
@@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module):
                     if (
                         name.endswith(".bias")
                         and name not in params_dict
-                        and not self.config.is_linear_attn
+                        and not _is_linear_attn(self.config)
                     ):  # noqa: E501
                         continue
                     # Remapping the name of FP8 kv-scale.
@@ -939,7 +991,7 @@ class Glm5NextForCausalLM(
         super().__init__()
         self.model_config = vllm_config.model_config
         self.vllm_config = vllm_config
-        self.config = self.model_config.hf_config
+        self.config = self.model_config.hf_text_config
         quant_config = vllm_config.quant_config
         self.quant_config = quant_config
         self.model = Glm5NextModel(
@@ -955,7 +1007,7 @@ class Glm5NextForCausalLM(
         else:
             self.lm_head = PPMissingLayer()
         self.logits_processor = LogitsProcessor(
-            self.config.vocab_size, scale=self.config.logit_scale
+            self.config.vocab_size, scale=_LOGIT_SCALE
         )
 
     def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -988,7 +1040,7 @@ class Glm5NextForCausalLM(
         cls, vllm_config: "VllmConfig"
     ) -> tuple[tuple[int, int], tuple[int, int, int]]:
         parallel_config = vllm_config.parallel_config
-        hf_config = vllm_config.model_config.hf_config
+        hf_config = vllm_config.model_config.hf_text_config
         tp_size = parallel_config.tensor_parallel_size
         num_spec = (
             vllm_config.speculative_config.num_speculative_tokens
@@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration(
             self.visual = Glm5NextVisionTransformer(
                 config.text_config,
                 config.vision_config,
-                # Read eps from the VISION sub-config, not the top-level
+                norm_eps=_VISION_RMS_NORM_EPS,
-                # `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors
-                # the latter onto text_config (1e-5), silently ignoring the
-                # vision tower's own (1e-6) rms_norm_eps.
-                norm_eps=config.vision_config.rms_norm_eps,
                 # Vision tower ships BF16 weights in this fp8 checkpoint (no
                 # weight_scale_inv for visual.*), so it must NOT inherit the
                 # global fp8 quant_config -- doing so incorrectly quantizes
@@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration(
 
 
 def get_spec_layer_idx_from_weight_name(
-    config: Glm5NextConfig, weight_name: str
+    config: Glm5NextTextConfig, weight_name: str
 ) -> int | None:
     if hasattr(config, "num_nextn_predict_layers") and (
         config.num_nextn_predict_layers > 0

vllm/models/glm5next/common/mtp.py

@@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
     def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
         super().__init__()
         assert vllm_config.speculative_config is not None
-        config = vllm_config.speculative_config.draft_model_config.hf_config
+        config = vllm_config.speculative_config.draft_model_config.hf_text_config
         self.config = config
 
         self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
 class Glm5NextMultiTokenPredictor(nn.Module):
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__()
-        config = vllm_config.model_config.hf_config
+        config = vllm_config.model_config.hf_text_config
         self.mtp_start_layer_idx = config.num_hidden_layers
         self.num_mtp_layers = config.num_nextn_predict_layers
         self.layers = torch.nn.ModuleDict(
@@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module):
 class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__()
-        self.config = vllm_config.model_config.hf_config
+        self.config = vllm_config.model_config.hf_text_config
         self.quant_config = vllm_config.quant_config
         self.model = Glm5NextMultiTokenPredictor(
             vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
@@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
         # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
         # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model.
         kv_a_pad_size = 0
-        if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
+        if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
             kv_a_pad_size = self.config.qk_rope_head_dim
         for name, loaded_weight in weights:
             if "rotary_emb.inv_freq" in name:

vllm/models/glm5next/common/multimodal.py

@@ -3,13 +3,14 @@
 """GLM-5.3-Flash vision tower and multimodal processor."""
 
 from collections.abc import Mapping
-from functools import cached_property, partial
+from functools import partial
 from typing import Any
 
 import numpy as np
 import torch
 import torch.nn as nn
 from einops import rearrange
+from transformers.video_utils import VideoMetadata
 
 from vllm.distributed import (
     get_tensor_model_parallel_world_size,
@@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import (
 )
 from vllm.models.common.ops import fused_q_kv_rmsnorm
 from vllm.multimodal.parse import ImageSize, MultiModalDataItems
-from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
 from vllm.v1.attention.backends.registry import AttentionBackendEnum
 
 
@@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module):
         return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
 
 
-class Glm5NextProcessingInfo(Glm4vProcessingInfo):
+_MAX_VIDEO_TOKENS = 30000
-    """Wires up the vLLM-native processor for the multimodal checkpoint.
 
-    The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
+
-    and stores its image/video processor configs inline (no standalone
+class Glm5NextProcessingInfo(Glm4vProcessingInfo):
-    ``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
+    """Token-budget geometry for the multimodal checkpoint.
-    config. We bypass it and build our own ``Glm5NextProcessor``
+
-    (``vllm/transformers_utils/processors/glm5next.py``), a port of the
+    This checkpoint's ``processor_config.json`` ships the token-budget style
-    training-side pipeline that no longer imports transformers' GLM processor
+    (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
-    classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
+    inherited Glm4v ``size.longest_edge`` path does not apply.
-    inside ``smart_resize``'s spatial factor.
     """
 
-    @cached_property
-    def _glm5_hf_processor(self):
-        from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
-
-        return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
-
     def get_hf_processor(self, **kwargs: object):
-        return self._glm5_hf_processor
+        processor = super().get_hf_processor(**kwargs)
+        # Cap video inputs to keep encoder profiling from starving the KV cache.
+        video_processor = processor.video_processor
+        video_processor.max_image_tokens = min(
+            video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
+        )
+        return processor
+
+    @staticmethod
+    def _alignment_factor(proc) -> int:
+        """Spatial factor canvases align to, as ``smart_resize`` takes it."""
+        return proc.patch_size * proc.merge_size * proc.patch_expand_factor
+
+    @classmethod
+    def _pixels_per_token(cls, proc) -> int:
+        """Pixels one vision token covers, matching ``smart_resize``'s own
+        ``temporal_factor * factor ** 2``."""
+        return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
 
     def _processor_pixel_budget(self, proc) -> tuple[int, int]:
-        from vllm.transformers_utils.processors.glm5next import _pixel_budget
+        """(min_pixels, max_pixels) from the processor's token bounds."""
-
+        if proc.min_image_tokens is None or proc.max_image_tokens is None:
-        return _pixel_budget(
+            raise ValueError(
-            proc.min_image_tokens,
+                "min_image_tokens and max_image_tokens must be provided by "
-            proc.max_image_tokens,
+                "processor_config.json (or per-call kwargs)."
-            proc.patch_size,
+            )
-            proc.merge_size,
+        pixels_per_token = self._pixels_per_token(proc)
-            proc.temporal_patch_size,
+        return (
+            proc.min_image_tokens * pixels_per_token,
+            proc.max_image_tokens * pixels_per_token,
         )
 
     def _get_image_max_pixels(self) -> int:
@@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
         else:
             frame_indices = [
                 int(idx)
-                for idx in glm_sample_frame_indices(
+                for idx in video_processor.sample_frames(
-                    int(metadata.get("total_num_frames", total_frames)),
+                    VideoMetadata(
-                    video_fps,
+                        total_num_frames=int(
-                    float(metadata.get("duration") or 0),
+                            metadata.get("total_num_frames", total_frames)
-                    target_fps=video_processor.fps_interval,
+                        ),
-                    max_frame_count=video_processor.max_frame_count_dynamic,
+                        fps=video_fps,
-                    temporal_patch_size=temporal_patch_size,
+                        duration=metadata.get("duration"),
+                    )
                 )
             ]
             num_frames = len(frame_indices)
 
         if not frame_indices or num_frames <= 0:
-            # ``duration * fps_interval < 1`` selects no frames at all; the
+            # ``duration * fps < 1`` selects no frames at all; the
             # pixel path raises an opaque IndexError on the same input a moment
             # later, so say what happened while there is still a request.
             raise ValueError(
@@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
 
         The inherited Glm4v path resolves the pixel budget from
         ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
-        checkpoint's ``processor_config.json`` ships the token-budget style
+        checkpoint ships token bounds instead, and the alignment factor carries
-        (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
+        ``patch_expand_factor`` — resolve both from the processor so profiling
-        the alignment factor carries ``patch_expand_factor`` — resolve both
+        matches runtime geometry.
-        from the vLLM-native processor so profiling matches runtime geometry.
         """
-        from vllm.transformers_utils.processors.glm5next import smart_resize
+        from transformers.models.glm5_next.image_processing_glm5_next import (
+            smart_resize,
+        )
 
         vision_config = self.get_hf_config().vision_config
         patch_size = vision_config.patch_size
@@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
         temporal_patch_size = vision_config.temporal_patch_size
 
         image_processor = self.get_hf_processor().image_processor
-        factor = patch_size * merge_size * image_processor.patch_expand_factor
+        factor = self._alignment_factor(image_processor)
-        # Keep the profiling search viable when the caller's budget is below
+        temporal_factor = image_processor.temporal_patch_size
-        # one aligned canvas of the requested duration.
+        # `smart_resize` denominates its bounds in vision tokens. Round down,
-        max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
+        # but keep the profiling search viable when the caller's budget is
+        # below one aligned canvas of the requested duration.
+        max_image_tokens = max(
+            max_image_pixels // self._pixels_per_token(image_processor), 1
+        )
 
         if do_resize:
-            t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
+            t = num_frames if num_frames > temporal_factor else temporal_factor
             resized_height, resized_width = smart_resize(
-                t=t,
+                num_frames=t,
-                h=image_height,
+                height=image_height,
-                w=image_width,
+                width=image_width,
-                t_factor=temporal_patch_size,
+                temporal_factor=temporal_factor,
-                h_factor=factor,
+                factor=factor,
-                w_factor=factor,
                 min_pixels=1,
-                max_pixels=max_image_pixels,
+                max_pixels=max_image_tokens,
             )
             preprocessed_size = ImageSize(width=resized_width, height=resized_height)
         else:
@@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
 
 
 class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
-    """The vLLM-native ``Glm5NextProcessor`` extracts image/video features
+    """``Glm5NextProcessor`` extracts image/video features
     only and passes the prompt text through unchanged, so prompt expansion
     (image token repeat, video frame/timestamp structure) is owned by vLLM's
     prompt-update machinery — the inherited ``_get_prompt_updates`` builds

vllm/models/qwen4_exp/amd/indexer_qsa.py

@@ -8,6 +8,7 @@ from typing import cast
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import VllmConfig
 from vllm.forward_context import get_forward_context
@@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
 from vllm.model_executor.layers.linear import ReplicatedLinear
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 
 from ..common.qsa_cache import (
     QSACompressedKeyCache,

vllm/models/qwen4_exp/amd/model.py

@@ -7,6 +7,7 @@ from itertools import islice
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
 
 from vllm.compilation.decorators import support_torch_compile
 from vllm.config import VllmConfig
@@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import (
 )
 from vllm.model_executor.models.qwen3_next import (
     Qwen3NextAttention,
-    Qwen3NextMLP,
     Qwen3NextSparseMoeBlock,
 )
 from vllm.model_executor.models.qwen3_vl import (
@@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.inputs import MultiModalFeatureSpec
 from vllm.sequence import IntermediateTensors
 from vllm.tokenizers.registry import cached_tokenizer_from_config
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
 from vllm.v1.kv_cache_interface import MambaSpec
 
-from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
 from .hyperconnection import GatedResidual, HyperConnectionConfig
 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
 from .ple_layer import Qwen4ExpPLELayer
@@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
     ) -> None:
         super().__init__()
         config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
-        model_config = vllm_config.model_config
-        cache_config = vllm_config.cache_config
         quant_config = vllm_config.quant_config
 
         self.config = config
@@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
                 prefix=f"{prefix}.linear_attn",
                 gqa_interleaved_layout=False,
             )
-        elif layer_type in ATTENTION_LAYER_TYPES:
+        elif layer_type == "qwen_sparse_attention":
-            use_qsa = (
+            use_qsa = getattr(config, "indexer_n_heads", None) is not None
-                layer_type == QSA_LAYER_TYPE
-                or getattr(config, "indexer_n_heads", None) is not None
-            )
             if not use_qsa:
                 self.self_attn = Qwen3NextAttention(
                     config,
-                    model_config=model_config,
+                    model_config=vllm_config.model_config,
-                    cache_config=cache_config,
+                    cache_config=vllm_config.cache_config,
                     quant_config=quant_config,
                     prefix=f"{prefix}.self_attn",
                 )
@@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
         else:
             raise ValueError(f"Invalid layer_type {layer_type}")
 
-        mlp_only_layers = getattr(config, "mlp_only_layers", [])
+        self.mlp = Qwen4ExpSparseMoeBlock(
-        num_experts = getattr(config, "num_experts", 0) or 0
+            vllm_config=vllm_config, prefix=f"{prefix}.mlp"
-        absolute_layer_id = self.layer_idx + 1
-        is_moe_layer = self.layer_idx not in mlp_only_layers and (
-            num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
         )
-        if is_moe_layer:
-            self.mlp = Qwen4ExpSparseMoeBlock(
-                vllm_config=vllm_config, prefix=f"{prefix}.mlp"
-            )
-        else:
-            self.mlp = Qwen3NextMLP(
-                hidden_size=config.hidden_size,
-                intermediate_size=config.intermediate_size,
-                hidden_act=config.hidden_act,
-                quant_config=quant_config,
-                prefix=f"{prefix}.mlp",
-            )
 
         hc_config = HyperConnectionConfig(
             hc_count=config.hc_count,
@@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
 
         if self.layer_type == "linear_attention":
             attn_out = self.linear_attn(hidden_states=block_input)
-        elif self.layer_type in ATTENTION_LAYER_TYPES:
+        elif self.layer_type == "qwen_sparse_attention":
             attn_out = self.self_attn(
                 hidden_states=block_input,
                 positions=positions,
@@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module):
         self._qsa_layer_ids = frozenset(
             layer_idx
             for layer_idx, layer_type in enumerate(config.layer_types)
-            if layer_type == QSA_LAYER_TYPE
+            if layer_type == "qwen_sparse_attention"
-            or (
+            and getattr(config, "indexer_n_heads", None) is not None
-                layer_type == "full_attention"
-                and getattr(config, "indexer_n_heads", None) is not None
-            )
         )
         self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
 

vllm/models/qwen4_exp/amd/mtp.py

@@ -18,6 +18,7 @@ from collections.abc import Iterable
 import regex as re
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.compilation.decorators import support_torch_compile
 from vllm.config import VllmConfig, replace, set_current_vllm_config
@@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import (
     maybe_prefix,
 )
 from vllm.sequence import IntermediateTensors
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 
 from .hyperconnection import GatedResidual, HyperConnectionConfig
 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
             self.layers = nn.ModuleList(
                 Qwen4ExpDecoderLayer(
                     draft_vllm_config,
-                    layer_type="full_attention",
+                    layer_type="qwen_sparse_attention",
                     prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
                 )
                 for idx in range(self.num_mtp_layers)

vllm/models/qwen4_exp/amd/ple_layer.py

@@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence
 import torch
 import torch.nn.functional as F
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
 from vllm.forward_context import get_forward_context
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
     QuantizationConfig,
 )
 from vllm.model_executor.models.utils import AutoWeightsLoader
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.utils.torch_utils import direct_register_custom_op
 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
 from vllm.v1.attention.backends.short_conv_attn import (

vllm/models/qwen4_exp/amd/qsa.py

@@ -8,6 +8,7 @@ from typing import ClassVar, cast
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import VllmConfig
 from vllm.config.cache import CacheDType
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import get_rope
 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
 from vllm.platforms import current_platform
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.utils.torch_utils import (
     LayerNameType,
     _encode_layer_name,

vllm/models/qwen4_exp/config.py

@@ -1,263 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Qwen4Exp model configuration."""
-
-from typing import Any, ClassVar, cast
-
-from transformers import PreTrainedConfig
-from transformers.models.qwen3_vl.configuration_qwen3_vl import (
-    Qwen3VLVisionConfig,
-)
-
-from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
-
-_QSA_CONFIG_FIELDS = (
-    "indexer_n_heads",
-    "indexer_kv_heads",
-    "indexer_head_dim",
-    "indexer_budget",
-    "indexer_compress_ratio",
-)
-
-# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
-# Older checkpoints label them `full_attention` and mark QSA by setting the
-# indexer fields for the whole model.
-QSA_LAYER_TYPE = "qwen_sparse_attention"
-ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
-
-
-class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
-    model_type = "qwen4_exp"
-    base_config_key = "vision_config"
-
-
-class Qwen4ExpTextConfig(Qwen3NextConfig):
-    model_type = "qwen4_exp_text"
-    base_config_key = "text_config"
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        hc_count: int = 4,
-        hc_lowrank: int = 320,
-        ple_layer_ids: list[int] | None = None,
-        ple_embed_dim: int | None = None,
-        ple_conv_kernel_size: int = 4,
-        ngram_size: int = 3,
-        heads_per_ngram: int = 8,
-        ngram_vocab_size_base: int = 20_000_000,
-        make_ngram_vocab_size_divisible_by: int = 128,
-        output_gate_type: str = "sigmoid",
-        rope_parameters: dict[str, Any] | None = None,
-        layer_types: list[str] | None = None,
-        **kwargs: Any,
-    ) -> None:
-        if hc_count <= 1:
-            raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
-
-        if rope_parameters is not None:
-            if kwargs.get("rope_scaling") is None:
-                kwargs["rope_scaling"] = rope_parameters
-            if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
-                kwargs["rope_theta"] = rope_parameters["rope_theta"]
-            if (
-                kwargs.get("partial_rotary_factor") is None
-                and "partial_rotary_factor" in rope_parameters
-            ):
-                kwargs["partial_rotary_factor"] = rope_parameters[
-                    "partial_rotary_factor"
-                ]
-
-        rope_scaling = kwargs.get("rope_scaling")
-        rope_theta = kwargs.get("rope_theta", 10_000.0)
-        super().__init__(layer_types=layer_types, **kwargs)
-
-        normalized_rope_parameters = self.rope_parameters
-        self.rope_scaling = (
-            rope_scaling or rope_parameters or normalized_rope_parameters
-        )
-        self.rope_parameters = rope_parameters or normalized_rope_parameters
-        self.rope_theta = rope_theta
-
-        self.hc_count = hc_count
-        self.hc_lowrank = hc_lowrank
-        self.ple_layer_ids = ple_layer_ids or []
-        self.ple_embed_dim = (
-            self.hidden_size if ple_embed_dim is None else ple_embed_dim
-        )
-        self.ple_conv_kernel_size = ple_conv_kernel_size
-        self.ngram_size = ngram_size
-        self.heads_per_ngram = heads_per_ngram
-        self.ngram_vocab_size_base = ngram_vocab_size_base
-        self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
-        self.output_gate_type = output_gate_type
-
-        self._validate_ple_config()
-        self._validate_ple_layer_ids()
-        self._validate_qsa_config()
-
-    def _validate_ple_config(self) -> None:
-        if self.hc_lowrank <= 0:
-            raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
-        if self.ngram_size < 2:
-            raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
-        if self.heads_per_ngram <= 0:
-            raise ValueError(
-                f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
-            )
-        if self.ple_embed_dim <= 0:
-            raise ValueError(
-                f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
-            )
-        ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
-        if self.ple_embed_dim % ngram_heads:
-            raise ValueError(
-                "ple_embed_dim must be divisible by total ngram heads: "
-                f"{self.ple_embed_dim} % {ngram_heads} != 0"
-            )
-        if self.ple_conv_kernel_size <= 0:
-            raise ValueError(
-                "ple_conv_kernel_size must be positive, got "
-                f"{self.ple_conv_kernel_size}"
-            )
-        if self.ngram_vocab_size_base <= 0:
-            raise ValueError("ngram_vocab_size_base must be positive")
-        if self.make_ngram_vocab_size_divisible_by <= 0:
-            raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
-
-    def _validate_ple_layer_ids(self) -> None:
-        invalid = [
-            layer_id
-            for layer_id in self.ple_layer_ids
-            if not 1 <= int(layer_id) <= self.num_hidden_layers
-        ]
-        if invalid:
-            raise ValueError(
-                "ple_layer_ids are 1-based and must refer to an existing layer; "
-                f"got {invalid} for {self.num_hidden_layers} layers"
-            )
-
-    def _validate_qsa_config(self) -> None:
-        configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
-        if all(value is None for value in configured.values()):
-            if QSA_LAYER_TYPE in (self.layer_types or ()):
-                raise ValueError(
-                    f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
-                    f"{list(_QSA_CONFIG_FIELDS)}"
-                )
-            return
-
-        missing = [name for name, value in configured.items() if value is None]
-        if missing:
-            raise ValueError(f"QSA config is missing required fields: {missing}")
-
-        values = {name: int(cast(int, value)) for name, value in configured.items()}
-        if any(value <= 0 for value in values.values()):
-            raise ValueError(f"QSA config values must be positive: {values}")
-        if values["indexer_kv_heads"] != 1:
-            raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
-        if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
-            raise ValueError(
-                "indexer_budget must be divisible by indexer_compress_ratio"
-            )
-        block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
-        if block_topk not in (512, 2048):
-            raise ValueError(
-                "QSA requires indexer_budget / indexer_compress_ratio "
-                f"to be 512 or 2048, got {block_topk}"
-            )
-        rotary_dim = int(self.head_dim * self.partial_rotary_factor)
-        if rotary_dim > values["indexer_head_dim"]:
-            raise ValueError(
-                "QSA indexer_head_dim must cover the attention rotary "
-                f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
-            )
-
-    @property
-    def layers_block_type(self) -> list[str]:
-        return [
-            "attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
-            for layer_type in self.layer_types
-        ]
-
-    @property
-    def short_conv_layer_ids(self) -> list[int]:
-        if not self.ple_layer_ids:
-            return []
-        return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
-
-    @property
-    def short_conv_state_shape(self) -> tuple[int, int] | None:
-        if not self.short_conv_layer_ids:
-            return None
-        ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
-        ple_channels = self.hidden_size * self.hc_count
-        return ple_channels, ple_state_len
-
-    @property
-    def ngram_context_len(self) -> int:
-        if not self.ple_layer_ids:
-            return 0
-        return max(int(self.ngram_size) - 1, 0)
-
-
-class Qwen4ExpConfig(PreTrainedConfig):
-    model_type = "qwen4_exp"
-    sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
-        "vision_config": Qwen4ExpVisionConfig,
-        "text_config": Qwen4ExpTextConfig,
-    }
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
-        vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
-        image_token_id: int = 248056,
-        video_token_id: int = 248057,
-        vision_start_token_id: int = 248053,
-        vision_end_token_id: int = 248054,
-        tie_word_embeddings: bool = False,
-        rope_parameters: dict[str, Any] | None = None,
-        **kwargs: Any,
-    ) -> None:
-        if text_config is not None:
-            kwargs.pop("split_ngram_parts", None)
-
-        text_kwargs = (
-            dict(kwargs)
-            if text_config is None
-            and "hidden_size" in kwargs
-            and "num_hidden_layers" in kwargs
-            else {}
-        )
-
-        if isinstance(vision_config, dict):
-            self.vision_config = self.sub_configs["vision_config"](**vision_config)
-        elif vision_config is None:
-            self.vision_config = self.sub_configs["vision_config"]()
-        else:
-            self.vision_config = vision_config
-
-        if isinstance(text_config, dict):
-            self.text_config = self.sub_configs["text_config"](**text_config)
-        elif text_config is None:
-            self.text_config = self.sub_configs["text_config"](**text_kwargs)
-        else:
-            self.text_config = text_config
-
-        self.image_token_id = image_token_id
-        self.video_token_id = video_token_id
-        self.vision_start_token_id = vision_start_token_id
-        self.vision_end_token_id = vision_end_token_id
-        self.rope_parameters = rope_parameters or getattr(
-            self.text_config, "rope_parameters", {}
-        )
-        super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
-
-
-__all__ = [
-    "Qwen4ExpConfig",
-    "Qwen4ExpTextConfig",
-    "Qwen4ExpVisionConfig",
-]

vllm/models/qwen4_exp/nvidia/indexer_qsa.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import VllmConfig
 from vllm.forward_context import get_forward_context
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
 from vllm.model_executor.layers.linear import ReplicatedLinear
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 
 from ..common.qsa_cache import (
     QSACompressedKeyCache,

vllm/models/qwen4_exp/nvidia/model.py

@@ -7,6 +7,7 @@ from itertools import islice
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
 
 from vllm.config import VllmConfig
 from vllm.distributed import get_pp_group
@@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import (
 )
 from vllm.model_executor.models.qwen3_next import (
     Qwen3NextAttention,
-    Qwen3NextMLP,
     Qwen3NextSparseMoeBlock,
 )
 from vllm.model_executor.models.qwen3_vl import (
@@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.inputs import MultiModalFeatureSpec
 from vllm.sequence import IntermediateTensors
 from vllm.tokenizers.registry import cached_tokenizer_from_config
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
 from vllm.v1.kv_cache_interface import MambaSpec
 
-from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
 from .hyperconnection import GatedResidual, HyperConnectionConfig
 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
 from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
@@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
     ) -> None:
         super().__init__()
         config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
-        model_config = vllm_config.model_config
-        cache_config = vllm_config.cache_config
         quant_config = vllm_config.quant_config
 
         self.config = config
@@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
                 prefix=f"{prefix}.linear_attn",
                 gqa_interleaved_layout=False,
             )
-        elif layer_type in ATTENTION_LAYER_TYPES:
+        elif layer_type == "qwen_sparse_attention":
-            use_qsa = (
+            use_qsa = getattr(config, "indexer_n_heads", None) is not None
-                layer_type == QSA_LAYER_TYPE
-                or getattr(config, "indexer_n_heads", None) is not None
-            )
             if not use_qsa:
                 self.self_attn = Qwen3NextAttention(
                     config,
-                    model_config=model_config,
+                    model_config=vllm_config.model_config,
-                    cache_config=cache_config,
+                    cache_config=vllm_config.cache_config,
                     quant_config=quant_config,
                     prefix=f"{prefix}.self_attn",
                 )
@@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
         else:
             raise ValueError(f"Invalid layer_type {layer_type}")
 
-        mlp_only_layers = getattr(config, "mlp_only_layers", [])
+        self.mlp = Qwen4ExpSparseMoeBlock(
-        num_experts = getattr(config, "num_experts", 0) or 0
+            vllm_config=vllm_config, prefix=f"{prefix}.mlp"
-        absolute_layer_id = self.layer_idx + 1
-        is_moe_layer = self.layer_idx not in mlp_only_layers and (
-            num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
         )
-        if is_moe_layer:
-            self.mlp = Qwen4ExpSparseMoeBlock(
-                vllm_config=vllm_config, prefix=f"{prefix}.mlp"
-            )
-        else:
-            self.mlp = Qwen3NextMLP(
-                hidden_size=config.hidden_size,
-                intermediate_size=config.intermediate_size,
-                hidden_act=config.hidden_act,
-                quant_config=quant_config,
-                prefix=f"{prefix}.mlp",
-            )
 
         hc_config = HyperConnectionConfig(
             hc_count=config.hc_count,
@@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
 
         if self.layer_type == "linear_attention":
             attn_out = self.linear_attn(hidden_states=block_input)
-        elif self.layer_type in ATTENTION_LAYER_TYPES:
+        elif self.layer_type == "qwen_sparse_attention":
             attn_out = self.self_attn(
                 hidden_states=block_input,
                 positions=positions,
@@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module):
         self._qsa_layer_ids = frozenset(
             layer_idx
             for layer_idx, layer_type in enumerate(config.layer_types)
-            if layer_type == QSA_LAYER_TYPE
+            if layer_type == "qwen_sparse_attention"
-            or (
+            and getattr(config, "indexer_n_heads", None) is not None
-                layer_type == "full_attention"
-                and getattr(config, "indexer_n_heads", None) is not None
-            )
         )
         self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
 

vllm/models/qwen4_exp/nvidia/mtp.py

@@ -18,6 +18,7 @@ from collections.abc import Iterable
 import regex as re
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import VllmConfig, replace, set_current_vllm_config
 from vllm.distributed import get_pp_group
@@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import (
     maybe_prefix,
 )
 from vllm.sequence import IntermediateTensors
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 
 from .hyperconnection import GatedResidual, HyperConnectionConfig
 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
             self.layers = nn.ModuleList(
                 Qwen4ExpDecoderLayer(
                     draft_vllm_config,
-                    layer_type="full_attention",
+                    layer_type="qwen_sparse_attention",
                     prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
                 )
                 for idx in range(self.num_mtp_layers)

vllm/models/qwen4_exp/nvidia/ngram_embedding.py

@@ -6,6 +6,7 @@ from collections.abc import Iterable
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.config import get_current_vllm_config
 from vllm.logger import init_logger
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import (
     QuantizationConfig,
 )
 from vllm.model_executor.models.utils import AutoWeightsLoader
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 
 from ..common.ngram_embedding import (
     Qwen4ExpPLEDeviceEmbedding,

vllm/models/qwen4_exp/nvidia/ple_layer.py

@@ -6,6 +6,7 @@ from collections.abc import Sequence
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
 from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
@@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
     MambaStateShapeCalculator,
     is_conv_state_dim_first,
 )
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
 from vllm.v1.attention.backends.short_conv_attn import (
     PleShortConvAttentionBackend,

vllm/models/qwen4_exp/nvidia/qsa.py

@@ -8,6 +8,7 @@ from typing import ClassVar, cast
 
 import torch
 from torch import nn
+from transformers import Qwen4ExpTextConfig
 
 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
 from vllm.config import VllmConfig
@@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro
 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
 from vllm.platforms import current_platform
 from vllm.platforms.interface import DeviceCapability
-from vllm.transformers_utils.configs.qwen4_exp import (
-    Qwen4ExpTextConfig,
-)
 from vllm.utils.torch_utils import (
     kv_cache_dtype_str_to_dtype,
 )

vllm/multimodal/video.py

@@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend):
 class Glm5NextVideoBackend(VideoBackend):
     """GLM-5.3-Flash fps-interval video backend.
 
-    Selects frames with the same ``glm_sample_frame_indices`` sampler the
+    Selects frames with the same sampler the processor falls back to, so only
-    processor falls back to, so only the sampled frames are
+    the sampled frames are materialized. ``fps_interval`` semantics (default
-    materialized. ``fps_interval`` semantics (default 2.0) with a
+    2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048,
-    temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
+    temporal pairs kept even. Request overrides: ``fps`` -> fps interval,
-    pairs kept even. Request overrides: ``fps`` -> fps interval,
     ``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
     """
 
     _SEEK_GAP_THRESHOLD: ClassVar[int] = 64
+    _DEFAULT_FPS: ClassVar[float] = 2.0
+    _DEFAULT_MAX_FRAMES: ClassVar[int] = 2048
+
+    @classmethod
+    def _sample_frame_indices(
+        cls,
+        total_frames: int,
+        fps: float,
+        duration: float,
+        *,
+        target_fps: float | None = None,
+        max_frame_count: int | None = None,
+        temporal_patch_size: int = 2,
+    ) -> list[int]:
+        """GLM video frame sampling (training-reference parity).
+
+        ``target_fps`` is the ``fps_interval`` request knob. The greedy walk
+        advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
+        frame-dense sources it collects more candidates than ``extract_t`` and
+        the ``> extract_t`` fixup re-spreads the picks uniformly with
+        ``np.linspace`` -- that fallback is the intended reference behavior, not
+        an accident. Short clips (fewer frames than ``extract_t``) are spread at
+        evenly spaced timestamps (``floor`` sampling; the linspace variant
+        samples frames unevenly and cost 4 points on video grounding evals).
+        Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
+        -> frame cap.
+        """
+        max_frame_idx = total_frames - 1
+        if not duration:
+            duration = (round(max_frame_idx / fps) + 1) if fps else 0
+        if max_frame_count is None:
+            max_frame_count = cls._DEFAULT_MAX_FRAMES
+        if target_fps is None:
+            target_fps = cls._DEFAULT_FPS
+
+        extract_t = int(duration * target_fps)
+        extract_t = min(extract_t, int(max_frame_count))
+
+        duration_per_frame = 1 / fps
+        timestamps = [i * duration_per_frame for i in range(total_frames)]
+        max_second = int(duration)
+
+        if total_frames < extract_t:
+            frame_indices = [
+                math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
+            ]
+        else:
+            frame_indices = []
+            current_second = 0.0
+            inv_fps = 1 / (temporal_patch_size * target_fps)
+            for frame_index in range(total_frames):
+                if timestamps[frame_index] >= current_second:
+                    current_second += inv_fps
+                    frame_indices.append(frame_index)
+                    if current_second >= max_second:
+                        break
+
+        if len(frame_indices) < extract_t:
+            if len(frame_indices) == 0:
+                start, end = 0, max(total_frames - 1, 0)
+            else:
+                start, end = frame_indices[0], frame_indices[-1]
+            frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
+        elif len(frame_indices) > extract_t:
+            frame_indices = np.linspace(
+                0, total_frames - 1, extract_t, dtype=int
+            ).tolist()
+
+        seen, uniq = set(), []
+        for idx in frame_indices:
+            if idx not in seen:
+                seen.add(idx)
+                uniq.append(int(idx))
+
+        if len(uniq) & 1:
+            uniq.append(uniq[-1])
+
+        return uniq
 
     @classmethod
     def compute_frames_index_to_sample(
@@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend):
         target: VideoTargetMetadata,
         **kwargs,
     ) -> list[int]:
-        # Lazy import: the processor module sits behind the
+        return cls._sample_frame_indices(
-        # transformers_utils package init, which multimodal must not pull in.
-        from vllm.transformers_utils.processors.glm5next import (
-            glm_sample_frame_indices,
-        )
-
-        return glm_sample_frame_indices(
             source.total_frames_num,
             source.original_fps,
             source.duration or 0,

vllm/transformers_utils/config.py

@@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
     k3_dspark="K3DSparkConfig",
     funaudiochat="FunAudioChatConfig",
     granite4_vision="Granite4VisionConfig",
-    glm5_next="Glm5NextConfig",
-    glm5_next_text="Glm5NextTextConfig",
-    glm5_next_vision="Glm5NextVisionConfig",
     hyperclovax="HyperCLOVAXConfig",
     hy_v3="HYV3Config",
     hy_v4="HYV4Config",
@@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
     qianfan_ocr="QianfanOCRConfig",
     qwen3_asr="Qwen3ASRConfig",
     qwen3_next="Qwen3NextConfig",
-    qwen4_exp="Qwen4ExpConfig",
-    qwen4_exp_text="Qwen4ExpTextConfig",
     qwen3_5="Qwen3_5Config",
     qwen3_5_text="Qwen3_5TextConfig",
     qwen3_5_moe="Qwen3_5MoeConfig",

vllm/transformers_utils/configs/__init__.py

@@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
     "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
     "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
     "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
-    "Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next",
-    "Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next",
-    "Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next",
     "HYV3Config": "vllm.transformers_utils.configs.hy_v3",
     "HYV4Config": "vllm.transformers_utils.configs.hy_v4",
     "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
@@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
     "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
     "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
     "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
-    "Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp",
-    "Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp",
-    "Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp",
     "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
     "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
     "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
@@ -140,9 +134,6 @@ __all__ = [
     "FunAudioChatConfig",
     "FunAudioChatAudioEncoderConfig",
     "Granite4VisionConfig",
-    "Glm5NextConfig",
-    "Glm5NextTextConfig",
-    "Glm5NextVisionConfig",
     "HYV3Config",
     "HYV4Config",
     "HyperCLOVAXConfig",
@@ -187,9 +178,6 @@ __all__ = [
     "QianfanOCRVisionConfig",
     "Qwen3ASRConfig",
     "Qwen3NextConfig",
-    "Qwen4ExpConfig",
-    "Qwen4ExpTextConfig",
-    "Qwen4ExpVisionConfig",
     "Qwen3_5Config",
     "Qwen3_5TextConfig",
     "Qwen3_5MoeConfig",

vllm/transformers_utils/configs/glm5_next.py

@@ -1,420 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from transformers.configuration_utils import PreTrainedConfig
-
-from vllm.logger import init_logger
-
-logger = init_logger(__name__)
-
-
-class Glm5NextTextConfig(PreTrainedConfig):
-    model_type = "glm5_next_text"
-    base_config_key = "text_config"
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        model_type="glm5_next_text",
-        vocab_size: int = 154880,
-        hidden_size: int = 4096,
-        head_dim: int | None = None,
-        intermediate_size: int = 12288,
-        num_hidden_layers: int = 45,
-        num_attention_heads: int = 64,
-        num_key_value_heads: int | None = None,
-        hidden_act: str = "silu",
-        rms_norm_eps: float = 1e-5,
-        pad_token_id: int | None = 151329,
-        bos_token_id: int | None = None,
-        eos_token_id: int | list[int] | None = None,
-        rope_parameters: dict | None = None,
-        max_position_embeddings: int = 1048576,
-        tie_word_embeddings: bool = False,
-        moe_intermediate_size: int = 2048,
-        moe_renormalize: bool = True,
-        scoring_func: str = "sigmoid",
-        n_routed_experts: int | None = 288,
-        num_experts_per_token: int = 7,
-        n_shared_experts: int = 1,
-        routed_scaling_factor: float = 2.5,
-        topk_method: str | None = None,
-        first_k_dense_replace: int = 0,
-        moe_layer_freq: int = 1,
-        use_grouped_topk: bool = True,
-        n_group: int = 1,
-        topk_group: int = 1,
-        mla: bool = True,
-        q_lora_rank: int | None = 1536,
-        kv_lora_rank: int | None = 512,
-        qk_nope_head_dim: int = 256,
-        qk_rope_head_dim: int = 0,
-        v_head_dim: int | None = 256,
-        mla_nope: bool | None = True,
-        num_nextn_predict_layers: int = 1,
-        # Per-layer layout: "linear_attention" | "deepseek_sparse_attention"
-        layer_types: list[str] | None = None,
-        # Per-layer MLP: "dense" | "sparse"
-        mlp_layer_types: list[str] | None = None,
-        # Linear-attention (KDA) head config (flattened from the old
-        # linear_attn_config dict).
-        linear_head_dim: int = 128,
-        linear_num_heads: int = 64,
-        linear_conv_kernel_dim: int = 4,
-        linear_lower_bound: float = -5.0,
-        index_head_dim: int | None = None,
-        index_topk: int | None = None,
-        index_n_heads: int | None = None,
-        index_dsa_use_layernorm: bool = True,
-        index_kpool_compress: bool = True,
-        # Every ``index_kpool`` indexer K entries pool into one stored entry
-        # (compress_ratio). The 300B checkpoint ships 4; topk runs at pool
-        # granularity (select_k = index_topk // index_kpool).
-        index_kpool: int | None = 4,
-        index_kpool_always_select_tail: bool = True,
-        indexer_rope_interleave: bool = False,
-        mhc: bool | None = True,
-        mhc_num_residual_streams: int = 4,
-        hc_eps: float | None = 1e-06,
-        mhc_tau: float = 0.05,
-        hres_vwnstyle: bool | None = True,
-        mhc_no_norm_weight: bool | None = False,
-        mhc_sinkhorn_iterations: int | None = 20,
-        mhc_post_mult_value: float | None = 2.0,
-        swiglu_limit: float | None = None,
-        logit_scale: float = 1.0,
-        **kwargs,
-    ):
-        # Preserve checkpoint field names and local aliases because their
-        # consumers use different spellings.
-        num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token)
-        moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize)
-        mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams)
-        mhc_sinkhorn_iterations = kwargs.get(
-            "hc_sinkhorn_iters", mhc_sinkhorn_iterations
-        )
-        # Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this
-        # alias self.mla_nope silently stays at the param default.
-        mla_nope = kwargs.get("mla_use_nope", mla_nope)
-        # Checkpoints ship the KDA head config as the ``linear_attn_config``
-        # dict (head_dim / num_heads / short_conv_kernel_size /
-        # gate_lower_bound) rather than the flattened top-level fields; fold it
-        # in so the trained values are read instead of the param defaults
-        # (which only match this checkpoint by coincidence).
-        linear_cfg = kwargs.get("linear_attn_config") or {}
-        if linear_cfg:
-            linear_head_dim = linear_cfg.get("head_dim", linear_head_dim)
-            linear_num_heads = linear_cfg.get("num_heads", linear_num_heads)
-            linear_conv_kernel_dim = linear_cfg.get(
-                "short_conv_kernel_size", linear_conv_kernel_dim
-            )
-            linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound)
-
-        if index_topk is not None:
-            if index_dsa_use_layernorm is not True:
-                raise NotImplementedError(
-                    "GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True"
-                )
-            if index_kpool_compress is not True:
-                raise NotImplementedError(
-                    "GLM-5.3 sparse indexer requires index_kpool_compress=True"
-                )
-            if index_kpool_always_select_tail is not True:
-                raise NotImplementedError(
-                    "GLM-5.3 sparse indexer requires "
-                    "index_kpool_always_select_tail=True"
-                )
-
-        if mhc:
-            if hres_vwnstyle is not True:
-                raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True")
-            if mhc_no_norm_weight not in (False, None):
-                raise NotImplementedError(
-                    "GLM-5.3 mHC requires mhc_no_norm_weight=False"
-                )
-
-        self.model_type = model_type
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.head_dim = (
-            head_dim if head_dim is not None else hidden_size // num_attention_heads
-        )
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-
-        # for backward compatibility
-        if num_key_value_heads is None:
-            num_key_value_heads = num_attention_heads
-
-        self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
-        self.rms_norm_eps = rms_norm_eps
-        self.max_position_embeddings = max_position_embeddings
-        self.rope_parameters = rope_parameters
-
-        # mla config
-        self.mla = mla
-        self.q_lora_rank = q_lora_rank
-        self.kv_lora_rank = kv_lora_rank
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.mla_nope = mla_nope
-        # moe config
-        self.n_routed_experts = n_routed_experts
-        self.num_experts_per_token = num_experts_per_token
-        self.moe_renormalize = moe_renormalize
-        self.n_shared_experts = n_shared_experts
-        self.routed_scaling_factor = routed_scaling_factor
-        self.topk_method = topk_method
-        self.scoring_func = scoring_func
-        assert self.scoring_func in ("softmax", "sigmoid")
-        self.moe_intermediate_size = moe_intermediate_size
-        self.first_k_dense_replace = first_k_dense_replace
-        self.moe_layer_freq = moe_layer_freq
-        self.use_grouped_topk = use_grouped_topk
-        self.n_group = n_group
-        self.topk_group = topk_group
-        self.num_nextn_predict_layers = num_nextn_predict_layers
-
-        # Per-layer attention / MLP layout. Normalize mlp_layer_types from
-        # first_k_dense_replace when the new-schema field is absent so layer
-        # construction sees a consistent layout (mirrors cohere2_moe).
-        self.layer_types = layer_types
-        if mlp_layer_types is None:
-            n = self.num_hidden_layers
-            if first_k_dense_replace is not None:
-                mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * (
-                    n - first_k_dense_replace
-                )
-            else:
-                mlp_layer_types = ["sparse"] * n
-        self.mlp_layer_types = mlp_layer_types
-
-        # Linear-attention (KDA) head config.
-        self.linear_head_dim = linear_head_dim
-        self.linear_num_heads = linear_num_heads
-        self.linear_conv_kernel_dim = linear_conv_kernel_dim
-        self.linear_lower_bound = linear_lower_bound
-
-        # dsa index config
-        self.index_head_dim = index_head_dim
-        self.index_topk = index_topk
-        self.index_n_heads = index_n_heads
-        self.index_dsa_use_layernorm = index_dsa_use_layernorm
-        self.index_kpool_compress = index_kpool_compress
-        self.index_kpool = index_kpool
-        self.index_kpool_always_select_tail = index_kpool_always_select_tail
-        self.indexer_rope_interleave = indexer_rope_interleave
-
-        # mhc config
-        self.mhc = mhc
-        self.mhc_num_residual_streams = mhc_num_residual_streams
-        self.mhc_tau = mhc_tau
-        self.hres_vwnstyle = hres_vwnstyle
-        self.hc_eps = hc_eps
-        self.mhc_no_norm_weight = mhc_no_norm_weight
-        self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
-        self.mhc_post_mult_value = mhc_post_mult_value
-
-        self.swiglu_limit = swiglu_limit
-        self.logit_scale = logit_scale
-
-        super().__init__(
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            tie_word_embeddings=tie_word_embeddings,
-            **kwargs,
-        )
-
-    @property
-    def is_mla(self):
-        return (
-            self.q_lora_rank is not None
-            or self.kv_lora_rank is not None
-            or self.qk_nope_head_dim is not None
-            or self.qk_rope_head_dim is not None
-            or self.v_head_dim is not None
-            or self.mla_nope is True
-        )
-
-    @property
-    def is_moe(self):
-        return self.n_routed_experts is not None
-
-    @property
-    def is_linear_attn(self) -> bool:
-        return self.layer_types is not None and any(
-            t == "linear_attention" for t in self.layer_types
-        )
-
-    def is_kda_layer(self, layer_idx: int):
-        return (
-            self.layer_types is not None
-            and layer_idx < len(self.layer_types)
-            and self.layer_types[layer_idx] == "linear_attention"
-        )
-
-    @property
-    def layers_block_type(self):
-        # Map the schema's per-layer types onto the block strings vLLM's hybrid
-        # accounting (get_num_layers_by_block_type) recognizes: linear-attention
-        # layers stay "linear_attention"; every other attention variant collapses
-        # to "attention".
-        if self.layer_types is None:
-            return ["attention"] * self.num_hidden_layers
-        return [
-            "linear_attention" if t == "linear_attention" else "attention"
-            for t in self.layer_types
-        ]
-
-
-class Glm5NextVisionConfig(PreTrainedConfig):
-    model_type = "glm5_next_vision"
-    base_config_key = "vision_config"
-
-    def __init__(
-        self,
-        depth: int = 24,
-        hidden_size: int = 1024,
-        hidden_act: str = "silu",
-        image_size: int = 448,
-        intermediate_size: int = 4096,
-        num_heads: int = 16,
-        out_hidden_size: int = 4096,
-        projection_intermediate_size: int = 10240,
-        in_channels: int = 3,
-        initializer_range: float = 0.02,
-        patch_size: int = 14,
-        rms_norm_eps: float = 1e-5,
-        spatial_merge_size: int = 2,
-        temporal_patch_size: int = 2,
-        attention_dropout: float = 0.0,
-        attention_bias: bool = True,
-        swiglu_limit: float | None = None,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-
-        self.depth = depth
-        self.hidden_size = hidden_size
-        self.hidden_act = hidden_act
-        self.image_size = image_size
-        self.intermediate_size = intermediate_size
-        self.num_heads = num_heads
-        self.out_hidden_size = out_hidden_size
-        # GLM-5.3-Flash merger bottleneck width (absent from the generic
-        # GLM-OCR vision config); the tower uses it as the PatchMerger
-        # context_dim instead of text_config.intermediate_size.
-        self.projection_intermediate_size = projection_intermediate_size
-        self.in_channels = in_channels
-        self.initializer_range = initializer_range
-        self.patch_size = patch_size
-        # GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5,
-        # but the vision tower was trained with 1e-6. Serving with 1e-5 drifts
-        # the RMSNorm and produces repetitive/degraded image descriptions, so
-        # force the trained value regardless of the checkpoint field.
-        self.rms_norm_eps = 1e-6
-        self.spatial_merge_size = spatial_merge_size
-        self.temporal_patch_size = temporal_patch_size
-        self.attention_dropout = attention_dropout
-        self.attention_bias = attention_bias
-        self.swiglu_limit = swiglu_limit
-
-
-class Glm5NextConfig(PreTrainedConfig):
-    model_type = "glm5_next"
-    sub_configs = {
-        "vision_config": Glm5NextVisionConfig,
-        "text_config": Glm5NextTextConfig,
-    }
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        text_config=None,
-        vision_config=None,
-        image_token_id: int = 154854,
-        video_token_id: int = 154855,
-        image_start_token_id: int = 154830,
-        image_end_token_id: int = 154831,
-        video_start_token_id: int = 154832,
-        video_end_token_id: int = 154833,
-        **kwargs,
-    ):
-        # Init super() first so base-class defaults don't clobber text-config
-        # values set below (PreTrainedConfig has many text-related defaults
-        # that differ from Glm5NextTextConfig).
-        super().__init__(**kwargs)
-
-        if isinstance(vision_config, dict):
-            self.vision_config = self.sub_configs["vision_config"](**vision_config)
-        elif vision_config is None:
-            self.vision_config = self.sub_configs["vision_config"]()
-        else:
-            self.vision_config = vision_config
-
-        if isinstance(text_config, dict):
-            self.text_config = self.sub_configs["text_config"](**text_config)
-        elif text_config is None:
-            # Backward compatibility: a flat top-level checkpoint (no nested
-            # text_config) folds its text fields into Glm5NextTextConfig.
-            self.text_config = self.sub_configs["text_config"](**kwargs)
-        else:
-            self.text_config = text_config
-
-        self.image_token_id = image_token_id
-        self.video_token_id = video_token_id
-        self.image_start_token_id = image_start_token_id
-        self.image_end_token_id = image_end_token_id
-        self.video_start_token_id = video_start_token_id
-        self.video_end_token_id = video_end_token_id
-
-        # Mirror attention implementation recursively onto sub-configs.
-        self._attn_implementation = kwargs.pop("attn_implementation", None)
-
-    # Config-metadata fields that belong to the top-level (multimodal) config
-    # and must NOT be mirrored onto text_config: ``architectures`` /
-    # ``torch_dtype`` differ between the top-level config and the text
-    # sub-config, and mirroring them makes the top-level ``architectures``
-    # silently read back as None (PreTrainedConfig initializes both to None),
-    # which then fails model-class resolution ("No model architectures are
-    # specified").
-    _UNMIRRORED_KEYS = [
-        "_name_or_path",
-        "model_type",
-        "dtype",
-        "torch_dtype",
-        "architectures",
-        "_attn_implementation_internal",
-    ]
-
-    def __setattr__(self, key, value):
-        unmirrored = type(self)._UNMIRRORED_KEYS
-        if (
-            (text_config := super().__getattribute__("__dict__").get("text_config"))
-            is not None
-            and key not in unmirrored
-            and key in text_config.__dict__
-        ):
-            setattr(text_config, key, value)
-        else:
-            super().__setattr__(key, value)
-
-    def __getattribute__(self, key):
-        unmirrored = type(self)._UNMIRRORED_KEYS
-        if (
-            "text_config" in super().__getattribute__("__dict__")
-            and key not in unmirrored
-        ):
-            text_config = super().__getattribute__("text_config")
-            # Forward both instance attributes AND class-defined properties/
-            # methods of the text config, so a flat text-only checkpoint
-            # (model_type "glm5_next", no nested text_config) sees is_moe /
-            # is_kda_layer / layers_block_type like a Glm5NextTextConfig.
-            if key in text_config.__dict__ or key in type(text_config).__dict__:
-                return getattr(text_config, key)
-
-        return super().__getattribute__(key)

vllm/transformers_utils/configs/qwen4_exp.py

@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Exports for the package-owned Qwen4Exp configs."""
-
-from vllm.models.qwen4_exp.config import (
-    Qwen4ExpConfig,
-    Qwen4ExpTextConfig,
-    Qwen4ExpVisionConfig,
-)
-
-__all__ = [
-    "Qwen4ExpConfig",
-    "Qwen4ExpTextConfig",
-    "Qwen4ExpVisionConfig",
-]

vllm/transformers_utils/processors/__init__.py

@@ -17,7 +17,6 @@ __all__ = [
     "FireRedASR2Processor",
     "FunASRProcessor",
     "GLM4VProcessor",
-    "Glm5NextProcessor",
     "Granite4VisionProcessor",
     "H2OVLProcessor",
     "Moondream3Processor",
@@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
     "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
     "FunASRProcessor": "vllm.transformers_utils.processors.funasr",
     "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
-    "Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
     "Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
     "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
     "InternVLProcessor": "vllm.transformers_utils.processors.internvl",

vllm/transformers_utils/processors/glm5next.py

@@ -1,953 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""vLLM-native multimodal processor for GLM-5.3-Flash."""
-
-import math
-
-import numpy as np
-import torch
-from torchvision.transforms.v2 import functional as tvF
-from transformers.image_processing_backends import (
-    TorchvisionBackend,
-    group_images_by_shape,
-    reorder_images,
-)
-from transformers.image_processing_utils import BatchFeature
-from transformers.image_utils import (
-    OPENAI_CLIP_MEAN,
-    OPENAI_CLIP_STD,
-    ChannelDimension,
-    ImageInput,
-    PILImageResampling,
-    SizeDict,
-    get_image_size,
-)
-from transformers.models.auto.image_processing_auto import get_image_processor_config
-from transformers.processing_utils import (
-    ImagesKwargs,
-    MultiModalData,
-    ProcessingKwargs,
-    ProcessorMixin,
-    Unpack,
-    VideosKwargs,
-)
-from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
-from transformers.utils import TensorType, logging
-from transformers.video_processing_utils import BaseVideoProcessor
-from transformers.video_utils import (
-    VideoInput,
-    VideoMetadata,
-    group_videos_by_shape,
-    reorder_videos,
-)
-
-from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
-
-logger = logging.get_logger(__name__)
-
-# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
-# starving the KV cache. Image inputs retain their checkpoint-defined budget.
-_MAX_VIDEO_TOKENS = 30000
-
-# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
-# max_frame_count_dynamic=2048); used when neither the request nor the
-# processor config overrides them.
-GLM_VIDEO_DEFAULT_FPS = 2.0
-GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
-
-
-def glm_sample_frame_indices(
-    total_frames: int,
-    fps: float,
-    duration: float,
-    *,
-    target_fps: float | None = None,
-    max_frame_count: int | None = None,
-    temporal_patch_size: int = 2,
-) -> list[int]:
-    """GLM video frame sampling (training-reference parity).
-
-    ``target_fps`` is the ``fps_interval`` request knob. The greedy walk
-    advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
-    frame-dense sources it collects more candidates than ``extract_t`` and
-    the ``> extract_t`` fixup re-spreads the picks uniformly with
-    ``np.linspace`` -- that fallback is the intended reference behavior, not
-    an accident. Short clips (fewer frames than ``extract_t``) are spread at
-    evenly spaced timestamps (``floor`` sampling; the linspace variant
-    samples frames unevenly and cost 4 points on video grounding evals).
-    Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
-    -> frame cap.
-    """
-    max_frame_idx = total_frames - 1
-    if not duration:
-        duration = (round(max_frame_idx / fps) + 1) if fps else 0
-    if max_frame_count is None:
-        max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
-    if target_fps is None:
-        target_fps = GLM_VIDEO_DEFAULT_FPS
-
-    extract_t = int(duration * target_fps)
-    extract_t = min(extract_t, int(max_frame_count))
-
-    duration_per_frame = 1 / fps
-    timestamps = [i * duration_per_frame for i in range(total_frames)]
-    max_second = int(duration)
-
-    if total_frames < extract_t:
-        frame_indices = [
-            math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
-        ]
-    else:
-        frame_indices = []
-        current_second = 0.0
-        inv_fps = 1 / (temporal_patch_size * target_fps)
-        for frame_index in range(total_frames):
-            if timestamps[frame_index] >= current_second:
-                current_second += inv_fps
-                frame_indices.append(frame_index)
-                if current_second >= max_second:
-                    break
-
-    if len(frame_indices) < extract_t:
-        if len(frame_indices) == 0:
-            start, end = 0, max(total_frames - 1, 0)
-        else:
-            start, end = frame_indices[0], frame_indices[-1]
-        frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
-    elif len(frame_indices) > extract_t:
-        frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
-
-    seen, uniq = set(), []
-    for idx in frame_indices:
-        if idx not in seen:
-            seen.add(idx)
-            uniq.append(int(idx))
-
-    if len(uniq) & 1:
-        uniq.append(uniq[-1])
-
-    return uniq
-
-
-def _ceil_to_factor(value: int, factor: int) -> int:
-    """Round a positive integer upward to the nearest multiple of factor."""
-    return math.ceil(value / factor) * factor
-
-
-def _fit_aligned_size_within_budget(
-    t: int,
-    h: int,
-    w: int,
-    h_factor: int,
-    w_factor: int,
-    max_pixels: int,
-) -> tuple[int, int]:
-    """Largest proportional size whose upward-aligned canvas fits the budget.
-
-    Binary search on the unaligned content height; each candidate is rounded
-    upward to h_factor/w_factor, so the returned canvas always satisfies
-    ``t * aligned_h * aligned_w <= max_pixels``.
-    """
-    minimum_pixels = t * h_factor * w_factor
-    if max_pixels < minimum_pixels:
-        raise ValueError(
-            f"max_pixels={max_pixels} is too small. At least "
-            f"{minimum_pixels} pixels are required for one aligned patch."
-        )
-
-    low, high = 1, h
-    best_h, best_w = h_factor, w_factor
-    while low <= high:
-        content_h = (low + high) // 2
-        content_w = max(1, math.floor(w * content_h / h))
-        aligned_h = _ceil_to_factor(content_h, h_factor)
-        aligned_w = _ceil_to_factor(content_w, w_factor)
-        if t * aligned_h * aligned_w <= max_pixels:
-            best_h, best_w = aligned_h, aligned_w
-            low = content_h + 1
-        else:
-            high = content_h - 1
-    return best_h, best_w
-
-
-def smart_resize(
-    t: int,
-    h: int,
-    w: int,
-    t_factor: int = 1,
-    h_factor: int = 28,
-    w_factor: int = 28,
-    min_pixels: int = 56 * 56,
-    max_pixels: int = 14 * 14 * 4 * 1280,
-) -> tuple[int, int]:
-    """GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
-    ``t_bar * h_bar * w_bar`` pixel budget.
-
-    Height/width always round UP to their factors (content is then padded,
-    never cropped or distorted); an over-budget canvas is refit by binary
-    search instead of one-shot square-root scaling. ``h_factor`` /
-    ``w_factor`` carry ``patch_expand_factor`` on top of
-    ``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
-    a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
-    temporal_patch_size``.
-    """
-    if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
-        raise ValueError("Image dimensions and alignment factors must be positive.")
-    if min_pixels <= 0 or max_pixels <= 0:
-        raise ValueError("min_pixels and max_pixels must be positive.")
-    if min_pixels > max_pixels:
-        raise ValueError("min_pixels must be less than or equal to max_pixels.")
-
-    t_bar = max(t_factor, round(t / t_factor) * t_factor)
-    h_bar = _ceil_to_factor(h, h_factor)
-    w_bar = _ceil_to_factor(w, w_factor)
-
-    if t_bar * h_bar * w_bar > max_pixels:
-        h_bar, w_bar = _fit_aligned_size_within_budget(
-            t=t_bar,
-            h=h,
-            w=w,
-            h_factor=h_factor,
-            w_factor=w_factor,
-            max_pixels=max_pixels,
-        )
-    elif t_bar * h_bar * w_bar < min_pixels:
-        beta = math.sqrt(min_pixels / (t * h * w))
-        h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
-        w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
-
-        # Alignment can push a candidate slightly over a tight max_pixels
-        # budget. Refit it when that happens.
-        if t_bar * h_bar * w_bar > max_pixels:
-            h_bar, w_bar = _fit_aligned_size_within_budget(
-                t=t_bar,
-                h=h,
-                w=w,
-                h_factor=h_factor,
-                w_factor=w_factor,
-                max_pixels=max_pixels,
-            )
-
-    return h_bar, w_bar
-
-
-def _get_pad_content_size(
-    image_height: int,
-    image_width: int,
-    canvas_height: int,
-    canvas_width: int,
-    allow_upscale: bool = False,
-) -> tuple[int, int]:
-    """Aspect-ratio-preserving content size that fits the canvas.
-
-    Oversized images are shrunk proportionally. Small images are enlarged
-    only when ``allow_upscale``. Padding is applied after the resize.
-    """
-    scale = min(canvas_height / image_height, canvas_width / image_width)
-    if not allow_upscale:
-        scale = min(1.0, scale)
-    content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
-    content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
-    return content_height, content_width
-
-
-def _resize_or_pad(
-    stacked_images: torch.Tensor,
-    target_height: int,
-    target_width: int,
-    resize_mode: str,
-    resample: "PILImageResampling | tvF.InterpolationMode | int | None",
-    resize,
-    allow_upscale: bool = False,
-) -> torch.Tensor:
-    """Resize onto the aligned canvas, or keep the aspect ratio and
-    zero-pad the right/bottom sides (``resize_mode="pad"``)."""
-    height, width = stacked_images.shape[-2:]
-
-    if resize_mode == "resize":
-        return resize(
-            stacked_images,
-            size=SizeDict(height=target_height, width=target_width),
-            resample=resample,
-        )
-
-    if resize_mode != "pad":
-        raise ValueError("resize_mode must be either 'resize' or 'pad'.")
-
-    content_height, content_width = _get_pad_content_size(
-        image_height=height,
-        image_width=width,
-        canvas_height=target_height,
-        canvas_width=target_width,
-        allow_upscale=allow_upscale,
-    )
-
-    if (content_height, content_width) != (height, width):
-        stacked_images = resize(
-            stacked_images,
-            size=SizeDict(height=content_height, width=content_width),
-            resample=resample,
-        )
-
-    # torchvision padding order: [left, top, right, bottom] -> pad only the
-    # right and bottom sides.
-    return tvF.pad(
-        stacked_images,
-        padding=[0, 0, target_width - content_width, target_height - content_height],
-        fill=0,
-    )
-
-
-def _pixel_budget(
-    min_image_tokens: int | None,
-    max_image_tokens: int | None,
-    patch_size: int,
-    merge_size: int,
-    temporal_patch_size: int,
-) -> tuple[int, int]:
-    """(min_pixels, max_pixels) from the token bounds of
-    ``processor_config.json``; one vision token covers
-    ``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
-    if min_image_tokens is None or max_image_tokens is None:
-        raise ValueError(
-            "min_image_tokens and max_image_tokens must be provided by "
-            "processor_config.json (or per-call kwargs)."
-        )
-    factor = temporal_patch_size * (patch_size * merge_size) ** 2
-    return min_image_tokens * factor, max_image_tokens * factor
-
-
-class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False):  # type: ignore[call-arg]
-    patch_size: int | None
-    temporal_patch_size: int | None
-    merge_size: int | None
-    patch_expand_factor: int | None
-    resize_mode: str | None
-    min_image_tokens: int | None
-    max_image_tokens: int | None
-
-
-class Glm5NextImageProcessor(TorchvisionBackend):
-    """Fast torchvision image processor for GLM-5.3-Flash.
-
-    ``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
-    factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
-    ``"pad"`` (default) preserves the aspect ratio and zero-pads the
-    right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
-    it. Defaults mirror the checkpoint's ``image_processor`` config.
-    """
-
-    do_resize = True
-    resample = PILImageResampling.BICUBIC
-    size = {"longest_edge": 1}  # unused: budgets come from the token bounds
-    do_rescale = True
-    do_normalize = True
-    image_mean = OPENAI_CLIP_MEAN
-    image_std = OPENAI_CLIP_STD
-    do_convert_rgb = True
-    patch_size = 14
-    temporal_patch_size = 2
-    merge_size = 2
-    patch_expand_factor = 1
-    resize_mode = "pad"
-    min_image_tokens = 16
-    max_image_tokens = 8000
-    valid_kwargs = Glm5NextImageProcessorKwargs
-    model_input_names = ["pixel_values", "image_grid_thw"]
-
-    def _preprocess(
-        self,
-        images: list[torch.Tensor],
-        do_resize: bool,
-        size: SizeDict,
-        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
-        do_rescale: bool,
-        rescale_factor: float,
-        do_normalize: bool,
-        image_mean: float | list[float] | None,
-        image_std: float | list[float] | None,
-        patch_size: int,
-        temporal_patch_size: int,
-        merge_size: int,
-        patch_expand_factor: int,
-        resize_mode: str | None,
-        min_image_tokens: int | None,
-        max_image_tokens: int | None,
-        disable_grouping: bool | None,
-        return_tensors: str | TensorType | None,
-        **kwargs,
-    ) -> BatchFeature:
-        resize_mode = resize_mode if resize_mode is not None else self.resize_mode
-        min_pixels, max_pixels = _pixel_budget(
-            min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
-            max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
-            patch_size,
-            merge_size,
-            temporal_patch_size,
-        )
-        grouped_images, grouped_images_index = group_images_by_shape(
-            images, disable_grouping=disable_grouping
-        )
-        resized_images_grouped = {}
-        for shape, stacked_images in grouped_images.items():
-            height, width = stacked_images.shape[-2:]
-            if do_resize:
-                resized_height, resized_width = smart_resize(
-                    t=temporal_patch_size,
-                    h=height,
-                    w=width,
-                    t_factor=temporal_patch_size,
-                    h_factor=patch_size * merge_size * patch_expand_factor,
-                    w_factor=patch_size * merge_size * patch_expand_factor,
-                    min_pixels=min_pixels,
-                    max_pixels=max_pixels,
-                )
-                stacked_images = _resize_or_pad(
-                    stacked_images,
-                    target_height=resized_height,
-                    target_width=resized_width,
-                    resize_mode=resize_mode,
-                    resample=resample,
-                    resize=self.resize,
-                    allow_upscale=(temporal_patch_size * height * width < min_pixels),
-                )
-            resized_images_grouped[shape] = stacked_images
-
-        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
-
-        grouped_images, grouped_images_index = group_images_by_shape(
-            resized_images, disable_grouping=disable_grouping
-        )
-        processed_images_grouped = {}
-        processed_grids = {}
-
-        for shape, stacked_images in grouped_images.items():
-            resized_height, resized_width = stacked_images.shape[-2:]
-
-            patches = self.rescale_and_normalize(
-                stacked_images,
-                do_rescale,
-                rescale_factor,
-                do_normalize,
-                image_mean,
-                image_std,
-            )
-            if patches.ndim == 4:  # (B, C, H, W)
-                patches = patches.unsqueeze(1)  # (B, T=1, C, H, W)
-
-            if patches.shape[1] % temporal_patch_size != 0:
-                repeats = patches[:, -1:].repeat(
-                    1,
-                    temporal_patch_size - (patches.shape[1] % temporal_patch_size),
-                    1,
-                    1,
-                    1,
-                )
-                patches = torch.cat([patches, repeats], dim=1)
-
-            batch_size, t_len, channel = patches.shape[:3]
-            grid_t = t_len // temporal_patch_size
-            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
-
-            patches = patches.view(
-                batch_size,
-                grid_t,
-                temporal_patch_size,
-                channel,
-                grid_h // merge_size,
-                merge_size,
-                patch_size,
-                grid_w // merge_size,
-                merge_size,
-                patch_size,
-            )
-            # (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
-            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
-
-            flatten_patches = patches.reshape(
-                batch_size,
-                grid_t * grid_h * grid_w,
-                channel * temporal_patch_size * patch_size * patch_size,
-            )
-
-            processed_images_grouped[shape] = flatten_patches
-            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
-
-        processed_images = reorder_images(
-            processed_images_grouped, grouped_images_index
-        )
-        processed_grids = reorder_images(processed_grids, grouped_images_index)
-
-        pixel_values = torch.cat(processed_images, dim=0)
-        image_grid_thw = torch.tensor(processed_grids)
-
-        return BatchFeature(
-            data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
-            tensor_type=return_tensors,
-        )
-
-    def preprocess(
-        self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
-    ) -> BatchFeature:
-        return super().preprocess(images, **kwargs)
-
-    def get_number_of_image_patches(
-        self, height: int, width: int, images_kwargs: dict | None = None
-    ) -> int:
-        """Number of image patches (pre-merge) for a given (height, width)."""
-        images_kwargs = images_kwargs or {}
-        patch_size = images_kwargs.get("patch_size", self.patch_size)
-        merge_size = images_kwargs.get("merge_size", self.merge_size)
-        patch_expand_factor = images_kwargs.get(
-            "patch_expand_factor", self.patch_expand_factor
-        )
-        min_pixels, max_pixels = _pixel_budget(
-            images_kwargs.get("min_image_tokens", self.min_image_tokens),
-            images_kwargs.get("max_image_tokens", self.max_image_tokens),
-            patch_size,
-            merge_size,
-            self.temporal_patch_size,
-        )
-        resized_height, resized_width = smart_resize(
-            t=self.temporal_patch_size,
-            h=height,
-            w=width,
-            t_factor=self.temporal_patch_size,
-            h_factor=patch_size * merge_size * patch_expand_factor,
-            w_factor=patch_size * merge_size * patch_expand_factor,
-            min_pixels=min_pixels,
-            max_pixels=max_pixels,
-        )
-        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
-        return grid_h * grid_w
-
-
-class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False):  # type: ignore[call-arg]
-    fps: list[float] | float
-    patch_size: int
-    temporal_patch_size: int
-    merge_size: int
-    patch_expand_factor: int
-    resize_mode: str | None
-    target_fps: float | None
-    max_frames: int | None
-    fps_interval: int | None
-    max_frame_count_dynamic: int | None
-    min_image_tokens: int | None
-    max_image_tokens: int | None
-
-
-class Glm5NextVideoProcessor(BaseVideoProcessor):
-    """Fast video processor for GLM-5.3-Flash.
-
-    Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
-    image processor, and adds GLM-5.3-Flash frame sampling
-    (``glm_sample_frame_indices``: ``fps_interval`` semantics with a
-    temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
-    ``video_processor`` config.
-    """
-
-    resample = PILImageResampling.BICUBIC
-    size = {"longest_edge": 1}  # unused: budgets come from the token bounds
-    image_mean = OPENAI_CLIP_MEAN
-    image_std = OPENAI_CLIP_STD
-    do_resize = True
-    do_rescale = True
-    do_normalize = True
-    do_convert_rgb = True
-    do_sample_frames = True
-    patch_size = 14
-    temporal_patch_size = 2
-    patch_expand_factor = 1
-    merge_size = 2
-    valid_kwargs = Glm5NextVideoProcessorKwargs
-    num_frames = 16
-    fps = 2
-    fps_interval = 2.0
-    max_frame_count_dynamic = 2048
-    resize_mode = "pad"
-    min_image_tokens = 16
-    max_image_tokens = 240000
-    model_input_names = ["pixel_values_videos", "video_grid_thw"]
-
-    def sample_frames(
-        self,
-        metadata: VideoMetadata,
-        fps: int | float | None = None,
-        **kwargs,
-    ) -> np.ndarray:
-        """Sample frame indices with GLM's fps-interval policy.
-
-        ``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
-        ``max_frame_count_dynamic`` are the overrides described in
-        :func:`glm_sample_frame_indices`.
-        """
-        if metadata is None or getattr(metadata, "fps", None) is None:
-            raise ValueError(
-                "Asked to sample frames per second but no video metadata was "
-                "provided which is required when sampling in GLM-5.3-Flash. Please "
-                "pass in `VideoMetadata` object or set `do_sample_frames=False`."
-            )
-
-        target_fps = fps if fps is not None else kwargs.get("target_fps")
-        if target_fps is None:
-            target_fps = self.fps_interval
-        indices = glm_sample_frame_indices(
-            metadata.total_num_frames,
-            metadata.fps,
-            metadata.duration or 0,
-            target_fps=target_fps,
-            max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
-            temporal_patch_size=self.temporal_patch_size,
-        )
-        return np.array(indices)
-
-    def _preprocess(
-        self,
-        videos: list[torch.Tensor],
-        do_convert_rgb: bool = True,
-        do_resize: bool = True,
-        size: SizeDict | None = None,
-        resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
-        do_rescale: bool = True,
-        rescale_factor: float = 1 / 255.0,
-        do_normalize: bool = True,
-        image_mean: float | list[float] | None = None,
-        image_std: float | list[float] | None = None,
-        patch_size: int | None = None,
-        temporal_patch_size: int | None = None,
-        patch_expand_factor: int | None = None,
-        merge_size: int | None = None,
-        resize_mode: str | None = None,
-        min_image_tokens: int | None = None,
-        max_image_tokens: int | None = None,
-        return_tensors: str | TensorType | None = None,
-        **kwargs,
-    ) -> BatchFeature:
-        patch_expand_factor = self.patch_expand_factor
-        patch_size = patch_size if patch_size is not None else self.patch_size
-        temporal_patch_size = (
-            temporal_patch_size
-            if temporal_patch_size is not None
-            else self.temporal_patch_size
-        )
-        merge_size = merge_size if merge_size is not None else self.merge_size
-        resize_mode = resize_mode if resize_mode is not None else self.resize_mode
-        min_pixels, max_pixels = _pixel_budget(
-            min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
-            max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
-            patch_size,
-            merge_size,
-            temporal_patch_size,
-        )
-        grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
-        resized_videos_grouped = {}
-        for shape, stacked_videos in grouped_videos.items():
-            if do_convert_rgb:
-                stacked_videos = self.convert_to_rgb(stacked_videos)
-            b, t_len, c, h, w = stacked_videos.shape
-            num_frames, height, width = t_len, h, w
-            if do_resize:
-                resized_height, resized_width = smart_resize(
-                    t=num_frames,
-                    h=height,
-                    w=width,
-                    t_factor=temporal_patch_size,
-                    h_factor=patch_size * merge_size * patch_expand_factor,
-                    w_factor=patch_size * merge_size * patch_expand_factor,
-                    min_pixels=min_pixels,
-                    max_pixels=max_pixels,
-                )
-                stacked_videos = stacked_videos.view(b * t_len, c, h, w)
-                stacked_videos = _resize_or_pad(
-                    stacked_videos,
-                    target_height=resized_height,
-                    target_width=resized_width,
-                    resize_mode=resize_mode,
-                    resample=resample,
-                    resize=self.resize,
-                    allow_upscale=(num_frames * height * width < min_pixels),
-                )
-                stacked_videos = stacked_videos.view(
-                    b, t_len, c, resized_height, resized_width
-                )
-            resized_videos_grouped[shape] = stacked_videos
-        resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
-
-        grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
-        processed_videos_grouped = {}
-        processed_grids = {}
-        for shape, stacked_videos in grouped_videos.items():
-            resized_height, resized_width = get_image_size(
-                stacked_videos[0], channel_dim=ChannelDimension.FIRST
-            )
-            stacked_videos = self.rescale_and_normalize(
-                stacked_videos,
-                do_rescale,
-                rescale_factor,
-                do_normalize,
-                image_mean,
-                image_std,
-            )
-            patches = stacked_videos
-
-            if pad := -patches.shape[1] % temporal_patch_size:
-                repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
-                patches = torch.cat((patches, repeats), dim=1)
-            batch_size, grid_t, channel = patches.shape[:3]
-            grid_t = grid_t // temporal_patch_size
-            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
-
-            patches = patches.view(
-                batch_size,
-                grid_t,
-                temporal_patch_size,
-                channel,
-                grid_h // merge_size,
-                merge_size,
-                patch_size,
-                grid_w // merge_size,
-                merge_size,
-                patch_size,
-            )
-            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
-            flatten_patches = patches.reshape(
-                batch_size,
-                grid_t * grid_h * grid_w,
-                channel * temporal_patch_size * patch_size * patch_size,
-            )
-
-            processed_videos_grouped[shape] = flatten_patches
-            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
-
-        processed_videos = reorder_videos(
-            processed_videos_grouped, grouped_videos_index
-        )
-        processed_grids = reorder_videos(processed_grids, grouped_videos_index)
-        pixel_values_videos = torch.cat(processed_videos, dim=0)
-        video_grid_thw = torch.tensor(processed_grids)
-        return BatchFeature(
-            data={
-                "pixel_values_videos": pixel_values_videos,
-                "video_grid_thw": video_grid_thw,
-            },
-            tensor_type=return_tensors,
-        )
-
-
-class Glm5NextProcessorKwargs(ProcessingKwargs, total=False):  # type: ignore[call-arg]
-    images_kwargs: Glm5NextImageProcessorKwargs
-    videos_kwargs: Glm5NextVideoProcessorKwargs
-    _defaults = {
-        "text_kwargs": {
-            "padding": False,
-            "return_token_type_ids": False,
-            "return_mm_token_type_ids": False,
-        },
-        "videos_kwargs": {"return_metadata": True},
-    }
-
-
-class Glm5NextProcessor(ProcessorMixin):
-    """Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
-
-    Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
-    frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
-    structure (mrope timestamps).
-    """
-
-    attributes = ["image_processor", "tokenizer", "video_processor"]
-    image_processor_class = "AutoImageProcessor"
-    video_processor_class = "AutoVideoProcessor"
-    tokenizer_class = ("PythonBackend", "TokenizersBackend")
-
-    def __init__(
-        self,
-        image_processor=None,
-        tokenizer=None,
-        video_processor=None,
-        chat_template=None,
-        **kwargs,
-    ) -> None:
-        super().__init__(
-            image_processor, tokenizer, video_processor, chat_template=chat_template
-        )
-        self.image_token = (
-            "<|image|>"
-            if not hasattr(tokenizer, "image_token")
-            else tokenizer.image_token
-        )
-        self.video_token = (
-            "<|video|>"
-            if not hasattr(tokenizer, "video_token")
-            else tokenizer.video_token
-        )
-        self.image_token_id = (
-            tokenizer.image_token_id
-            if getattr(tokenizer, "image_token_id", None)
-            else tokenizer.convert_tokens_to_ids(self.image_token)
-        )
-        self.video_token_id = (
-            tokenizer.video_token_id
-            if getattr(tokenizer, "video_token_id", None)
-            else tokenizer.convert_tokens_to_ids(self.video_token)
-        )
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
-        """Build the processor directly from the checkpoint config.
-
-        GLM-5.3-Flash stores nested image/video configs in
-        ``processor_config.json`` and declares a custom processor class. This
-        method reads those configs directly and caps only the video token budget.
-        """
-        from transformers import AutoTokenizer
-
-        model_path = pretrained_model_name_or_path
-        tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
-
-        def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
-            # Video keeps a serving token cap (the checkpoint's 240k-token
-            # budget would starve the KV cache at startup profiling); images
-            # follow the checkpoint budget verbatim so preprocessing matches
-            # the HF reference exactly.
-            if is_video and cfg.get("max_image_tokens") is not None:
-                cfg["max_image_tokens"] = min(
-                    cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
-                )
-            return cfg
-
-        ip_cfg = _cap_cfg(
-            dict(get_image_processor_config(model_path, **kwargs)), is_video=False
-        )
-        image_processor = Glm5NextImageProcessor(
-            **{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
-        )
-
-        processor_config = get_hf_file_to_dict(
-            "processor_config.json",
-            model_path,
-            revision=kwargs.get("revision", "main"),
-        )
-        if processor_config is None:
-            raise ValueError(f"Missing processor_config.json for {model_path}")
-        vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
-        video_processor = Glm5NextVideoProcessor(
-            **{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
-        )
-
-        return cls(
-            image_processor=image_processor,
-            tokenizer=tokenizer,
-            video_processor=video_processor,
-        )
-
-    def __call__(
-        self,
-        images: ImageInput | None = None,
-        text: TextInput
-        | PreTokenizedInput
-        | list[TextInput]
-        | list[PreTokenizedInput] = None,
-        videos: VideoInput | None = None,
-        **kwargs: Unpack[Glm5NextProcessorKwargs],
-    ) -> BatchFeature:
-        output_kwargs = self._merge_kwargs(
-            Glm5NextProcessorKwargs,
-            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
-            **kwargs,
-        )
-        if images is not None:
-            image_inputs = self.image_processor(
-                images=images, **output_kwargs["images_kwargs"]
-            )
-        else:
-            image_inputs = {}
-
-        if videos is not None:
-            videos_inputs = self.video_processor(
-                videos=videos, **output_kwargs["videos_kwargs"]
-            )
-            if "return_metadata" not in kwargs:
-                videos_inputs.pop("video_metadata")
-        else:
-            videos_inputs = {}
-
-        if not isinstance(text, list):
-            text = [text]
-
-        # Prompt updates expand the unchanged image/video markers after this call.
-        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
-        return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
-            "return_mm_token_type_ids", False
-        )
-        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
-
-        if return_mm_token_type_ids:
-            array_ids = np.array(text_inputs["input_ids"])
-            mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
-            mm_token_type_ids[array_ids == self.image_token_id] = 1
-            text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
-        return BatchFeature(
-            data={**text_inputs, **image_inputs, **videos_inputs},
-            tensor_type=return_tensors,
-        )
-
-    def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
-        vision_data = {}
-        if image_sizes is not None:
-            images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
-            images_kwargs.update(kwargs)
-            merge_size = (
-                images_kwargs.get("merge_size", None) or self.image_processor.merge_size
-            )
-
-            num_image_patches = [
-                self.image_processor.get_number_of_image_patches(
-                    *image_size, images_kwargs
-                )
-                for image_size in image_sizes
-            ]
-            num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
-            vision_data.update(
-                {
-                    "num_image_tokens": num_image_tokens,
-                    "num_image_patches": num_image_patches,
-                }
-            )
-
-        if video_sizes is not None:
-            videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
-            videos_kwargs.update(kwargs)
-            num_video_patches = [
-                self.video_processor.get_number_of_video_patches(
-                    *video_size, videos_kwargs
-                )
-                for video_size in video_sizes
-            ]
-            num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
-            vision_data["num_video_tokens"] = num_video_tokens
-
-        return MultiModalData(**vision_data)
-
-    def post_process_image_text_to_text(
-        self,
-        generated_outputs,
-        skip_special_tokens=True,
-        clean_up_tokenization_spaces=False,
-        **kwargs,
-    ):
-        return self.tokenizer.batch_decode(
-            generated_outputs,
-            skip_special_tokens=skip_special_tokens,
-            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-            **kwargs,
-        )
-
-
-__all__ = [
-    "Glm5NextImageProcessor",
-    "Glm5NextVideoProcessor",
-    "Glm5NextProcessor",
-    "smart_resize",
-]