From e3cae8d2ac6be8dfd319f3bb8061a1ba4d19bf52 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:25:18 +0100 Subject: [PATCH] [Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- requirements/common.txt | 2 +- .../models/glm5next/test_sequence_parallel.py | 2 +- .../multimodal/processing/test_glm5next.py | 49 +- tests/models/qwen4_exp/test_config.py | 10 +- tests/models/qwen4_exp/test_ple.py | 2 +- tests/multimodal/test_video.py | 18 +- tests/test_config.py | 44 + .../processors/test_glm5next.py | 389 ------- tests/transformers_utils/test_config.py | 53 - tests/v1/spec_decode/test_mtp.py | 2 +- vllm/config/model.py | 13 +- vllm/config/speculative.py | 2 +- vllm/models/glm5next/common/attention.py | 32 +- vllm/models/glm5next/common/kda.py | 4 +- vllm/models/glm5next/common/model.py | 100 +- vllm/models/glm5next/common/mtp.py | 8 +- vllm/models/glm5next/common/multimodal.py | 122 ++- vllm/models/qwen4_exp/amd/indexer_qsa.py | 4 +- vllm/models/qwen4_exp/amd/model.py | 47 +- vllm/models/qwen4_exp/amd/mtp.py | 6 +- vllm/models/qwen4_exp/amd/ple_layer.py | 4 +- vllm/models/qwen4_exp/amd/qsa.py | 4 +- vllm/models/qwen4_exp/config.py | 263 ----- vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 4 +- vllm/models/qwen4_exp/nvidia/model.py | 47 +- vllm/models/qwen4_exp/nvidia/mtp.py | 6 +- .../qwen4_exp/nvidia/ngram_embedding.py | 4 +- vllm/models/qwen4_exp/nvidia/ple_layer.py | 4 +- vllm/models/qwen4_exp/nvidia/qsa.py | 4 +- vllm/multimodal/video.py | 95 +- vllm/transformers_utils/config.py | 5 - vllm/transformers_utils/configs/__init__.py | 12 - vllm/transformers_utils/configs/glm5_next.py | 420 -------- vllm/transformers_utils/configs/qwen4_exp.py | 15 - .../transformers_utils/processors/__init__.py | 2 - .../transformers_utils/processors/glm5next.py | 953 ------------------ 36 files changed, 366 insertions(+), 2385 deletions(-) delete mode 100644 tests/transformers_utils/processors/test_glm5next.py delete mode 100644 vllm/models/qwen4_exp/config.py delete mode 100644 vllm/transformers_utils/configs/glm5_next.py delete mode 100644 vllm/transformers_utils/configs/qwen4_exp.py delete mode 100644 vllm/transformers_utils/processors/glm5next.py diff --git a/requirements/common.txt b/requirements/common.txt index eec3c050bc..70bfa93253 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -7,7 +7,7 @@ requests >= 2.33.0 tqdm blake3 py-cpuinfo -transformers >= 5.10.4 +transformers >= 5.16.1 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.21.1 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 diff --git a/tests/models/glm5next/test_sequence_parallel.py b/tests/models/glm5next/test_sequence_parallel.py index 2769f786bb..c0f59d6186 100644 --- a/tests/models/glm5next/test_sequence_parallel.py +++ b/tests/models/glm5next/test_sequence_parallel.py @@ -15,11 +15,11 @@ from types import SimpleNamespace import pytest import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.model_executor import parameter from vllm.model_executor.layers import linear from vllm.models.glm5next.common import model as glm_model -from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2 diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 0e1e7b9876..719154a2b4 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU. import pytest import torch from PIL import Image +from transformers.models.glm5_next.video_processing_glm5_next import ( + Glm5NextVideoProcessor, + smart_resize, +) +from transformers.video_utils import VideoMetadata from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform -from vllm.transformers_utils.processors.glm5next import ( - Glm5NextVideoProcessor, - _pixel_budget, - glm_sample_frame_indices, - smart_resize, -) from ...utils import build_model_context @@ -48,27 +47,16 @@ def _pixel_path_grid( width: int, ) -> tuple[int, int, int]: """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds.""" - min_pixels, max_pixels = _pixel_budget( - video_processor.min_image_tokens, - video_processor.max_image_tokens, - video_processor.patch_size, - video_processor.merge_size, - video_processor.temporal_patch_size, - ) - factor = ( - video_processor.patch_size - * video_processor.merge_size - * video_processor.patch_expand_factor - ) resized_height, resized_width = smart_resize( - t=num_frames, - h=height, - w=width, - t_factor=video_processor.temporal_patch_size, - h_factor=factor, - w_factor=factor, - min_pixels=min_pixels, - max_pixels=max_pixels, + num_frames=num_frames, + height=height, + width=width, + temporal_factor=video_processor.temporal_patch_size, + factor=video_processor.patch_size + * video_processor.merge_size + * video_processor.patch_expand_factor, + min_pixels=video_processor.min_image_tokens, + max_pixels=video_processor.max_image_tokens, ) padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size) return ( @@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows( info = processor.info video_processor = info.get_video_processor() - frame_indices = glm_sample_frame_indices( - total_num_frames, - fps, - duration, - target_fps=video_processor.fps_interval, - max_frame_count=video_processor.max_frame_count_dynamic, - temporal_patch_size=video_processor.temporal_patch_size, + frame_indices = video_processor.sample_frames( + VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration) ) grid_t, grid_h, grid_w = _pixel_path_grid( video_processor, len(frame_indices), height, width diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py index 9d30365b81..6fde77eda2 100644 --- a/tests/models/qwen4_exp/test_config.py +++ b/tests/models/qwen4_exp/test_config.py @@ -7,16 +7,13 @@ from unittest.mock import patch import pytest import torch +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.config.speculative import SpeculativeConfig from vllm.model_executor.models.config import ( Qwen3_5ForConditionalGenerationConfig, Qwen4ExpForConditionalGenerationConfig, ) -from vllm.models.qwen4_exp.config import ( - Qwen4ExpConfig, - Qwen4ExpTextConfig, -) from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState @@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig: "linear_num_value_heads": 2, "linear_key_head_dim": 8, "linear_value_head_dim": 8, - "num_experts": 0, + "num_experts": 4, + "num_experts_per_tok": 2, + # `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled. + "eos_token_id": 1, "hc_count": 2, "hc_lowrank": 4, "ple_layer_ids": [1], diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index d271877dfc..fb191c1b4f 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -10,6 +10,7 @@ import pytest import torch from torch import nn from torch.nn import functional as F +from transformers import Qwen4ExpTextConfig import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module @@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import ( compute_ple_shard_overlap, copy_ple_embedding_shard_, ) -from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpNGramEmbedding, Qwen4ExpPLEDeviceEmbedding, diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 2bb77b2ed7..d593edba29 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -1591,10 +1591,6 @@ def test_glm5next_backend_indices_match_sampler( ): """The loader must select exactly the frames the processor's sampler would, with target.fps mapping onto the raw-fps override.""" - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - source = VideoSourceMetadata( total_frames_num=total_frames, original_fps=original_fps, duration=duration ) @@ -1604,7 +1600,7 @@ def test_glm5next_backend_indices_match_sampler( source, target, max_frames=max_frames ) - assert indices == glm_sample_frame_indices( + assert indices == Glm5NextVideoBackend._sample_frame_indices( total_frames, original_fps, duration, @@ -1692,10 +1688,6 @@ def test_glm5next_backend_codec_parity(tmp_path, backend): if backend == "torchcodec": pytest.importorskip("torchcodec") - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - total_frames, fps = 120, 10 path = _write_gray_video(tmp_path, total_frames, fps) # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20). @@ -1703,7 +1695,7 @@ def test_glm5next_backend_codec_parity(tmp_path, backend): kwargs: dict[str, Any] = ( {} if max_frames is None else {"max_frames": max_frames} ) - expected = glm_sample_frame_indices( + expected = Glm5NextVideoBackend._sample_frame_indices( total_frames, float(fps), 12.0, max_frame_count=max_frames ) @@ -1726,11 +1718,7 @@ def test_glm5next_backend_decodes_only_sampled_frames(tmp_path): total_frames, fps = 60, 10 path = _write_gray_video(tmp_path, total_frames, fps) - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - - expected = glm_sample_frame_indices(total_frames, float(fps), 6.0) + expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0) frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes()) diff --git a/tests/test_config.py b/tests/test_config.py index 48d286b521..a2e75b1072 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4114,3 +4114,47 @@ def test_revision_resolved_for_model(mock_resolve): assert isinstance(config.revision, ResolvedRevision) assert config.revision.resolved == REVISION mock_resolve.assert_any_call(model, None, config.hf_token) + + +@pytest.mark.parametrize( + ("layer_types", "expected_attention"), + [ + # Qwen3-Next / Qwen3.5 spell their attention layers "full_attention". + (["linear_attention", "full_attention"], 1), + # GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches + # every token and so must count as attention. + (["linear_attention", "deepseek_sparse_attention"], 1), + (["linear_attention", "qwen_sparse_attention"], 1), + (["linear_attention", "linear_attention"], 0), + ], +) +def test_hybrid_layer_counts_sparse_attention_as_attention( + layer_types, expected_attention +): + """Sparse-attention layer types consume a full attention KV cache.""" + model_config = object.__new__(ModelConfig) + model_config.hf_config = SimpleNamespace() + model_config.hf_text_config = SimpleNamespace(layer_types=layer_types) + model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next") + + parallel_config = SimpleNamespace() + with ( + patch.object(ModelConfig, "is_hybrid", True), + patch.object(ModelConfig, "has_noops", False), + patch.object(ModelConfig, "is_attention_free", False), + patch.object( + ModelConfig, + "get_layers_start_end_indices", + return_value=(0, len(layer_types)), + ), + ): + assert ( + model_config.get_num_layers_by_block_type(parallel_config, "attention") + == expected_attention + ) + assert ( + model_config.get_num_layers_by_block_type( + parallel_config, "linear_attention" + ) + == len(layer_types) - expected_attention + ) diff --git a/tests/transformers_utils/processors/test_glm5next.py b/tests/transformers_utils/processors/test_glm5next.py deleted file mode 100644 index 917a3f627c..0000000000 --- a/tests/transformers_utils/processors/test_glm5next.py +++ /dev/null @@ -1,389 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor.""" - -import math - -import pytest -import torch -from PIL import Image -from transformers.image_utils import PILImageResampling -from transformers.video_utils import VideoMetadata - -from vllm.transformers_utils.processors.glm5next import ( - Glm5NextImageProcessor, - Glm5NextVideoProcessor, - _get_pad_content_size, - _resize_or_pad, - glm_sample_frame_indices, - smart_resize, -) - -PATCH_SIZE = 14 -MERGE_SIZE = 2 -PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json -FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28 -PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568 -MIN_TOKENS = 16 -MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small -MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN -MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN - - -def resize(height: int, width: int, t: int = 2) -> tuple[int, int]: - return smart_resize( - t, - height, - width, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MIN_PIXELS, - max_pixels=MAX_PIXELS, - ) - - -@pytest.mark.parametrize( - ("height", "width", "t", "expected"), - [ - (300, 400, 2, (308, 420)), # ceil alignment, no budget pressure - (50, 80, 2, (112, 168)), # below min budget: proportional upscale - (57, 3, 2, (504, 28)), # slim side ceils to one factor - (2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap - (112, 11200, 2, (84, 7420)), # extreme aspect stays proportional - (300, 400, 16, (252, 308)), # frame count eats into the budget - (112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds - ], -) -def test_smart_resize_reference_values(height, width, t, expected): - got = smart_resize( - t, - height, - width, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MIN_PIXELS, - max_pixels=MAX_PIXELS, - ) - assert got == expected - # The aligned canvas never exceeds the budget it was fitted against. - t_bar = max(2, round(t / 2) * 2) - assert t_bar * got[0] * got[1] <= MAX_PIXELS - - -def test_smart_resize_stays_snapped_and_positive(): - heights = (1, 27, 57, 111, 113, 300, 720, 2160) - widths = (1, 27, 57, 111, 113, 400, 1280, 11200) - for h in heights: - for w in widths: - for t in (2, 4, 16, 64): - resized_h, resized_w = resize(h, w, t) - assert resized_h > 0 and resized_w > 0 - assert resized_h % FACTOR == 0 - assert resized_w % FACTOR == 0 - t_bar = max(2, round(t / 2) * 2) - assert t_bar * resized_h * resized_w <= MAX_PIXELS - - -@pytest.mark.parametrize( - ("height", "width", "match"), - [ - (0, 100, "must be positive"), - (100, 0, "must be positive"), - ], -) -def test_smart_resize_rejects_degenerate_inputs(height, width, match): - with pytest.raises(ValueError, match=match): - resize(height, width, 2) - - -def test_smart_resize_rejects_inverted_budget(): - with pytest.raises(ValueError, match="min_pixels must be less than or equal"): - smart_resize( - 2, - 100, - 100, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MAX_PIXELS, - max_pixels=MIN_PIXELS, - ) - - -def test_get_pad_content_size(): - # Oversized content shrinks proportionally, never upscales by default. - assert _get_pad_content_size(300, 400, 308, 420) == (300, 400) - assert _get_pad_content_size(600, 800, 308, 420) == (308, 410) - # allow_upscale enlarges small content toward the canvas. - assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112) - - -def test_resize_or_pad_pads_right_and_bottom(): - stacked = torch.rand(1, 3, 300, 400) - - def identity_resize(x, size, resample=None): - assert (size.height, size.width) == (300, 400) - return x - - padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize) - assert padded.shape == (1, 3, 308, 420) - torch.testing.assert_close(padded[..., :300, :400], stacked) - assert padded[..., 300:, :].eq(0).all() - assert padded[..., :, 400:].eq(0).all() - - def force_resize(x, size, resample=None): - return torch.zeros(x.shape[0], x.shape[1], size.height, size.width) - - assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == ( - 1, - 3, - 308, - 420, - ) - with pytest.raises(ValueError, match="resize_mode"): - _resize_or_pad(stacked, 308, 420, "crop", None, identity_resize) - - -@pytest.fixture(scope="module") -def image_processor(): - return Glm5NextImageProcessor( - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS - ) - - -@pytest.mark.parametrize( - ("height", "width"), - [(50, 80), (300, 400), (2160, 3840), (112, 11200)], -) -def test_image_grid_matches_smart_resize(image_processor, height, width): - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt") - resized_h, resized_w = resize(height, width) - grid = out["image_grid_thw"][0].tolist() - assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE] - assert out["pixel_values"].shape == ( - grid[1] * grid[2], - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - - -@pytest.mark.parametrize( - ("height", "width"), - [(50, 80), (300, 400), (2160, 3840), (112, 11200)], -) -def test_image_patch_count_matches_preprocess(image_processor, height, width): - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt") - grid = out["image_grid_thw"][0].tolist() - expected = grid[1] * grid[2] - assert image_processor.get_number_of_image_patches(height, width) == expected - - -@pytest.fixture(scope="module") -def video_processor(): - return Glm5NextVideoProcessor( - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS - ) - - -def run_video_preprocess(video_processor, frames, **overrides): - kwargs = dict( - resample=PILImageResampling.BICUBIC, - image_mean=(0.48145466, 0.4578275, 0.40821073), - image_std=(0.26862954, 0.26130258, 0.27577711), - patch_size=PATCH_SIZE, - temporal_patch_size=2, - merge_size=MERGE_SIZE, - patch_expand_factor=PATCH_EXPAND_FACTOR, - return_tensors="pt", - ) - kwargs.update(overrides) - return video_processor._preprocess([frames], **kwargs) - - -def test_video_preprocess_pads_odd_frame_count(video_processor): - out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400)) - grid = out["video_grid_thw"][0].tolist() - # 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420). - assert grid == [4, 22, 30] - assert out["pixel_values_videos"].shape == ( - 4 * 22 * 30, - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - - -def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor): - out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200)) - # Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap. - assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0 - - -def test_video_preprocess_pads_content_not_distort(video_processor): - frames = torch.zeros(4, 3, 300, 400) - frames[..., 50, 60] = 1.0 # marker inside the content area - # Bypass rescale/normalize so zero padding stays exactly zero. - out = run_video_preprocess( - video_processor, frames, do_rescale=False, do_normalize=False - ) - # Pad mode keeps the 300x400 content aspect on the (308, 420) canvas - # with zero padding on the right/bottom. - grid = out["video_grid_thw"][0].tolist() - assert grid == [2, 22, 30] - patches = out["pixel_values_videos"].view( - grid[0], - grid[1] // MERGE_SIZE, - grid[2] // MERGE_SIZE, - MERGE_SIZE, - MERGE_SIZE, - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1) - # Patch columns fully right of the 400px content are pure padding. - assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0 - - -@pytest.mark.parametrize( - ("total_frames", "fps", "duration", "expected_len", "first", "last"), - [ - # Dense source: the tp-scaled greedy overshoots extract_t, so the - # fixup spreads picks uniformly across the whole video (linspace). - (900, 30.0, 30.0, 60, 0, 899), - (3000, 30.0, 100.0, 200, 0, 2999), - # extract_t capped at 2048 frames. - (72000, 30.0, 2400.0, 2048, 0, 71999), - # Low container fps: the greedy picks every frame. - (48, 2.0, 24.0, 48, 0, 47), - # Duration derived from the frame count when metadata lacks it. - (300, 25.0, 0, 26, 0, 299), - ], -) -def test_glm_sample_frame_indices_behaviour( - total_frames, fps, duration, expected_len, first, last -): - indices = glm_sample_frame_indices(total_frames, fps, duration) - assert len(indices) == expected_len - assert indices[0] == first - assert indices[-1] == last - assert len(indices) % 2 == 0 - assert indices == sorted(indices) - assert all(0 <= idx < total_frames for idx in indices) - - -def test_glm_sample_frame_indices_short_clip_floor_spread(): - # extract_t (20) > total (7): evenly spaced timestamps, deduplicated, - # then pair-padded to an even count. - assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6] - - -@pytest.mark.parametrize( - ("kwargs", "expected_len"), - [ - ({"target_fps": 0.5}, 16), - ({"max_frame_count": 16}, 16), - ({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap - ], -) -def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len): - indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs) - assert len(indices) == expected_len - assert len(indices) % 2 == 0 - - -def test_sample_frames_fps_interval(video_processor): - indices = video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0) - ) - assert len(indices) == 60 - assert indices[0] == 0 - assert indices[-1] == 899 # uniform spread reaches the final frame - - # Request overrides reach the sampler through both kwarg spellings. The - # 15-frame spread is odd, so pair-padding duplicates the last frame. - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - fps=0.5, - ) - ) - == 16 - ) - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - target_fps=0.5, - ) - ) - == 16 - ) - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - max_frames=16, - ) - ) - == 16 - ) - - -def test_defaults_mirror_checkpoint_config(): - """Bare instantiation matches the checkpoint's ``processor_config.json`` - token-budget style defaults; ``size`` carries no budget.""" - image_processor = Glm5NextImageProcessor() - assert image_processor.patch_expand_factor == 1 - assert image_processor.min_image_tokens == 16 - assert image_processor.max_image_tokens == 8000 - assert image_processor.resize_mode == "pad" - - video_processor = Glm5NextVideoProcessor() - assert video_processor.patch_expand_factor == 1 - assert video_processor.min_image_tokens == 16 - assert video_processor.max_image_tokens == 240000 - assert video_processor.fps_interval == 2.0 - assert video_processor.max_frame_count_dynamic == 2048 - - -def test_token_budgets_drive_geometry(): - """The token bounds fully determine the pixel budget and the grid.""" - token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512) - img = Image.new("RGB", (400, 300)) - grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist() - # Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420) - # without hitting either bound -> 22x30 patches. - assert grid == [1, 22, 30] - assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30 - - -def test_missing_token_budgets_rejected(): - proc = Glm5NextImageProcessor(min_image_tokens=None) - with pytest.raises(ValueError, match="min_image_tokens"): - proc(Image.new("RGB", (400, 300)), return_tensors="pt") - - -def test_video_config_fields_land(): - """fps_interval / max_frame_count_dynamic from the dedicated config - shape sampling without any request overrides.""" - proc = Glm5NextVideoProcessor( - fps_interval=4, - max_frame_count_dynamic=32, - ) - indices = proc.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0) - ) - # 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32. - assert len(indices) == 32 - assert indices[0] == 0 - assert indices[-1] == 899 - - # Request overrides still win over the config values. - assert ( - len( - proc.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - fps=0.5, - ) - ) - == 16 - ) diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 19773dfda4..4fd27ba278 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -24,11 +24,6 @@ from vllm.transformers_utils.config import ( try_get_generation_config, uses_mrope, ) -from vllm.transformers_utils.configs.glm5_next import ( - Glm5NextConfig, - Glm5NextTextConfig, - Glm5NextVisionConfig, -) from vllm.transformers_utils.configs.mistral import adapt_config_dict @@ -174,54 +169,6 @@ def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling(): assert config.rope_parameters["attention_factor"] == 1.0 -def test_glm5_next_accepts_deepseek_sparse_attention_layers(): - layer_types = ["linear_attention", "deepseek_sparse_attention"] - - config = Glm5NextTextConfig( - num_hidden_layers=len(layer_types), layer_types=layer_types - ) - - assert config.layer_types == layer_types - assert config.layers_block_type == ["linear_attention", "attention"] - - -def test_glm5_next_accepts_prebuilt_subconfigs(): - text_config = Glm5NextTextConfig(hidden_size=1024) - vision_config = Glm5NextVisionConfig(hidden_size=768) - - config = Glm5NextConfig( - text_config=text_config, - vision_config=vision_config, - ) - - assert config.text_config is text_config - assert config.vision_config is vision_config - - -@pytest.mark.parametrize( - ("kwargs", "option"), - [ - ( - {"index_topk": 2048, "index_dsa_use_layernorm": False}, - "index_dsa_use_layernorm", - ), - ( - {"index_topk": 2048, "index_kpool_compress": False}, - "index_kpool_compress", - ), - ( - {"index_topk": 2048, "index_kpool_always_select_tail": False}, - "index_kpool_always_select_tail", - ), - ({"hres_vwnstyle": False}, "hres_vwnstyle"), - ({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"), - ], -) -def test_glm5_next_rejects_unimplemented_config_options(kwargs, option): - with pytest.raises(NotImplementedError, match=option): - Glm5NextTextConfig(**kwargs) - - def test_get_llama3_eos_token(): model_name = "meta-llama/Llama-3.2-1B-Instruct" diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index 46c7064b0d..112ff77167 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config): index_kpool=4, ) vllm_config = mock.MagicMock() - vllm_config.speculative_config.draft_model_config.hf_config = config + vllm_config.speculative_config.draft_model_config.hf_text_config = config vllm_config.scheduler_config.max_num_batched_tokens = 4 with ( diff --git a/vllm/config/model.py b/vllm/config/model.py index 4c9ed9e13f..4de030e8f3 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -108,6 +108,16 @@ HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig] ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"] LayerBlockType = Literal["attention", "linear_attention", "mamba"] +_ATTENTION_LAYER_TYPES = frozenset( + { + "full_attention", + "deepseek_sparse_attention", + "qwen_sparse_attention", + } +) +"""`layer_types` spellings that consume a full attention KV cache. Sparse +attention still caches every token, so it counts as attention here.""" + _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = { "generate": [], "pooling": ["embed", "classify"], @@ -1718,7 +1728,8 @@ class ModelConfig: if layer_types_value is not None: if block_type == "attention": return sum( - t == "full_attention" for t in layer_types_value[start:end] + t in _ATTENTION_LAYER_TYPES + for t in layer_types_value[start:end] ) elif block_type == "linear_attention": return sum( diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index dbec9fd743..ff4f76153d 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1059,7 +1059,7 @@ class SpeculativeConfig: ) if hf_config.model_type == "glm5_next": hf_config.model_type = "glm5_next_mtp" - n_predict = hf_config.num_nextn_predict_layers + n_predict = hf_config.get_text_config().num_nextn_predict_layers hf_config.update( {"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]} ) diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 2e17424f1e..d7bf70af07 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -4,6 +4,7 @@ import torch import torch.nn.functional as F from torch import nn +from transformers import Glm5NextTextConfig from vllm.config import ( CacheConfig, @@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8 from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool from vllm.platforms import current_platform -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec @@ -221,7 +221,7 @@ class Indexer(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, hidden_size: int, q_lora_rank: int, quant_config: QuantizationConfig | None, @@ -236,7 +236,7 @@ class Indexer(nn.Module): # self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"] # Indexer is only constructed for v32 configs, where these sparse-indexer # fields are guaranteed populated; narrow away the `int | None` declared - # on Glm5NextConfig for the optional-indexer case. + # on Glm5NextTextConfig for the optional-indexer case. assert config.index_topk is not None assert config.index_n_heads is not None assert config.index_head_dim is not None @@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, hidden_size: int, num_heads: int, qk_nope_head_dim: int, @@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module): prefix=f"{prefix}.o_proj", ) + # `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`; + # a checkpoint may still ship one. + rope_parameters = getattr(config, "rope_parameters", None) + if not skip_rope: - assert config.rope_parameters is not None - if config.rope_parameters["rope_type"] != "default": - config.rope_parameters["rope_type"] = ( + assert rope_parameters is not None + if rope_parameters["rope_type"] != "default": + rope_parameters["rope_type"] = ( "deepseek_llama_scaling" - if config.rope_parameters.get("attention_factor") == 1.0 + if rope_parameters.get("attention_factor") == 1.0 else "deepseek_yarn" ) self.rotary_emb: RotaryEmbedding | None = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, + rope_parameters=rope_parameters, is_neox_style=False, ) if ( - config.rope_parameters["rope_type"] != "default" - and config.rope_parameters["rope_type"] == "deepseek_yarn" + rope_parameters["rope_type"] != "default" + and rope_parameters["rope_type"] == "deepseek_yarn" ): - mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False) - scaling_factor = config.rope_parameters["factor"] + mscale_all_dim = rope_parameters.get("mscale_all_dim", False) + scaling_factor = rope_parameters["factor"] mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale else: @@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module): self.indexer_rope_emb: RotaryEmbedding | None = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, + rope_parameters=rope_parameters, is_neox_style=not config.indexer_rope_interleave, ) # The sparse indexer projects from the MLA q-lora rank, which is diff --git a/vllm/models/glm5next/common/kda.py b/vllm/models/glm5next/common/kda.py index c67ac5b012..141a8cc7cb 100644 --- a/vllm/models/glm5next/common/kda.py +++ b/vllm/models/glm5next/common/kda.py @@ -6,6 +6,7 @@ from dataclasses import replace import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig, get_current_vllm_config @@ -44,7 +45,6 @@ from vllm.model_executor.utils import ( ) from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.worker.workspace import current_workspace_manager @@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): def __init__( self, - config: Glm5NextConfig, + config: Glm5NextTextConfig, vllm_config: VllmConfig, prefix: str = "", ) -> None: diff --git a/vllm/models/glm5next/common/model.py b/vllm/models/glm5next/common/model.py index 3ff3476375..e15c00ecb8 100644 --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -6,6 +6,7 @@ from typing import ClassVar, Literal import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.config import ParallelConfig, VllmConfig from vllm.distributed import ( @@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import ( from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from .attention import Glm5NextMLAAttention from .kda import Glm5NextLinearAttention @@ -95,6 +95,57 @@ from .multimodal import ( logger = init_logger(__name__) +_MHC_TAU = 0.05 +"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_MHC_POST_MULT_VALUE = 2.0 +"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_LOGIT_SCALE = 1.0 +"""Output logit scale. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_VISION_RMS_NORM_EPS = 1e-6 +"""Vision tower RMSNorm epsilon. + +GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the +vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and +produces repetitive/degraded image descriptions, so force the trained value +regardless of the checkpoint field. +""" + + +def _is_moe(config: Glm5NextTextConfig) -> bool: + return config.n_routed_experts is not None + + +def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool: + layer_types = config.layer_types + return ( + layer_types is not None + and layer_idx < len(layer_types) + and layer_types[layer_idx] == "linear_attention" + ) + + +def _is_linear_attn(config: Glm5NextTextConfig) -> bool: + layer_types = config.layer_types + return layer_types is not None and "linear_attention" in layer_types + + +def _validate_supported_config(config: Glm5NextTextConfig) -> None: + """Reject checkpoints using config options this implementation lacks. + + The kpool indexer kernels always keep the incomplete trailing pool, so a + checkpoint asking otherwise would be served silently wrong. + """ + if config.index_topk is not None and not config.index_kpool_always_select_tail: + raise NotImplementedError( + "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True" + ) + class Glm5NextMLP(nn.Module): def __init__( @@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module): class Glm5NextMoE(nn.Module): def __init__( self, - config: Glm5NextConfig, + config: Glm5NextTextConfig, parallel_config: ParallelConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module): shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, - top_k=config.num_experts_per_token, + top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, - renormalize=config.moe_renormalize, + renormalize=config.norm_topk_prob, quant_config=quant_config, use_grouped_topk=True, num_expert_group=config.n_group, @@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, layer_idx: int, prefix: str = "", topk_indices_buffer: torch.Tensor | None = None, @@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module): self.hidden_size = config.hidden_size self.layer_idx = layer_idx - self.is_moe = config.is_moe + self.is_moe = _is_moe(config) self.num_hidden_layers = config.num_hidden_layers self.rms_norm_eps = config.rms_norm_eps self.num_experts = config.n_routed_experts self.is_mtp_layer = is_mtp_layer self.mhc = config.mhc - is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx) + is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx) self.layer_kind = "kda" if is_kda_layer else "mla" self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe @@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module): quant_config=None, # MLA projections are BF16 in checkpoint prefix=f"{prefix}.self_attn", topk_indices_buffer=topk_indices_buffer, - skip_rope=config.mla_nope, + skip_rope=config.mla_use_nope, ) # MTP layers sit past the base model's hidden layers (layer_idx >= @@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module): if self.mhc and not is_mtp_layer: # mhc config - self.mhc_num_residual_streams = config.mhc_num_residual_streams - self.mhc_tau = config.mhc_tau + self.mhc_num_residual_streams = config.hc_mult + self.mhc_tau = _MHC_TAU self.hc_eps = config.hc_eps - self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations - self.mhc_post_mult_value = config.mhc_post_mult_value + self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters + self.mhc_post_mult_value = _MHC_POST_MULT_VALUE - n = config.mhc_num_residual_streams + n = config.hc_mult d_model = n * self.hidden_size mix_hc = (2 + n) * n @@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_text_config + _validate_supported_config(config) self.config = config self.vocab_size = config.vocab_size @@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module): (".in_proj_qkvbfg_a", ".f_a_proj", 4), (".in_proj_qkvbfg_a", ".g_a_proj", 5), ] - if self.config.is_moe: + if _is_moe(self.config): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) # EPLB: the mapping enumerates physical experts, so it must cover @@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module): # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from # ``kv_a_proj_with_mqa``; pad them with zeros for the model shape. kv_a_pad_size = 0 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0: + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0: kv_a_pad_size = self.config.qk_rope_head_dim _pending_wk_fp8: dict = {} @@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module): if ( name.endswith(".bias") and name not in params_dict - and not self.config.is_linear_attn + and not _is_linear_attn(self.config) ): # noqa: E501 continue # Remapping the name of FP8 kv-scale. @@ -939,7 +991,7 @@ class Glm5NextForCausalLM( super().__init__() self.model_config = vllm_config.model_config self.vllm_config = vllm_config - self.config = self.model_config.hf_config + self.config = self.model_config.hf_text_config quant_config = vllm_config.quant_config self.quant_config = quant_config self.model = Glm5NextModel( @@ -955,7 +1007,7 @@ class Glm5NextForCausalLM( else: self.lm_head = PPMissingLayer() self.logits_processor = LogitsProcessor( - self.config.vocab_size, scale=self.config.logit_scale + self.config.vocab_size, scale=_LOGIT_SCALE ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: @@ -988,7 +1040,7 @@ class Glm5NextForCausalLM( cls, vllm_config: "VllmConfig" ) -> tuple[tuple[int, int], tuple[int, int, int]]: parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_config + hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size num_spec = ( vllm_config.speculative_config.num_speculative_tokens @@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration( self.visual = Glm5NextVisionTransformer( config.text_config, config.vision_config, - # Read eps from the VISION sub-config, not the top-level - # `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors - # the latter onto text_config (1e-5), silently ignoring the - # vision tower's own (1e-6) rms_norm_eps. - norm_eps=config.vision_config.rms_norm_eps, + norm_eps=_VISION_RMS_NORM_EPS, # Vision tower ships BF16 weights in this fp8 checkpoint (no # weight_scale_inv for visual.*), so it must NOT inherit the # global fp8 quant_config -- doing so incorrectly quantizes @@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration( def get_spec_layer_idx_from_weight_name( - config: Glm5NextConfig, weight_name: str + config: Glm5NextTextConfig, weight_name: str ) -> int | None: if hasattr(config, "num_nextn_predict_layers") and ( config.num_nextn_predict_layers > 0 diff --git a/vllm/models/glm5next/common/mtp.py b/vllm/models/glm5next/common/mtp.py index 8361a053b8..816cc46fdc 100644 --- a/vllm/models/glm5next/common/mtp.py +++ b/vllm/models/glm5next/common/mtp.py @@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: super().__init__() assert vllm_config.speculative_config is not None - config = vllm_config.speculative_config.draft_model_config.hf_config + config = vllm_config.speculative_config.draft_model_config.hf_text_config self.config = config self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): class Glm5NextMultiTokenPredictor(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_text_config self.mtp_start_layer_idx = config.num_hidden_layers self.num_mtp_layers = config.num_nextn_predict_layers self.layers = torch.nn.ModuleDict( @@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module): class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - self.config = vllm_config.model_config.hf_config + self.config = vllm_config.model_config.hf_text_config self.quant_config = vllm_config.quant_config self.model = Glm5NextMultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") @@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. kv_a_pad_size = 0 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0: + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0: kv_a_pad_size = self.config.qk_rope_head_dim for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index 60b5b47a60..75921b697f 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -3,13 +3,14 @@ """GLM-5.3-Flash vision tower and multimodal processor.""" from collections.abc import Mapping -from functools import cached_property, partial +from functools import partial from typing import Any import numpy as np import torch import torch.nn as nn from einops import rearrange +from transformers.video_utils import VideoMetadata from vllm.distributed import ( get_tensor_model_parallel_world_size, @@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import ( ) from vllm.models.common.ops import fused_q_kv_rmsnorm from vllm.multimodal.parse import ImageSize, MultiModalDataItems -from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) -class Glm5NextProcessingInfo(Glm4vProcessingInfo): - """Wires up the vLLM-native processor for the multimodal checkpoint. +_MAX_VIDEO_TOKENS = 30000 - The checkpoint's ``processor_config.json`` declares a custom ``processor_class`` - and stores its image/video processor configs inline (no standalone - ``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the - config. We bypass it and build our own ``Glm5NextProcessor`` - (``vllm/transformers_utils/processors/glm5next.py``), a port of the - training-side pipeline that no longer imports transformers' GLM processor - classes. The port applies ``patch_expand_factor`` (checkpoint ships 1) - inside ``smart_resize``'s spatial factor. + +class Glm5NextProcessingInfo(Glm4vProcessingInfo): + """Token-budget geometry for the multimodal checkpoint. + + This checkpoint's ``processor_config.json`` ships the token-budget style + (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the + inherited Glm4v ``size.longest_edge`` path does not apply. """ - @cached_property - def _glm5_hf_processor(self): - from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor - - return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model) - def get_hf_processor(self, **kwargs: object): - return self._glm5_hf_processor + processor = super().get_hf_processor(**kwargs) + # Cap video inputs to keep encoder profiling from starving the KV cache. + video_processor = processor.video_processor + video_processor.max_image_tokens = min( + video_processor.max_image_tokens, _MAX_VIDEO_TOKENS + ) + return processor + + @staticmethod + def _alignment_factor(proc) -> int: + """Spatial factor canvases align to, as ``smart_resize`` takes it.""" + return proc.patch_size * proc.merge_size * proc.patch_expand_factor + + @classmethod + def _pixels_per_token(cls, proc) -> int: + """Pixels one vision token covers, matching ``smart_resize``'s own + ``temporal_factor * factor ** 2``.""" + return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2 def _processor_pixel_budget(self, proc) -> tuple[int, int]: - from vllm.transformers_utils.processors.glm5next import _pixel_budget - - return _pixel_budget( - proc.min_image_tokens, - proc.max_image_tokens, - proc.patch_size, - proc.merge_size, - proc.temporal_patch_size, + """(min_pixels, max_pixels) from the processor's token bounds.""" + if proc.min_image_tokens is None or proc.max_image_tokens is None: + raise ValueError( + "min_image_tokens and max_image_tokens must be provided by " + "processor_config.json (or per-call kwargs)." + ) + pixels_per_token = self._pixels_per_token(proc) + return ( + proc.min_image_tokens * pixels_per_token, + proc.max_image_tokens * pixels_per_token, ) def _get_image_max_pixels(self) -> int: @@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): else: frame_indices = [ int(idx) - for idx in glm_sample_frame_indices( - int(metadata.get("total_num_frames", total_frames)), - video_fps, - float(metadata.get("duration") or 0), - target_fps=video_processor.fps_interval, - max_frame_count=video_processor.max_frame_count_dynamic, - temporal_patch_size=temporal_patch_size, + for idx in video_processor.sample_frames( + VideoMetadata( + total_num_frames=int( + metadata.get("total_num_frames", total_frames) + ), + fps=video_fps, + duration=metadata.get("duration"), + ) ) ] num_frames = len(frame_indices) if not frame_indices or num_frames <= 0: - # ``duration * fps_interval < 1`` selects no frames at all; the + # ``duration * fps < 1`` selects no frames at all; the # pixel path raises an opaque IndexError on the same input a moment # later, so say what happened while there is still a request. raise ValueError( @@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): The inherited Glm4v path resolves the pixel budget from ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This - checkpoint's ``processor_config.json`` ships the token-budget style - (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and - the alignment factor carries ``patch_expand_factor`` — resolve both - from the vLLM-native processor so profiling matches runtime geometry. + checkpoint ships token bounds instead, and the alignment factor carries + ``patch_expand_factor`` — resolve both from the processor so profiling + matches runtime geometry. """ - from vllm.transformers_utils.processors.glm5next import smart_resize + from transformers.models.glm5_next.image_processing_glm5_next import ( + smart_resize, + ) vision_config = self.get_hf_config().vision_config patch_size = vision_config.patch_size @@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): temporal_patch_size = vision_config.temporal_patch_size image_processor = self.get_hf_processor().image_processor - factor = patch_size * merge_size * image_processor.patch_expand_factor - # Keep the profiling search viable when the caller's budget is below - # one aligned canvas of the requested duration. - max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor) + factor = self._alignment_factor(image_processor) + temporal_factor = image_processor.temporal_patch_size + # `smart_resize` denominates its bounds in vision tokens. Round down, + # but keep the profiling search viable when the caller's budget is + # below one aligned canvas of the requested duration. + max_image_tokens = max( + max_image_pixels // self._pixels_per_token(image_processor), 1 + ) if do_resize: - t = num_frames if num_frames > temporal_patch_size else temporal_patch_size + t = num_frames if num_frames > temporal_factor else temporal_factor resized_height, resized_width = smart_resize( - t=t, - h=image_height, - w=image_width, - t_factor=temporal_patch_size, - h_factor=factor, - w_factor=factor, + num_frames=t, + height=image_height, + width=image_width, + temporal_factor=temporal_factor, + factor=factor, min_pixels=1, - max_pixels=max_image_pixels, + max_pixels=max_image_tokens, ) preprocessed_size = ImageSize(width=resized_width, height=resized_height) else: @@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor): - """The vLLM-native ``Glm5NextProcessor`` extracts image/video features + """``Glm5NextProcessor`` extracts image/video features only and passes the prompt text through unchanged, so prompt expansion (image token repeat, video frame/timestamp structure) is owned by vLLM's prompt-update machinery — the inherited ``_get_prompt_updates`` builds diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py index 2b8c7b1444..4345763fd4 100644 --- a/vllm/models/qwen4_exp/amd/indexer_qsa.py +++ b/vllm/models/qwen4_exp/amd/indexer_qsa.py @@ -8,6 +8,7 @@ from typing import cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.forward_context import get_forward_context @@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.qsa_cache import ( QSACompressedKeyCache, diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index 2cdc3c26fa..16509d609c 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -7,6 +7,7 @@ from itertools import islice import torch from torch import nn +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import ( ) from vllm.model_executor.models.qwen3_next import ( Qwen3NextAttention, - Qwen3NextMLP, Qwen3NextSparseMoeBlock, ) from vllm.model_executor.models.qwen3_vl import ( @@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ple_layer import Qwen4ExpPLELayer @@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module): ) -> None: super().__init__() config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.config = config @@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type in ATTENTION_LAYER_TYPES: - use_qsa = ( - layer_type == QSA_LAYER_TYPE - or getattr(config, "indexer_n_heads", None) is not None - ) + elif layer_type == "qwen_sparse_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( config, - model_config=model_config, - cache_config=cache_config, + model_config=vllm_config.model_config, + cache_config=vllm_config.cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module): else: raise ValueError(f"Invalid layer_type {layer_type}") - mlp_only_layers = getattr(config, "mlp_only_layers", []) - num_experts = getattr(config, "num_experts", 0) or 0 - absolute_layer_id = self.layer_idx + 1 - is_moe_layer = self.layer_idx not in mlp_only_layers and ( - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" ) - if is_moe_layer: - self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" - ) - else: - self.mlp = Qwen3NextMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) hc_config = HyperConnectionConfig( hc_count=config.hc_count, @@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type in ATTENTION_LAYER_TYPES: + elif self.layer_type == "qwen_sparse_attention": attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == QSA_LAYER_TYPE - or ( - layer_type == "full_attention" - and getattr(config, "indexer_n_heads", None) is not None - ) + if layer_type == "qwen_sparse_attention" + and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py index f4302aa757..d06f6436e7 100644 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -18,6 +18,7 @@ from collections.abc import Iterable import regex as re import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig, replace, set_current_vllm_config @@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm @@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): self.layers = nn.ModuleList( Qwen4ExpDecoderLayer( draft_vllm_config, - layer_type="full_attention", + layer_type="qwen_sparse_attention", prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", ) for idx in range(self.num_mtp_layers) diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py index 11291ff385..bd0607b613 100644 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence import torch import torch.nn.functional as F from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context @@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, ) from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import direct_register_custom_op from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py index 3c8a61bd54..325e0e3d91 100644 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -8,6 +8,7 @@ from typing import ClassVar, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.config.cache import CacheDType @@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.models.qwen3_next import Qwen3NextAttention from vllm.platforms import current_platform -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, diff --git a/vllm/models/qwen4_exp/config.py b/vllm/models/qwen4_exp/config.py deleted file mode 100644 index dfac816d5e..0000000000 --- a/vllm/models/qwen4_exp/config.py +++ /dev/null @@ -1,263 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Qwen4Exp model configuration.""" - -from typing import Any, ClassVar, cast - -from transformers import PreTrainedConfig -from transformers.models.qwen3_vl.configuration_qwen3_vl import ( - Qwen3VLVisionConfig, -) - -from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig - -_QSA_CONFIG_FIELDS = ( - "indexer_n_heads", - "indexer_kv_heads", - "indexer_head_dim", - "indexer_budget", - "indexer_compress_ratio", -) - -# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`. -# Older checkpoints label them `full_attention` and mark QSA by setting the -# indexer fields for the whole model. -QSA_LAYER_TYPE = "qwen_sparse_attention" -ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE) - - -class Qwen4ExpVisionConfig(Qwen3VLVisionConfig): - model_type = "qwen4_exp" - base_config_key = "vision_config" - - -class Qwen4ExpTextConfig(Qwen3NextConfig): - model_type = "qwen4_exp_text" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - hc_count: int = 4, - hc_lowrank: int = 320, - ple_layer_ids: list[int] | None = None, - ple_embed_dim: int | None = None, - ple_conv_kernel_size: int = 4, - ngram_size: int = 3, - heads_per_ngram: int = 8, - ngram_vocab_size_base: int = 20_000_000, - make_ngram_vocab_size_divisible_by: int = 128, - output_gate_type: str = "sigmoid", - rope_parameters: dict[str, Any] | None = None, - layer_types: list[str] | None = None, - **kwargs: Any, - ) -> None: - if hc_count <= 1: - raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.") - - if rope_parameters is not None: - if kwargs.get("rope_scaling") is None: - kwargs["rope_scaling"] = rope_parameters - if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters: - kwargs["rope_theta"] = rope_parameters["rope_theta"] - if ( - kwargs.get("partial_rotary_factor") is None - and "partial_rotary_factor" in rope_parameters - ): - kwargs["partial_rotary_factor"] = rope_parameters[ - "partial_rotary_factor" - ] - - rope_scaling = kwargs.get("rope_scaling") - rope_theta = kwargs.get("rope_theta", 10_000.0) - super().__init__(layer_types=layer_types, **kwargs) - - normalized_rope_parameters = self.rope_parameters - self.rope_scaling = ( - rope_scaling or rope_parameters or normalized_rope_parameters - ) - self.rope_parameters = rope_parameters or normalized_rope_parameters - self.rope_theta = rope_theta - - self.hc_count = hc_count - self.hc_lowrank = hc_lowrank - self.ple_layer_ids = ple_layer_ids or [] - self.ple_embed_dim = ( - self.hidden_size if ple_embed_dim is None else ple_embed_dim - ) - self.ple_conv_kernel_size = ple_conv_kernel_size - self.ngram_size = ngram_size - self.heads_per_ngram = heads_per_ngram - self.ngram_vocab_size_base = ngram_vocab_size_base - self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by - self.output_gate_type = output_gate_type - - self._validate_ple_config() - self._validate_ple_layer_ids() - self._validate_qsa_config() - - def _validate_ple_config(self) -> None: - if self.hc_lowrank <= 0: - raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}") - if self.ngram_size < 2: - raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") - if self.heads_per_ngram <= 0: - raise ValueError( - f"heads_per_ngram must be positive, got {self.heads_per_ngram}" - ) - if self.ple_embed_dim <= 0: - raise ValueError( - f"ple_embed_dim must be positive, got {self.ple_embed_dim}" - ) - ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram - if self.ple_embed_dim % ngram_heads: - raise ValueError( - "ple_embed_dim must be divisible by total ngram heads: " - f"{self.ple_embed_dim} % {ngram_heads} != 0" - ) - if self.ple_conv_kernel_size <= 0: - raise ValueError( - "ple_conv_kernel_size must be positive, got " - f"{self.ple_conv_kernel_size}" - ) - if self.ngram_vocab_size_base <= 0: - raise ValueError("ngram_vocab_size_base must be positive") - if self.make_ngram_vocab_size_divisible_by <= 0: - raise ValueError("make_ngram_vocab_size_divisible_by must be positive") - - def _validate_ple_layer_ids(self) -> None: - invalid = [ - layer_id - for layer_id in self.ple_layer_ids - if not 1 <= int(layer_id) <= self.num_hidden_layers - ] - if invalid: - raise ValueError( - "ple_layer_ids are 1-based and must refer to an existing layer; " - f"got {invalid} for {self.num_hidden_layers} layers" - ) - - def _validate_qsa_config(self) -> None: - configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS} - if all(value is None for value in configured.values()): - if QSA_LAYER_TYPE in (self.layer_types or ()): - raise ValueError( - f"{QSA_LAYER_TYPE} layers require the QSA config fields: " - f"{list(_QSA_CONFIG_FIELDS)}" - ) - return - - missing = [name for name, value in configured.items() if value is None] - if missing: - raise ValueError(f"QSA config is missing required fields: {missing}") - - values = {name: int(cast(int, value)) for name, value in configured.items()} - if any(value <= 0 for value in values.values()): - raise ValueError(f"QSA config values must be positive: {values}") - if values["indexer_kv_heads"] != 1: - raise ValueError("the QSA MQA operators require indexer_kv_heads=1") - if values["indexer_budget"] % values["indexer_compress_ratio"] != 0: - raise ValueError( - "indexer_budget must be divisible by indexer_compress_ratio" - ) - block_topk = values["indexer_budget"] // values["indexer_compress_ratio"] - if block_topk not in (512, 2048): - raise ValueError( - "QSA requires indexer_budget / indexer_compress_ratio " - f"to be 512 or 2048, got {block_topk}" - ) - rotary_dim = int(self.head_dim * self.partial_rotary_factor) - if rotary_dim > values["indexer_head_dim"]: - raise ValueError( - "QSA indexer_head_dim must cover the attention rotary " - f"dimension, got {values['indexer_head_dim']} < {rotary_dim}" - ) - - @property - def layers_block_type(self) -> list[str]: - return [ - "attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type - for layer_type in self.layer_types - ] - - @property - def short_conv_layer_ids(self) -> list[int]: - if not self.ple_layer_ids: - return [] - return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids}) - - @property - def short_conv_state_shape(self) -> tuple[int, int] | None: - if not self.short_conv_layer_ids: - return None - ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size - ple_channels = self.hidden_size * self.hc_count - return ple_channels, ple_state_len - - @property - def ngram_context_len(self) -> int: - if not self.ple_layer_ids: - return 0 - return max(int(self.ngram_size) - 1, 0) - - -class Qwen4ExpConfig(PreTrainedConfig): - model_type = "qwen4_exp" - sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = { - "vision_config": Qwen4ExpVisionConfig, - "text_config": Qwen4ExpTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None, - vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None, - image_token_id: int = 248056, - video_token_id: int = 248057, - vision_start_token_id: int = 248053, - vision_end_token_id: int = 248054, - tie_word_embeddings: bool = False, - rope_parameters: dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - if text_config is not None: - kwargs.pop("split_ngram_parts", None) - - text_kwargs = ( - dict(kwargs) - if text_config is None - and "hidden_size" in kwargs - and "num_hidden_layers" in kwargs - else {} - ) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - self.text_config = self.sub_configs["text_config"](**text_kwargs) - else: - self.text_config = text_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - self.rope_parameters = rope_parameters or getattr( - self.text_config, "rope_parameters", {} - ) - super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings) - - -__all__ = [ - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", -] diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index 14137de695..c8ffa0dbfb 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.forward_context import get_forward_context @@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.qsa_cache import ( QSACompressedKeyCache, diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 20521f1800..95e920bc21 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -7,6 +7,7 @@ from itertools import islice import torch from torch import nn +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.distributed import get_pp_group @@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import ( ) from vllm.model_executor.models.qwen3_next import ( Qwen3NextAttention, - Qwen3NextMLP, Qwen3NextSparseMoeBlock, ) from vllm.model_executor.models.qwen3_vl import ( @@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup @@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module): ) -> None: super().__init__() config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.config = config @@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type in ATTENTION_LAYER_TYPES: - use_qsa = ( - layer_type == QSA_LAYER_TYPE - or getattr(config, "indexer_n_heads", None) is not None - ) + elif layer_type == "qwen_sparse_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( config, - model_config=model_config, - cache_config=cache_config, + model_config=vllm_config.model_config, + cache_config=vllm_config.cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module): else: raise ValueError(f"Invalid layer_type {layer_type}") - mlp_only_layers = getattr(config, "mlp_only_layers", []) - num_experts = getattr(config, "num_experts", 0) or 0 - absolute_layer_id = self.layer_idx + 1 - is_moe_layer = self.layer_idx not in mlp_only_layers and ( - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" ) - if is_moe_layer: - self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" - ) - else: - self.mlp = Qwen3NextMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) hc_config = HyperConnectionConfig( hc_count=config.hc_count, @@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type in ATTENTION_LAYER_TYPES: + elif self.layer_type == "qwen_sparse_attention": attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == QSA_LAYER_TYPE - or ( - layer_type == "full_attention" - and getattr(config, "indexer_n_heads", None) is not None - ) + if layer_type == "qwen_sparse_attention" + and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index d9f26fc022..61290617ae 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -18,6 +18,7 @@ from collections.abc import Iterable import regex as re import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig, replace, set_current_vllm_config from vllm.distributed import get_pp_group @@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm @@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): self.layers = nn.ModuleList( Qwen4ExpDecoderLayer( draft_vllm_config, - layer_type="full_attention", + layer_type="qwen_sparse_attention", prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", ) for idx in range(self.num_mtp_layers) diff --git a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py index 0b6fd650b1..3be6ea6013 100644 --- a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py +++ b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py @@ -6,6 +6,7 @@ from collections.abc import Iterable import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import get_current_vllm_config from vllm.logger import init_logger @@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, ) from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.ngram_embedding import ( Qwen4ExpPLEDeviceEmbedding, diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 2aac4f0f58..ee8a2b47dd 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -6,6 +6,7 @@ from collections.abc import Sequence import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config @@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, is_conv_state_dim_first, ) -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionBackend, diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index 62aa6459fb..d00cd186b0 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -8,6 +8,7 @@ from typing import ClassVar, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig @@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro from vllm.model_executor.models.qwen3_next import Qwen3NextAttention from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import ( kv_cache_dtype_str_to_dtype, ) diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index ff74183a55..44a8e9452d 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend): class Glm5NextVideoBackend(VideoBackend): """GLM-5.3-Flash fps-interval video backend. - Selects frames with the same ``glm_sample_frame_indices`` sampler the - processor falls back to, so only the sampled frames are - materialized. ``fps_interval`` semantics (default 2.0) with a - temporal-patch-scaled greedy walk, frame count capped at 2048, temporal - pairs kept even. Request overrides: ``fps`` -> fps interval, + Selects frames with the same sampler the processor falls back to, so only + the sampled frames are materialized. ``fps_interval`` semantics (default + 2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048, + temporal pairs kept even. Request overrides: ``fps`` -> fps interval, ``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2). """ _SEEK_GAP_THRESHOLD: ClassVar[int] = 64 + _DEFAULT_FPS: ClassVar[float] = 2.0 + _DEFAULT_MAX_FRAMES: ClassVar[int] = 2048 + + @classmethod + def _sample_frame_indices( + cls, + total_frames: int, + fps: float, + duration: float, + *, + target_fps: float | None = None, + max_frame_count: int | None = None, + temporal_patch_size: int = 2, + ) -> list[int]: + """GLM video frame sampling (training-reference parity). + + ``target_fps`` is the ``fps_interval`` request knob. The greedy walk + advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on + frame-dense sources it collects more candidates than ``extract_t`` and + the ``> extract_t`` fixup re-spreads the picks uniformly with + ``np.linspace`` -- that fallback is the intended reference behavior, not + an accident. Short clips (fewer frames than ``extract_t``) are spread at + evenly spaced timestamps (``floor`` sampling; the linspace variant + samples frames unevenly and cost 4 points on video grounding evals). + Request overrides: ``target_fps`` -> fps interval, ``max_frame_count`` + -> frame cap. + """ + max_frame_idx = total_frames - 1 + if not duration: + duration = (round(max_frame_idx / fps) + 1) if fps else 0 + if max_frame_count is None: + max_frame_count = cls._DEFAULT_MAX_FRAMES + if target_fps is None: + target_fps = cls._DEFAULT_FPS + + extract_t = int(duration * target_fps) + extract_t = min(extract_t, int(max_frame_count)) + + duration_per_frame = 1 / fps + timestamps = [i * duration_per_frame for i in range(total_frames)] + max_second = int(duration) + + if total_frames < extract_t: + frame_indices = [ + math.floor(_i * total_frames / extract_t) for _i in range(extract_t) + ] + else: + frame_indices = [] + current_second = 0.0 + inv_fps = 1 / (temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) < extract_t: + if len(frame_indices) == 0: + start, end = 0, max(total_frames - 1, 0) + else: + start, end = frame_indices[0], frame_indices[-1] + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: + frame_indices = np.linspace( + 0, total_frames - 1, extract_t, dtype=int + ).tolist() + + seen, uniq = set(), [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(int(idx)) + + if len(uniq) & 1: + uniq.append(uniq[-1]) + + return uniq @classmethod def compute_frames_index_to_sample( @@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend): target: VideoTargetMetadata, **kwargs, ) -> list[int]: - # Lazy import: the processor module sits behind the - # transformers_utils package init, which multimodal must not pull in. - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - - return glm_sample_frame_indices( + return cls._sample_frame_indices( source.total_frames_num, source.original_fps, source.duration or 0, diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 432fb1379f..273c504f9d 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( k3_dspark="K3DSparkConfig", funaudiochat="FunAudioChatConfig", granite4_vision="Granite4VisionConfig", - glm5_next="Glm5NextConfig", - glm5_next_text="Glm5NextTextConfig", - glm5_next_vision="Glm5NextVisionConfig", hyperclovax="HyperCLOVAXConfig", hy_v3="HYV3Config", hy_v4="HYV4Config", @@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( qianfan_ocr="QianfanOCRConfig", qwen3_asr="Qwen3ASRConfig", qwen3_next="Qwen3NextConfig", - qwen4_exp="Qwen4ExpConfig", - qwen4_exp_text="Qwen4ExpTextConfig", qwen3_5="Qwen3_5Config", qwen3_5_text="Qwen3_5TextConfig", qwen3_5_moe="Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index a302d5d729..5bd47fb948 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat", "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision", - "Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next", - "Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next", - "Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HYV4Config": "vllm.transformers_utils.configs.hy_v4", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", @@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr", "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr", "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next", - "Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp", - "Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp", - "Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp", "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe", @@ -140,9 +134,6 @@ __all__ = [ "FunAudioChatConfig", "FunAudioChatAudioEncoderConfig", "Granite4VisionConfig", - "Glm5NextConfig", - "Glm5NextTextConfig", - "Glm5NextVisionConfig", "HYV3Config", "HYV4Config", "HyperCLOVAXConfig", @@ -187,9 +178,6 @@ __all__ = [ "QianfanOCRVisionConfig", "Qwen3ASRConfig", "Qwen3NextConfig", - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", "Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/glm5_next.py b/vllm/transformers_utils/configs/glm5_next.py deleted file mode 100644 index 4a565884a2..0000000000 --- a/vllm/transformers_utils/configs/glm5_next.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.configuration_utils import PreTrainedConfig - -from vllm.logger import init_logger - -logger = init_logger(__name__) - - -class Glm5NextTextConfig(PreTrainedConfig): - model_type = "glm5_next_text" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - model_type="glm5_next_text", - vocab_size: int = 154880, - hidden_size: int = 4096, - head_dim: int | None = None, - intermediate_size: int = 12288, - num_hidden_layers: int = 45, - num_attention_heads: int = 64, - num_key_value_heads: int | None = None, - hidden_act: str = "silu", - rms_norm_eps: float = 1e-5, - pad_token_id: int | None = 151329, - bos_token_id: int | None = None, - eos_token_id: int | list[int] | None = None, - rope_parameters: dict | None = None, - max_position_embeddings: int = 1048576, - tie_word_embeddings: bool = False, - moe_intermediate_size: int = 2048, - moe_renormalize: bool = True, - scoring_func: str = "sigmoid", - n_routed_experts: int | None = 288, - num_experts_per_token: int = 7, - n_shared_experts: int = 1, - routed_scaling_factor: float = 2.5, - topk_method: str | None = None, - first_k_dense_replace: int = 0, - moe_layer_freq: int = 1, - use_grouped_topk: bool = True, - n_group: int = 1, - topk_group: int = 1, - mla: bool = True, - q_lora_rank: int | None = 1536, - kv_lora_rank: int | None = 512, - qk_nope_head_dim: int = 256, - qk_rope_head_dim: int = 0, - v_head_dim: int | None = 256, - mla_nope: bool | None = True, - num_nextn_predict_layers: int = 1, - # Per-layer layout: "linear_attention" | "deepseek_sparse_attention" - layer_types: list[str] | None = None, - # Per-layer MLP: "dense" | "sparse" - mlp_layer_types: list[str] | None = None, - # Linear-attention (KDA) head config (flattened from the old - # linear_attn_config dict). - linear_head_dim: int = 128, - linear_num_heads: int = 64, - linear_conv_kernel_dim: int = 4, - linear_lower_bound: float = -5.0, - index_head_dim: int | None = None, - index_topk: int | None = None, - index_n_heads: int | None = None, - index_dsa_use_layernorm: bool = True, - index_kpool_compress: bool = True, - # Every ``index_kpool`` indexer K entries pool into one stored entry - # (compress_ratio). The 300B checkpoint ships 4; topk runs at pool - # granularity (select_k = index_topk // index_kpool). - index_kpool: int | None = 4, - index_kpool_always_select_tail: bool = True, - indexer_rope_interleave: bool = False, - mhc: bool | None = True, - mhc_num_residual_streams: int = 4, - hc_eps: float | None = 1e-06, - mhc_tau: float = 0.05, - hres_vwnstyle: bool | None = True, - mhc_no_norm_weight: bool | None = False, - mhc_sinkhorn_iterations: int | None = 20, - mhc_post_mult_value: float | None = 2.0, - swiglu_limit: float | None = None, - logit_scale: float = 1.0, - **kwargs, - ): - # Preserve checkpoint field names and local aliases because their - # consumers use different spellings. - num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token) - moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize) - mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams) - mhc_sinkhorn_iterations = kwargs.get( - "hc_sinkhorn_iters", mhc_sinkhorn_iterations - ) - # Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this - # alias self.mla_nope silently stays at the param default. - mla_nope = kwargs.get("mla_use_nope", mla_nope) - # Checkpoints ship the KDA head config as the ``linear_attn_config`` - # dict (head_dim / num_heads / short_conv_kernel_size / - # gate_lower_bound) rather than the flattened top-level fields; fold it - # in so the trained values are read instead of the param defaults - # (which only match this checkpoint by coincidence). - linear_cfg = kwargs.get("linear_attn_config") or {} - if linear_cfg: - linear_head_dim = linear_cfg.get("head_dim", linear_head_dim) - linear_num_heads = linear_cfg.get("num_heads", linear_num_heads) - linear_conv_kernel_dim = linear_cfg.get( - "short_conv_kernel_size", linear_conv_kernel_dim - ) - linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound) - - if index_topk is not None: - if index_dsa_use_layernorm is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True" - ) - if index_kpool_compress is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires index_kpool_compress=True" - ) - if index_kpool_always_select_tail is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires " - "index_kpool_always_select_tail=True" - ) - - if mhc: - if hres_vwnstyle is not True: - raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True") - if mhc_no_norm_weight not in (False, None): - raise NotImplementedError( - "GLM-5.3 mHC requires mhc_no_norm_weight=False" - ) - - self.model_type = model_type - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.head_dim = ( - head_dim if head_dim is not None else hidden_size // num_attention_heads - ) - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.rms_norm_eps = rms_norm_eps - self.max_position_embeddings = max_position_embeddings - self.rope_parameters = rope_parameters - - # mla config - self.mla = mla - self.q_lora_rank = q_lora_rank - self.kv_lora_rank = kv_lora_rank - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_rope_head_dim = qk_rope_head_dim - self.v_head_dim = v_head_dim - self.mla_nope = mla_nope - # moe config - self.n_routed_experts = n_routed_experts - self.num_experts_per_token = num_experts_per_token - self.moe_renormalize = moe_renormalize - self.n_shared_experts = n_shared_experts - self.routed_scaling_factor = routed_scaling_factor - self.topk_method = topk_method - self.scoring_func = scoring_func - assert self.scoring_func in ("softmax", "sigmoid") - self.moe_intermediate_size = moe_intermediate_size - self.first_k_dense_replace = first_k_dense_replace - self.moe_layer_freq = moe_layer_freq - self.use_grouped_topk = use_grouped_topk - self.n_group = n_group - self.topk_group = topk_group - self.num_nextn_predict_layers = num_nextn_predict_layers - - # Per-layer attention / MLP layout. Normalize mlp_layer_types from - # first_k_dense_replace when the new-schema field is absent so layer - # construction sees a consistent layout (mirrors cohere2_moe). - self.layer_types = layer_types - if mlp_layer_types is None: - n = self.num_hidden_layers - if first_k_dense_replace is not None: - mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * ( - n - first_k_dense_replace - ) - else: - mlp_layer_types = ["sparse"] * n - self.mlp_layer_types = mlp_layer_types - - # Linear-attention (KDA) head config. - self.linear_head_dim = linear_head_dim - self.linear_num_heads = linear_num_heads - self.linear_conv_kernel_dim = linear_conv_kernel_dim - self.linear_lower_bound = linear_lower_bound - - # dsa index config - self.index_head_dim = index_head_dim - self.index_topk = index_topk - self.index_n_heads = index_n_heads - self.index_dsa_use_layernorm = index_dsa_use_layernorm - self.index_kpool_compress = index_kpool_compress - self.index_kpool = index_kpool - self.index_kpool_always_select_tail = index_kpool_always_select_tail - self.indexer_rope_interleave = indexer_rope_interleave - - # mhc config - self.mhc = mhc - self.mhc_num_residual_streams = mhc_num_residual_streams - self.mhc_tau = mhc_tau - self.hres_vwnstyle = hres_vwnstyle - self.hc_eps = hc_eps - self.mhc_no_norm_weight = mhc_no_norm_weight - self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations - self.mhc_post_mult_value = mhc_post_mult_value - - self.swiglu_limit = swiglu_limit - self.logit_scale = logit_scale - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - @property - def is_mla(self): - return ( - self.q_lora_rank is not None - or self.kv_lora_rank is not None - or self.qk_nope_head_dim is not None - or self.qk_rope_head_dim is not None - or self.v_head_dim is not None - or self.mla_nope is True - ) - - @property - def is_moe(self): - return self.n_routed_experts is not None - - @property - def is_linear_attn(self) -> bool: - return self.layer_types is not None and any( - t == "linear_attention" for t in self.layer_types - ) - - def is_kda_layer(self, layer_idx: int): - return ( - self.layer_types is not None - and layer_idx < len(self.layer_types) - and self.layer_types[layer_idx] == "linear_attention" - ) - - @property - def layers_block_type(self): - # Map the schema's per-layer types onto the block strings vLLM's hybrid - # accounting (get_num_layers_by_block_type) recognizes: linear-attention - # layers stay "linear_attention"; every other attention variant collapses - # to "attention". - if self.layer_types is None: - return ["attention"] * self.num_hidden_layers - return [ - "linear_attention" if t == "linear_attention" else "attention" - for t in self.layer_types - ] - - -class Glm5NextVisionConfig(PreTrainedConfig): - model_type = "glm5_next_vision" - base_config_key = "vision_config" - - def __init__( - self, - depth: int = 24, - hidden_size: int = 1024, - hidden_act: str = "silu", - image_size: int = 448, - intermediate_size: int = 4096, - num_heads: int = 16, - out_hidden_size: int = 4096, - projection_intermediate_size: int = 10240, - in_channels: int = 3, - initializer_range: float = 0.02, - patch_size: int = 14, - rms_norm_eps: float = 1e-5, - spatial_merge_size: int = 2, - temporal_patch_size: int = 2, - attention_dropout: float = 0.0, - attention_bias: bool = True, - swiglu_limit: float | None = None, - **kwargs, - ): - super().__init__(**kwargs) - - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.image_size = image_size - self.intermediate_size = intermediate_size - self.num_heads = num_heads - self.out_hidden_size = out_hidden_size - # GLM-5.3-Flash merger bottleneck width (absent from the generic - # GLM-OCR vision config); the tower uses it as the PatchMerger - # context_dim instead of text_config.intermediate_size. - self.projection_intermediate_size = projection_intermediate_size - self.in_channels = in_channels - self.initializer_range = initializer_range - self.patch_size = patch_size - # GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5, - # but the vision tower was trained with 1e-6. Serving with 1e-5 drifts - # the RMSNorm and produces repetitive/degraded image descriptions, so - # force the trained value regardless of the checkpoint field. - self.rms_norm_eps = 1e-6 - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.attention_dropout = attention_dropout - self.attention_bias = attention_bias - self.swiglu_limit = swiglu_limit - - -class Glm5NextConfig(PreTrainedConfig): - model_type = "glm5_next" - sub_configs = { - "vision_config": Glm5NextVisionConfig, - "text_config": Glm5NextTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config=None, - vision_config=None, - image_token_id: int = 154854, - video_token_id: int = 154855, - image_start_token_id: int = 154830, - image_end_token_id: int = 154831, - video_start_token_id: int = 154832, - video_end_token_id: int = 154833, - **kwargs, - ): - # Init super() first so base-class defaults don't clobber text-config - # values set below (PreTrainedConfig has many text-related defaults - # that differ from Glm5NextTextConfig). - super().__init__(**kwargs) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - # Backward compatibility: a flat top-level checkpoint (no nested - # text_config) folds its text fields into Glm5NextTextConfig. - self.text_config = self.sub_configs["text_config"](**kwargs) - else: - self.text_config = text_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.image_start_token_id = image_start_token_id - self.image_end_token_id = image_end_token_id - self.video_start_token_id = video_start_token_id - self.video_end_token_id = video_end_token_id - - # Mirror attention implementation recursively onto sub-configs. - self._attn_implementation = kwargs.pop("attn_implementation", None) - - # Config-metadata fields that belong to the top-level (multimodal) config - # and must NOT be mirrored onto text_config: ``architectures`` / - # ``torch_dtype`` differ between the top-level config and the text - # sub-config, and mirroring them makes the top-level ``architectures`` - # silently read back as None (PreTrainedConfig initializes both to None), - # which then fails model-class resolution ("No model architectures are - # specified"). - _UNMIRRORED_KEYS = [ - "_name_or_path", - "model_type", - "dtype", - "torch_dtype", - "architectures", - "_attn_implementation_internal", - ] - - def __setattr__(self, key, value): - unmirrored = type(self)._UNMIRRORED_KEYS - if ( - (text_config := super().__getattribute__("__dict__").get("text_config")) - is not None - and key not in unmirrored - and key in text_config.__dict__ - ): - setattr(text_config, key, value) - else: - super().__setattr__(key, value) - - def __getattribute__(self, key): - unmirrored = type(self)._UNMIRRORED_KEYS - if ( - "text_config" in super().__getattribute__("__dict__") - and key not in unmirrored - ): - text_config = super().__getattribute__("text_config") - # Forward both instance attributes AND class-defined properties/ - # methods of the text config, so a flat text-only checkpoint - # (model_type "glm5_next", no nested text_config) sees is_moe / - # is_kda_layer / layers_block_type like a Glm5NextTextConfig. - if key in text_config.__dict__ or key in type(text_config).__dict__: - return getattr(text_config, key) - - return super().__getattribute__(key) diff --git a/vllm/transformers_utils/configs/qwen4_exp.py b/vllm/transformers_utils/configs/qwen4_exp.py deleted file mode 100644 index ed4c6cca8e..0000000000 --- a/vllm/transformers_utils/configs/qwen4_exp.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Exports for the package-owned Qwen4Exp configs.""" - -from vllm.models.qwen4_exp.config import ( - Qwen4ExpConfig, - Qwen4ExpTextConfig, - Qwen4ExpVisionConfig, -) - -__all__ = [ - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", -] diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index b46183f4e1..b913545377 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -17,7 +17,6 @@ __all__ = [ "FireRedASR2Processor", "FunASRProcessor", "GLM4VProcessor", - "Glm5NextProcessor", "Granite4VisionProcessor", "H2OVLProcessor", "Moondream3Processor", @@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2", "FunASRProcessor": "vllm.transformers_utils.processors.funasr", "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v", - "Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next", "Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision", "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl", "InternVLProcessor": "vllm.transformers_utils.processors.internvl", diff --git a/vllm/transformers_utils/processors/glm5next.py b/vllm/transformers_utils/processors/glm5next.py deleted file mode 100644 index a6cba9d77f..0000000000 --- a/vllm/transformers_utils/processors/glm5next.py +++ /dev/null @@ -1,953 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""vLLM-native multimodal processor for GLM-5.3-Flash.""" - -import math - -import numpy as np -import torch -from torchvision.transforms.v2 import functional as tvF -from transformers.image_processing_backends import ( - TorchvisionBackend, - group_images_by_shape, - reorder_images, -) -from transformers.image_processing_utils import BatchFeature -from transformers.image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - ChannelDimension, - ImageInput, - PILImageResampling, - SizeDict, - get_image_size, -) -from transformers.models.auto.image_processing_auto import get_image_processor_config -from transformers.processing_utils import ( - ImagesKwargs, - MultiModalData, - ProcessingKwargs, - ProcessorMixin, - Unpack, - VideosKwargs, -) -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput -from transformers.utils import TensorType, logging -from transformers.video_processing_utils import BaseVideoProcessor -from transformers.video_utils import ( - VideoInput, - VideoMetadata, - group_videos_by_shape, - reorder_videos, -) - -from vllm.transformers_utils.repo_utils import get_hf_file_to_dict - -logger = logging.get_logger(__name__) - -# Cap video inputs at 30,000 vision tokens to keep encoder profiling from -# starving the KV cache. Image inputs retain their checkpoint-defined budget. -_MAX_VIDEO_TOKENS = 30000 - -# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 / -# max_frame_count_dynamic=2048); used when neither the request nor the -# processor config overrides them. -GLM_VIDEO_DEFAULT_FPS = 2.0 -GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048 - - -def glm_sample_frame_indices( - total_frames: int, - fps: float, - duration: float, - *, - target_fps: float | None = None, - max_frame_count: int | None = None, - temporal_patch_size: int = 2, -) -> list[int]: - """GLM video frame sampling (training-reference parity). - - ``target_fps`` is the ``fps_interval`` request knob. The greedy walk - advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on - frame-dense sources it collects more candidates than ``extract_t`` and - the ``> extract_t`` fixup re-spreads the picks uniformly with - ``np.linspace`` -- that fallback is the intended reference behavior, not - an accident. Short clips (fewer frames than ``extract_t``) are spread at - evenly spaced timestamps (``floor`` sampling; the linspace variant - samples frames unevenly and cost 4 points on video grounding evals). - Request overrides: ``target_fps`` -> fps interval, ``max_frame_count`` - -> frame cap. - """ - max_frame_idx = total_frames - 1 - if not duration: - duration = (round(max_frame_idx / fps) + 1) if fps else 0 - if max_frame_count is None: - max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES - if target_fps is None: - target_fps = GLM_VIDEO_DEFAULT_FPS - - extract_t = int(duration * target_fps) - extract_t = min(extract_t, int(max_frame_count)) - - duration_per_frame = 1 / fps - timestamps = [i * duration_per_frame for i in range(total_frames)] - max_second = int(duration) - - if total_frames < extract_t: - frame_indices = [ - math.floor(_i * total_frames / extract_t) for _i in range(extract_t) - ] - else: - frame_indices = [] - current_second = 0.0 - inv_fps = 1 / (temporal_patch_size * target_fps) - for frame_index in range(total_frames): - if timestamps[frame_index] >= current_second: - current_second += inv_fps - frame_indices.append(frame_index) - if current_second >= max_second: - break - - if len(frame_indices) < extract_t: - if len(frame_indices) == 0: - start, end = 0, max(total_frames - 1, 0) - else: - start, end = frame_indices[0], frame_indices[-1] - frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() - elif len(frame_indices) > extract_t: - frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() - - seen, uniq = set(), [] - for idx in frame_indices: - if idx not in seen: - seen.add(idx) - uniq.append(int(idx)) - - if len(uniq) & 1: - uniq.append(uniq[-1]) - - return uniq - - -def _ceil_to_factor(value: int, factor: int) -> int: - """Round a positive integer upward to the nearest multiple of factor.""" - return math.ceil(value / factor) * factor - - -def _fit_aligned_size_within_budget( - t: int, - h: int, - w: int, - h_factor: int, - w_factor: int, - max_pixels: int, -) -> tuple[int, int]: - """Largest proportional size whose upward-aligned canvas fits the budget. - - Binary search on the unaligned content height; each candidate is rounded - upward to h_factor/w_factor, so the returned canvas always satisfies - ``t * aligned_h * aligned_w <= max_pixels``. - """ - minimum_pixels = t * h_factor * w_factor - if max_pixels < minimum_pixels: - raise ValueError( - f"max_pixels={max_pixels} is too small. At least " - f"{minimum_pixels} pixels are required for one aligned patch." - ) - - low, high = 1, h - best_h, best_w = h_factor, w_factor - while low <= high: - content_h = (low + high) // 2 - content_w = max(1, math.floor(w * content_h / h)) - aligned_h = _ceil_to_factor(content_h, h_factor) - aligned_w = _ceil_to_factor(content_w, w_factor) - if t * aligned_h * aligned_w <= max_pixels: - best_h, best_w = aligned_h, aligned_w - low = content_h + 1 - else: - high = content_h - 1 - return best_h, best_w - - -def smart_resize( - t: int, - h: int, - w: int, - t_factor: int = 1, - h_factor: int = 28, - w_factor: int = 28, - min_pixels: int = 56 * 56, - max_pixels: int = 14 * 14 * 4 * 1280, -) -> tuple[int, int]: - """GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a - ``t_bar * h_bar * w_bar`` pixel budget. - - Height/width always round UP to their factors (content is then padded, - never cropped or distorted); an over-budget canvas is refit by binary - search instead of one-shot square-root scaling. ``h_factor`` / - ``w_factor`` carry ``patch_expand_factor`` on top of - ``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For - a still image ``t = t_factor = temporal_patch_size`` so ``t_bar = - temporal_patch_size``. - """ - if min(t, h, w, t_factor, h_factor, w_factor) <= 0: - raise ValueError("Image dimensions and alignment factors must be positive.") - if min_pixels <= 0 or max_pixels <= 0: - raise ValueError("min_pixels and max_pixels must be positive.") - if min_pixels > max_pixels: - raise ValueError("min_pixels must be less than or equal to max_pixels.") - - t_bar = max(t_factor, round(t / t_factor) * t_factor) - h_bar = _ceil_to_factor(h, h_factor) - w_bar = _ceil_to_factor(w, w_factor) - - if t_bar * h_bar * w_bar > max_pixels: - h_bar, w_bar = _fit_aligned_size_within_budget( - t=t_bar, - h=h, - w=w, - h_factor=h_factor, - w_factor=w_factor, - max_pixels=max_pixels, - ) - elif t_bar * h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (t * h * w)) - h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor) - w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor) - - # Alignment can push a candidate slightly over a tight max_pixels - # budget. Refit it when that happens. - if t_bar * h_bar * w_bar > max_pixels: - h_bar, w_bar = _fit_aligned_size_within_budget( - t=t_bar, - h=h, - w=w, - h_factor=h_factor, - w_factor=w_factor, - max_pixels=max_pixels, - ) - - return h_bar, w_bar - - -def _get_pad_content_size( - image_height: int, - image_width: int, - canvas_height: int, - canvas_width: int, - allow_upscale: bool = False, -) -> tuple[int, int]: - """Aspect-ratio-preserving content size that fits the canvas. - - Oversized images are shrunk proportionally. Small images are enlarged - only when ``allow_upscale``. Padding is applied after the resize. - """ - scale = min(canvas_height / image_height, canvas_width / image_width) - if not allow_upscale: - scale = min(1.0, scale) - content_height = max(1, min(canvas_height, math.floor(image_height * scale))) - content_width = max(1, min(canvas_width, math.floor(image_width * scale))) - return content_height, content_width - - -def _resize_or_pad( - stacked_images: torch.Tensor, - target_height: int, - target_width: int, - resize_mode: str, - resample: "PILImageResampling | tvF.InterpolationMode | int | None", - resize, - allow_upscale: bool = False, -) -> torch.Tensor: - """Resize onto the aligned canvas, or keep the aspect ratio and - zero-pad the right/bottom sides (``resize_mode="pad"``).""" - height, width = stacked_images.shape[-2:] - - if resize_mode == "resize": - return resize( - stacked_images, - size=SizeDict(height=target_height, width=target_width), - resample=resample, - ) - - if resize_mode != "pad": - raise ValueError("resize_mode must be either 'resize' or 'pad'.") - - content_height, content_width = _get_pad_content_size( - image_height=height, - image_width=width, - canvas_height=target_height, - canvas_width=target_width, - allow_upscale=allow_upscale, - ) - - if (content_height, content_width) != (height, width): - stacked_images = resize( - stacked_images, - size=SizeDict(height=content_height, width=content_width), - resample=resample, - ) - - # torchvision padding order: [left, top, right, bottom] -> pad only the - # right and bottom sides. - return tvF.pad( - stacked_images, - padding=[0, 0, target_width - content_width, target_height - content_height], - fill=0, - ) - - -def _pixel_budget( - min_image_tokens: int | None, - max_image_tokens: int | None, - patch_size: int, - merge_size: int, - temporal_patch_size: int, -) -> tuple[int, int]: - """(min_pixels, max_pixels) from the token bounds of - ``processor_config.json``; one vision token covers - ``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels.""" - if min_image_tokens is None or max_image_tokens is None: - raise ValueError( - "min_image_tokens and max_image_tokens must be provided by " - "processor_config.json (or per-call kwargs)." - ) - factor = temporal_patch_size * (patch_size * merge_size) ** 2 - return min_image_tokens * factor, max_image_tokens * factor - - -class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg] - patch_size: int | None - temporal_patch_size: int | None - merge_size: int | None - patch_expand_factor: int | None - resize_mode: str | None - min_image_tokens: int | None - max_image_tokens: int | None - - -class Glm5NextImageProcessor(TorchvisionBackend): - """Fast torchvision image processor for GLM-5.3-Flash. - - ``patch_expand_factor`` multiplies into the ``smart_resize`` spatial - factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry: - ``"pad"`` (default) preserves the aspect ratio and zero-pads the - right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto - it. Defaults mirror the checkpoint's ``image_processor`` config. - """ - - do_resize = True - resample = PILImageResampling.BICUBIC - size = {"longest_edge": 1} # unused: budgets come from the token bounds - do_rescale = True - do_normalize = True - image_mean = OPENAI_CLIP_MEAN - image_std = OPENAI_CLIP_STD - do_convert_rgb = True - patch_size = 14 - temporal_patch_size = 2 - merge_size = 2 - patch_expand_factor = 1 - resize_mode = "pad" - min_image_tokens = 16 - max_image_tokens = 8000 - valid_kwargs = Glm5NextImageProcessorKwargs - model_input_names = ["pixel_values", "image_grid_thw"] - - def _preprocess( - self, - images: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - resample: "PILImageResampling | tvF.InterpolationMode | int | None", - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: float | list[float] | None, - image_std: float | list[float] | None, - patch_size: int, - temporal_patch_size: int, - merge_size: int, - patch_expand_factor: int, - resize_mode: str | None, - min_image_tokens: int | None, - max_image_tokens: int | None, - disable_grouping: bool | None, - return_tensors: str | TensorType | None, - **kwargs, - ) -> BatchFeature: - resize_mode = resize_mode if resize_mode is not None else self.resize_mode - min_pixels, max_pixels = _pixel_budget( - min_image_tokens if min_image_tokens is not None else self.min_image_tokens, - max_image_tokens if max_image_tokens is not None else self.max_image_tokens, - patch_size, - merge_size, - temporal_patch_size, - ) - grouped_images, grouped_images_index = group_images_by_shape( - images, disable_grouping=disable_grouping - ) - resized_images_grouped = {} - for shape, stacked_images in grouped_images.items(): - height, width = stacked_images.shape[-2:] - if do_resize: - resized_height, resized_width = smart_resize( - t=temporal_patch_size, - h=height, - w=width, - t_factor=temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - stacked_images = _resize_or_pad( - stacked_images, - target_height=resized_height, - target_width=resized_width, - resize_mode=resize_mode, - resample=resample, - resize=self.resize, - allow_upscale=(temporal_patch_size * height * width < min_pixels), - ) - resized_images_grouped[shape] = stacked_images - - resized_images = reorder_images(resized_images_grouped, grouped_images_index) - - grouped_images, grouped_images_index = group_images_by_shape( - resized_images, disable_grouping=disable_grouping - ) - processed_images_grouped = {} - processed_grids = {} - - for shape, stacked_images in grouped_images.items(): - resized_height, resized_width = stacked_images.shape[-2:] - - patches = self.rescale_and_normalize( - stacked_images, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - if patches.ndim == 4: # (B, C, H, W) - patches = patches.unsqueeze(1) # (B, T=1, C, H, W) - - if patches.shape[1] % temporal_patch_size != 0: - repeats = patches[:, -1:].repeat( - 1, - temporal_patch_size - (patches.shape[1] % temporal_patch_size), - 1, - 1, - 1, - ) - patches = torch.cat([patches, repeats], dim=1) - - batch_size, t_len, channel = patches.shape[:3] - grid_t = t_len // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - # (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channel * temporal_patch_size * patch_size * patch_size, - ) - - processed_images_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_images = reorder_images( - processed_images_grouped, grouped_images_index - ) - processed_grids = reorder_images(processed_grids, grouped_images_index) - - pixel_values = torch.cat(processed_images, dim=0) - image_grid_thw = torch.tensor(processed_grids) - - return BatchFeature( - data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, - tensor_type=return_tensors, - ) - - def preprocess( - self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs] - ) -> BatchFeature: - return super().preprocess(images, **kwargs) - - def get_number_of_image_patches( - self, height: int, width: int, images_kwargs: dict | None = None - ) -> int: - """Number of image patches (pre-merge) for a given (height, width).""" - images_kwargs = images_kwargs or {} - patch_size = images_kwargs.get("patch_size", self.patch_size) - merge_size = images_kwargs.get("merge_size", self.merge_size) - patch_expand_factor = images_kwargs.get( - "patch_expand_factor", self.patch_expand_factor - ) - min_pixels, max_pixels = _pixel_budget( - images_kwargs.get("min_image_tokens", self.min_image_tokens), - images_kwargs.get("max_image_tokens", self.max_image_tokens), - patch_size, - merge_size, - self.temporal_patch_size, - ) - resized_height, resized_width = smart_resize( - t=self.temporal_patch_size, - h=height, - w=width, - t_factor=self.temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - return grid_h * grid_w - - -class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg] - fps: list[float] | float - patch_size: int - temporal_patch_size: int - merge_size: int - patch_expand_factor: int - resize_mode: str | None - target_fps: float | None - max_frames: int | None - fps_interval: int | None - max_frame_count_dynamic: int | None - min_image_tokens: int | None - max_image_tokens: int | None - - -class Glm5NextVideoProcessor(BaseVideoProcessor): - """Fast video processor for GLM-5.3-Flash. - - Shares ``smart_resize`` / the pad-mode geometry / the patchify with the - image processor, and adds GLM-5.3-Flash frame sampling - (``glm_sample_frame_indices``: ``fps_interval`` semantics with a - temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's - ``video_processor`` config. - """ - - resample = PILImageResampling.BICUBIC - size = {"longest_edge": 1} # unused: budgets come from the token bounds - image_mean = OPENAI_CLIP_MEAN - image_std = OPENAI_CLIP_STD - do_resize = True - do_rescale = True - do_normalize = True - do_convert_rgb = True - do_sample_frames = True - patch_size = 14 - temporal_patch_size = 2 - patch_expand_factor = 1 - merge_size = 2 - valid_kwargs = Glm5NextVideoProcessorKwargs - num_frames = 16 - fps = 2 - fps_interval = 2.0 - max_frame_count_dynamic = 2048 - resize_mode = "pad" - min_image_tokens = 16 - max_image_tokens = 240000 - model_input_names = ["pixel_values_videos", "video_grid_thw"] - - def sample_frames( - self, - metadata: VideoMetadata, - fps: int | float | None = None, - **kwargs, - ) -> np.ndarray: - """Sample frame indices with GLM's fps-interval policy. - - ``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` / - ``max_frame_count_dynamic`` are the overrides described in - :func:`glm_sample_frame_indices`. - """ - if metadata is None or getattr(metadata, "fps", None) is None: - raise ValueError( - "Asked to sample frames per second but no video metadata was " - "provided which is required when sampling in GLM-5.3-Flash. Please " - "pass in `VideoMetadata` object or set `do_sample_frames=False`." - ) - - target_fps = fps if fps is not None else kwargs.get("target_fps") - if target_fps is None: - target_fps = self.fps_interval - indices = glm_sample_frame_indices( - metadata.total_num_frames, - metadata.fps, - metadata.duration or 0, - target_fps=target_fps, - max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic, - temporal_patch_size=self.temporal_patch_size, - ) - return np.array(indices) - - def _preprocess( - self, - videos: list[torch.Tensor], - do_convert_rgb: bool = True, - do_resize: bool = True, - size: SizeDict | None = None, - resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: float = 1 / 255.0, - do_normalize: bool = True, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - patch_size: int | None = None, - temporal_patch_size: int | None = None, - patch_expand_factor: int | None = None, - merge_size: int | None = None, - resize_mode: str | None = None, - min_image_tokens: int | None = None, - max_image_tokens: int | None = None, - return_tensors: str | TensorType | None = None, - **kwargs, - ) -> BatchFeature: - patch_expand_factor = self.patch_expand_factor - patch_size = patch_size if patch_size is not None else self.patch_size - temporal_patch_size = ( - temporal_patch_size - if temporal_patch_size is not None - else self.temporal_patch_size - ) - merge_size = merge_size if merge_size is not None else self.merge_size - resize_mode = resize_mode if resize_mode is not None else self.resize_mode - min_pixels, max_pixels = _pixel_budget( - min_image_tokens if min_image_tokens is not None else self.min_image_tokens, - max_image_tokens if max_image_tokens is not None else self.max_image_tokens, - patch_size, - merge_size, - temporal_patch_size, - ) - grouped_videos, grouped_videos_index = group_videos_by_shape(videos) - resized_videos_grouped = {} - for shape, stacked_videos in grouped_videos.items(): - if do_convert_rgb: - stacked_videos = self.convert_to_rgb(stacked_videos) - b, t_len, c, h, w = stacked_videos.shape - num_frames, height, width = t_len, h, w - if do_resize: - resized_height, resized_width = smart_resize( - t=num_frames, - h=height, - w=width, - t_factor=temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - stacked_videos = stacked_videos.view(b * t_len, c, h, w) - stacked_videos = _resize_or_pad( - stacked_videos, - target_height=resized_height, - target_width=resized_width, - resize_mode=resize_mode, - resample=resample, - resize=self.resize, - allow_upscale=(num_frames * height * width < min_pixels), - ) - stacked_videos = stacked_videos.view( - b, t_len, c, resized_height, resized_width - ) - resized_videos_grouped[shape] = stacked_videos - resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) - - grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) - processed_videos_grouped = {} - processed_grids = {} - for shape, stacked_videos in grouped_videos.items(): - resized_height, resized_width = get_image_size( - stacked_videos[0], channel_dim=ChannelDimension.FIRST - ) - stacked_videos = self.rescale_and_normalize( - stacked_videos, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - patches = stacked_videos - - if pad := -patches.shape[1] % temporal_patch_size: - repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1) - patches = torch.cat((patches, repeats), dim=1) - batch_size, grid_t, channel = patches.shape[:3] - grid_t = grid_t // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channel * temporal_patch_size * patch_size * patch_size, - ) - - processed_videos_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_videos = reorder_videos( - processed_videos_grouped, grouped_videos_index - ) - processed_grids = reorder_videos(processed_grids, grouped_videos_index) - pixel_values_videos = torch.cat(processed_videos, dim=0) - video_grid_thw = torch.tensor(processed_grids) - return BatchFeature( - data={ - "pixel_values_videos": pixel_values_videos, - "video_grid_thw": video_grid_thw, - }, - tensor_type=return_tensors, - ) - - -class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - images_kwargs: Glm5NextImageProcessorKwargs - videos_kwargs: Glm5NextVideoProcessorKwargs - _defaults = { - "text_kwargs": { - "padding": False, - "return_token_type_ids": False, - "return_mm_token_type_ids": False, - }, - "videos_kwargs": {"return_metadata": True}, - } - - -class Glm5NextProcessor(ProcessorMixin): - """Wraps a GLM-5.3-Flash image processor, video processor and tokenizer. - - Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video - frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds`` - structure (mrope timestamps). - """ - - attributes = ["image_processor", "tokenizer", "video_processor"] - image_processor_class = "AutoImageProcessor" - video_processor_class = "AutoVideoProcessor" - tokenizer_class = ("PythonBackend", "TokenizersBackend") - - def __init__( - self, - image_processor=None, - tokenizer=None, - video_processor=None, - chat_template=None, - **kwargs, - ) -> None: - super().__init__( - image_processor, tokenizer, video_processor, chat_template=chat_template - ) - self.image_token = ( - "<|image|>" - if not hasattr(tokenizer, "image_token") - else tokenizer.image_token - ) - self.video_token = ( - "<|video|>" - if not hasattr(tokenizer, "video_token") - else tokenizer.video_token - ) - self.image_token_id = ( - tokenizer.image_token_id - if getattr(tokenizer, "image_token_id", None) - else tokenizer.convert_tokens_to_ids(self.image_token) - ) - self.video_token_id = ( - tokenizer.video_token_id - if getattr(tokenizer, "video_token_id", None) - else tokenizer.convert_tokens_to_ids(self.video_token) - ) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - """Build the processor directly from the checkpoint config. - - GLM-5.3-Flash stores nested image/video configs in - ``processor_config.json`` and declares a custom processor class. This - method reads those configs directly and caps only the video token budget. - """ - from transformers import AutoTokenizer - - model_path = pretrained_model_name_or_path - tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs) - - def _cap_cfg(cfg: dict, *, is_video: bool) -> dict: - # Video keeps a serving token cap (the checkpoint's 240k-token - # budget would starve the KV cache at startup profiling); images - # follow the checkpoint budget verbatim so preprocessing matches - # the HF reference exactly. - if is_video and cfg.get("max_image_tokens") is not None: - cfg["max_image_tokens"] = min( - cfg["max_image_tokens"], _MAX_VIDEO_TOKENS - ) - return cfg - - ip_cfg = _cap_cfg( - dict(get_image_processor_config(model_path, **kwargs)), is_video=False - ) - image_processor = Glm5NextImageProcessor( - **{k: v for k, v in ip_cfg.items() if k != "image_processor_type"} - ) - - processor_config = get_hf_file_to_dict( - "processor_config.json", - model_path, - revision=kwargs.get("revision", "main"), - ) - if processor_config is None: - raise ValueError(f"Missing processor_config.json for {model_path}") - vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True) - video_processor = Glm5NextVideoProcessor( - **{k: v for k, v in vp_cfg.items() if k != "video_processor_type"} - ) - - return cls( - image_processor=image_processor, - tokenizer=tokenizer, - video_processor=video_processor, - ) - - def __call__( - self, - images: ImageInput | None = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - videos: VideoInput | None = None, - **kwargs: Unpack[Glm5NextProcessorKwargs], - ) -> BatchFeature: - output_kwargs = self._merge_kwargs( - Glm5NextProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - if images is not None: - image_inputs = self.image_processor( - images=images, **output_kwargs["images_kwargs"] - ) - else: - image_inputs = {} - - if videos is not None: - videos_inputs = self.video_processor( - videos=videos, **output_kwargs["videos_kwargs"] - ) - if "return_metadata" not in kwargs: - videos_inputs.pop("video_metadata") - else: - videos_inputs = {} - - if not isinstance(text, list): - text = [text] - - # Prompt updates expand the unchanged image/video markers after this call. - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) - return_mm_token_type_ids = output_kwargs["text_kwargs"].pop( - "return_mm_token_type_ids", False - ) - text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) - - if return_mm_token_type_ids: - array_ids = np.array(text_inputs["input_ids"]) - mm_token_type_ids = np.zeros_like(text_inputs["input_ids"]) - mm_token_type_ids[array_ids == self.image_token_id] = 1 - text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist() - return BatchFeature( - data={**text_inputs, **image_inputs, **videos_inputs}, - tensor_type=return_tensors, - ) - - def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs): - vision_data = {} - if image_sizes is not None: - images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {}) - images_kwargs.update(kwargs) - merge_size = ( - images_kwargs.get("merge_size", None) or self.image_processor.merge_size - ) - - num_image_patches = [ - self.image_processor.get_number_of_image_patches( - *image_size, images_kwargs - ) - for image_size in image_sizes - ] - num_image_tokens = [(n // merge_size**2) for n in num_image_patches] - vision_data.update( - { - "num_image_tokens": num_image_tokens, - "num_image_patches": num_image_patches, - } - ) - - if video_sizes is not None: - videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {}) - videos_kwargs.update(kwargs) - num_video_patches = [ - self.video_processor.get_number_of_video_patches( - *video_size, videos_kwargs - ) - for video_size in video_sizes - ] - num_video_tokens = [(n // merge_size**2) for n in num_video_patches] - vision_data["num_video_tokens"] = num_video_tokens - - return MultiModalData(**vision_data) - - def post_process_image_text_to_text( - self, - generated_outputs, - skip_special_tokens=True, - clean_up_tokenization_spaces=False, - **kwargs, - ): - return self.tokenizer.batch_decode( - generated_outputs, - skip_special_tokens=skip_special_tokens, - clean_up_tokenization_spaces=clean_up_tokenization_spaces, - **kwargs, - ) - - -__all__ = [ - "Glm5NextImageProcessor", - "Glm5NextVideoProcessor", - "Glm5NextProcessor", - "smart_resize", -]