iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 02:25:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 866f02db8a
commit e3cae8d2ac
36 changed files with 366 additions and 2385 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements/common.txt
+1 -1
View File
@@ -7,7 +7,7 @@ requests >= 2.33.0
tqdm
blake3
py-cpuinfo
transformers >= 5.10.4
transformers >= 5.16.1
huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export
tokenizers >= 0.21.1 # Required for fast incremental detokenization.
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
tests/models/glm5next/test_sequence_parallel.py
+1 -1
View File
@@ -15,11 +15,11 @@ from types import SimpleNamespace
import pytest
import torch
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.model_executor import parameter
from vllm.model_executor.layers import linear
from vllm.models.glm5next.common import model as glm_model
from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
tests/models/multimodal/processing/test_glm5next.py
+16 -33
View File
@@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU.
import pytest
import torch
from PIL import Image
from transformers.models.glm5_next.video_processing_glm5_next import (
Glm5NextVideoProcessor,
smart_resize,
)
from transformers.video_utils import VideoMetadata
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.glm5next import (
Glm5NextVideoProcessor,
_pixel_budget,
glm_sample_frame_indices,
smart_resize,
)
from ...utils import build_model_context
@@ -48,27 +47,16 @@ def _pixel_path_grid(
width: int,
) -> tuple[int, int, int]:
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
min_pixels, max_pixels = _pixel_budget(
video_processor.min_image_tokens,
video_processor.max_image_tokens,
video_processor.patch_size,
video_processor.merge_size,
video_processor.temporal_patch_size,
)
factor = (
video_processor.patch_size
* video_processor.merge_size
* video_processor.patch_expand_factor
)
resized_height, resized_width = smart_resize(
t=num_frames,
h=height,
w=width,
t_factor=video_processor.temporal_patch_size,
h_factor=factor,
w_factor=factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
num_frames=num_frames,
height=height,
width=width,
temporal_factor=video_processor.temporal_patch_size,
factor=video_processor.patch_size
* video_processor.merge_size
* video_processor.patch_expand_factor,
min_pixels=video_processor.min_image_tokens,
max_pixels=video_processor.max_image_tokens,
)
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
return (
@@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows(
info = processor.info
video_processor = info.get_video_processor()
frame_indices = glm_sample_frame_indices(
total_num_frames,
fps,
duration,
target_fps=video_processor.fps_interval,
max_frame_count=video_processor.max_frame_count_dynamic,
temporal_patch_size=video_processor.temporal_patch_size,
frame_indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
)
grid_t, grid_h, grid_w = _pixel_path_grid(
video_processor, len(frame_indices), height, width
tests/models/qwen4_exp/test_config.py
+5 -5
View File
@@ -7,16 +7,13 @@ from unittest.mock import patch
import pytest
import torch
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
)
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
@@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 0,
"num_experts": 4,
"num_experts_per_tok": 2,
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
"eos_token_id": 1,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
tests/models/qwen4_exp/test_ple.py
+1 -1
View File
@@ -10,6 +10,7 @@ import pytest
import torch
from torch import nn
from torch.nn import functional as F
from transformers import Qwen4ExpTextConfig
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
@@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import (
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEDeviceEmbedding,
tests/multimodal/test_video.py
+3 -15
View File
@@ -1591,10 +1591,6 @@ def test_glm5next_backend_indices_match_sampler(
):
"""The loader must select exactly the frames the processor's sampler
would, with target.fps mapping onto the raw-fps override."""
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
source = VideoSourceMetadata(
total_frames_num=total_frames, original_fps=original_fps, duration=duration
)
@@ -1604,7 +1600,7 @@ def test_glm5next_backend_indices_match_sampler(
source, target, max_frames=max_frames
)
assert indices == glm_sample_frame_indices(
assert indices == Glm5NextVideoBackend._sample_frame_indices(
total_frames,
original_fps,
duration,
@@ -1692,10 +1688,6 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
if backend == "torchcodec":
pytest.importorskip("torchcodec")
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
total_frames, fps = 120, 10
path = _write_gray_video(tmp_path, total_frames, fps)
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
@@ -1703,7 +1695,7 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
kwargs: dict[str, Any] = (
{} if max_frames is None else {"max_frames": max_frames}
)
expected = glm_sample_frame_indices(
expected = Glm5NextVideoBackend._sample_frame_indices(
total_frames, float(fps), 12.0, max_frame_count=max_frames
)
@@ -1726,11 +1718,7 @@ def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
total_frames, fps = 60, 10
path = _write_gray_video(tmp_path, total_frames, fps)
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0)
frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
tests/test_config.py
+44
View File
@@ -4114,3 +4114,47 @@ def test_revision_resolved_for_model(mock_resolve):
assert isinstance(config.revision, ResolvedRevision)
assert config.revision.resolved == REVISION
mock_resolve.assert_any_call(model, None, config.hf_token)
@pytest.mark.parametrize(
("layer_types", "expected_attention"),
[
# Qwen3-Next / Qwen3.5 spell their attention layers "full_attention".
(["linear_attention", "full_attention"], 1),
# GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches
# every token and so must count as attention.
(["linear_attention", "deepseek_sparse_attention"], 1),
(["linear_attention", "qwen_sparse_attention"], 1),
(["linear_attention", "linear_attention"], 0),
],
)
def test_hybrid_layer_counts_sparse_attention_as_attention(
layer_types, expected_attention
):
"""Sparse-attention layer types consume a full attention KV cache."""
model_config = object.__new__(ModelConfig)
model_config.hf_config = SimpleNamespace()
model_config.hf_text_config = SimpleNamespace(layer_types=layer_types)
model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next")
parallel_config = SimpleNamespace()
with (
patch.object(ModelConfig, "is_hybrid", True),
patch.object(ModelConfig, "has_noops", False),
patch.object(ModelConfig, "is_attention_free", False),
patch.object(
ModelConfig,
"get_layers_start_end_indices",
return_value=(0, len(layer_types)),
),
):
assert (
model_config.get_num_layers_by_block_type(parallel_config, "attention")
== expected_attention
)
assert (
model_config.get_num_layers_by_block_type(
parallel_config, "linear_attention"
)
== len(layer_types) - expected_attention
)
tests/transformers_utils/processors/test_glm5next.py
-389
View File
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor."""
import math
import pytest
import torch
from PIL import Image
from transformers.image_utils import PILImageResampling
from transformers.video_utils import VideoMetadata
from vllm.transformers_utils.processors.glm5next import (
Glm5NextImageProcessor,
Glm5NextVideoProcessor,
_get_pad_content_size,
_resize_or_pad,
glm_sample_frame_indices,
smart_resize,
)
PATCH_SIZE = 14
MERGE_SIZE = 2
PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json
FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28
PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568
MIN_TOKENS = 16
MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small
MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN
MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN
def resize(height: int, width: int, t: int = 2) -> tuple[int, int]:
return smart_resize(
t,
height,
width,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MIN_PIXELS,
max_pixels=MAX_PIXELS,
)
@pytest.mark.parametrize(
("height", "width", "t", "expected"),
[
(300, 400, 2, (308, 420)), # ceil alignment, no budget pressure
(50, 80, 2, (112, 168)), # below min budget: proportional upscale
(57, 3, 2, (504, 28)), # slim side ceils to one factor
(2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap
(112, 11200, 2, (84, 7420)), # extreme aspect stays proportional
(300, 400, 16, (252, 308)), # frame count eats into the budget
(112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds
],
)
def test_smart_resize_reference_values(height, width, t, expected):
got = smart_resize(
t,
height,
width,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MIN_PIXELS,
max_pixels=MAX_PIXELS,
)
assert got == expected
# The aligned canvas never exceeds the budget it was fitted against.
t_bar = max(2, round(t / 2) * 2)
assert t_bar * got[0] * got[1] <= MAX_PIXELS
def test_smart_resize_stays_snapped_and_positive():
heights = (1, 27, 57, 111, 113, 300, 720, 2160)
widths = (1, 27, 57, 111, 113, 400, 1280, 11200)
for h in heights:
for w in widths:
for t in (2, 4, 16, 64):
resized_h, resized_w = resize(h, w, t)
assert resized_h > 0 and resized_w > 0
assert resized_h % FACTOR == 0
assert resized_w % FACTOR == 0
t_bar = max(2, round(t / 2) * 2)
assert t_bar * resized_h * resized_w <= MAX_PIXELS
@pytest.mark.parametrize(
("height", "width", "match"),
[
(0, 100, "must be positive"),
(100, 0, "must be positive"),
],
)
def test_smart_resize_rejects_degenerate_inputs(height, width, match):
with pytest.raises(ValueError, match=match):
resize(height, width, 2)
def test_smart_resize_rejects_inverted_budget():
with pytest.raises(ValueError, match="min_pixels must be less than or equal"):
smart_resize(
2,
100,
100,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MAX_PIXELS,
max_pixels=MIN_PIXELS,
)
def test_get_pad_content_size():
# Oversized content shrinks proportionally, never upscales by default.
assert _get_pad_content_size(300, 400, 308, 420) == (300, 400)
assert _get_pad_content_size(600, 800, 308, 420) == (308, 410)
# allow_upscale enlarges small content toward the canvas.
assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112)
def test_resize_or_pad_pads_right_and_bottom():
stacked = torch.rand(1, 3, 300, 400)
def identity_resize(x, size, resample=None):
assert (size.height, size.width) == (300, 400)
return x
padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize)
assert padded.shape == (1, 3, 308, 420)
torch.testing.assert_close(padded[..., :300, :400], stacked)
assert padded[..., 300:, :].eq(0).all()
assert padded[..., :, 400:].eq(0).all()
def force_resize(x, size, resample=None):
return torch.zeros(x.shape[0], x.shape[1], size.height, size.width)
assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == (
1,
3,
308,
420,
)
with pytest.raises(ValueError, match="resize_mode"):
_resize_or_pad(stacked, 308, 420, "crop", None, identity_resize)
@pytest.fixture(scope="module")
def image_processor():
return Glm5NextImageProcessor(
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
)
@pytest.mark.parametrize(
("height", "width"),
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
)
def test_image_grid_matches_smart_resize(image_processor, height, width):
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
resized_h, resized_w = resize(height, width)
grid = out["image_grid_thw"][0].tolist()
assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE]
assert out["pixel_values"].shape == (
grid[1] * grid[2],
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
@pytest.mark.parametrize(
("height", "width"),
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
)
def test_image_patch_count_matches_preprocess(image_processor, height, width):
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
grid = out["image_grid_thw"][0].tolist()
expected = grid[1] * grid[2]
assert image_processor.get_number_of_image_patches(height, width) == expected
@pytest.fixture(scope="module")
def video_processor():
return Glm5NextVideoProcessor(
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
)
def run_video_preprocess(video_processor, frames, **overrides):
kwargs = dict(
resample=PILImageResampling.BICUBIC,
image_mean=(0.48145466, 0.4578275, 0.40821073),
image_std=(0.26862954, 0.26130258, 0.27577711),
patch_size=PATCH_SIZE,
temporal_patch_size=2,
merge_size=MERGE_SIZE,
patch_expand_factor=PATCH_EXPAND_FACTOR,
return_tensors="pt",
)
kwargs.update(overrides)
return video_processor._preprocess([frames], **kwargs)
def test_video_preprocess_pads_odd_frame_count(video_processor):
out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400))
grid = out["video_grid_thw"][0].tolist()
# 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420).
assert grid == [4, 22, 30]
assert out["pixel_values_videos"].shape == (
4 * 22 * 30,
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor):
out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200))
# Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap.
assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0
def test_video_preprocess_pads_content_not_distort(video_processor):
frames = torch.zeros(4, 3, 300, 400)
frames[..., 50, 60] = 1.0 # marker inside the content area
# Bypass rescale/normalize so zero padding stays exactly zero.
out = run_video_preprocess(
video_processor, frames, do_rescale=False, do_normalize=False
)
# Pad mode keeps the 300x400 content aspect on the (308, 420) canvas
# with zero padding on the right/bottom.
grid = out["video_grid_thw"][0].tolist()
assert grid == [2, 22, 30]
patches = out["pixel_values_videos"].view(
grid[0],
grid[1] // MERGE_SIZE,
grid[2] // MERGE_SIZE,
MERGE_SIZE,
MERGE_SIZE,
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1)
# Patch columns fully right of the 400px content are pure padding.
assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0
@pytest.mark.parametrize(
("total_frames", "fps", "duration", "expected_len", "first", "last"),
[
# Dense source: the tp-scaled greedy overshoots extract_t, so the
# fixup spreads picks uniformly across the whole video (linspace).
(900, 30.0, 30.0, 60, 0, 899),
(3000, 30.0, 100.0, 200, 0, 2999),
# extract_t capped at 2048 frames.
(72000, 30.0, 2400.0, 2048, 0, 71999),
# Low container fps: the greedy picks every frame.
(48, 2.0, 24.0, 48, 0, 47),
# Duration derived from the frame count when metadata lacks it.
(300, 25.0, 0, 26, 0, 299),
],
)
def test_glm_sample_frame_indices_behaviour(
total_frames, fps, duration, expected_len, first, last
):
indices = glm_sample_frame_indices(total_frames, fps, duration)
assert len(indices) == expected_len
assert indices[0] == first
assert indices[-1] == last
assert len(indices) % 2 == 0
assert indices == sorted(indices)
assert all(0 <= idx < total_frames for idx in indices)
def test_glm_sample_frame_indices_short_clip_floor_spread():
# extract_t (20) > total (7): evenly spaced timestamps, deduplicated,
# then pair-padded to an even count.
assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6]
@pytest.mark.parametrize(
("kwargs", "expected_len"),
[
({"target_fps": 0.5}, 16),
({"max_frame_count": 16}, 16),
({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap
],
)
def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len):
indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs)
assert len(indices) == expected_len
assert len(indices) % 2 == 0
def test_sample_frames_fps_interval(video_processor):
indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
)
assert len(indices) == 60
assert indices[0] == 0
assert indices[-1] == 899 # uniform spread reaches the final frame
# Request overrides reach the sampler through both kwarg spellings. The
# 15-frame spread is odd, so pair-padding duplicates the last frame.
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
fps=0.5,
)
)
== 16
)
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
target_fps=0.5,
)
)
== 16
)
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
max_frames=16,
)
)
== 16
)
def test_defaults_mirror_checkpoint_config():
"""Bare instantiation matches the checkpoint's ``processor_config.json``
token-budget style defaults; ``size`` carries no budget."""
image_processor = Glm5NextImageProcessor()
assert image_processor.patch_expand_factor == 1
assert image_processor.min_image_tokens == 16
assert image_processor.max_image_tokens == 8000
assert image_processor.resize_mode == "pad"
video_processor = Glm5NextVideoProcessor()
assert video_processor.patch_expand_factor == 1
assert video_processor.min_image_tokens == 16
assert video_processor.max_image_tokens == 240000
assert video_processor.fps_interval == 2.0
assert video_processor.max_frame_count_dynamic == 2048
def test_token_budgets_drive_geometry():
"""The token bounds fully determine the pixel budget and the grid."""
token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512)
img = Image.new("RGB", (400, 300))
grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist()
# Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420)
# without hitting either bound -> 22x30 patches.
assert grid == [1, 22, 30]
assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30
def test_missing_token_budgets_rejected():
proc = Glm5NextImageProcessor(min_image_tokens=None)
with pytest.raises(ValueError, match="min_image_tokens"):
proc(Image.new("RGB", (400, 300)), return_tensors="pt")
def test_video_config_fields_land():
"""fps_interval / max_frame_count_dynamic from the dedicated config
shape sampling without any request overrides."""
proc = Glm5NextVideoProcessor(
fps_interval=4,
max_frame_count_dynamic=32,
)
indices = proc.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
)
# 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32.
assert len(indices) == 32
assert indices[0] == 0
assert indices[-1] == 899
# Request overrides still win over the config values.
assert (
len(
proc.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
fps=0.5,
)
)
== 16
)
tests/transformers_utils/test_config.py
-53
View File
@@ -24,11 +24,6 @@ from vllm.transformers_utils.config import (
try_get_generation_config,
uses_mrope,
)
from vllm.transformers_utils.configs.glm5_next import (
Glm5NextConfig,
Glm5NextTextConfig,
Glm5NextVisionConfig,
)
from vllm.transformers_utils.configs.mistral import adapt_config_dict
@@ -174,54 +169,6 @@ def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling():
assert config.rope_parameters["attention_factor"] == 1.0
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
layer_types = ["linear_attention", "deepseek_sparse_attention"]
config = Glm5NextTextConfig(
num_hidden_layers=len(layer_types), layer_types=layer_types
)
assert config.layer_types == layer_types
assert config.layers_block_type == ["linear_attention", "attention"]
def test_glm5_next_accepts_prebuilt_subconfigs():
text_config = Glm5NextTextConfig(hidden_size=1024)
vision_config = Glm5NextVisionConfig(hidden_size=768)
config = Glm5NextConfig(
text_config=text_config,
vision_config=vision_config,
)
assert config.text_config is text_config
assert config.vision_config is vision_config
@pytest.mark.parametrize(
("kwargs", "option"),
[
(
{"index_topk": 2048, "index_dsa_use_layernorm": False},
"index_dsa_use_layernorm",
),
(
{"index_topk": 2048, "index_kpool_compress": False},
"index_kpool_compress",
),
(
{"index_topk": 2048, "index_kpool_always_select_tail": False},
"index_kpool_always_select_tail",
),
({"hres_vwnstyle": False}, "hres_vwnstyle"),
({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"),
],
)
def test_glm5_next_rejects_unimplemented_config_options(kwargs, option):
with pytest.raises(NotImplementedError, match=option):
Glm5NextTextConfig(**kwargs)
def test_get_llama3_eos_token():
model_name = "meta-llama/Llama-3.2-1B-Instruct"
tests/v1/spec_decode/test_mtp.py
+1 -1
View File
@@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config):
index_kpool=4,
)
vllm_config = mock.MagicMock()
vllm_config.speculative_config.draft_model_config.hf_config = config
vllm_config.speculative_config.draft_model_config.hf_text_config = config
vllm_config.scheduler_config.max_num_batched_tokens = 4
with (
vllm/config/model.py
+12 -1
View File
@@ -108,6 +108,16 @@ HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig]
ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"]
LayerBlockType = Literal["attention", "linear_attention", "mamba"]
_ATTENTION_LAYER_TYPES = frozenset(
{
"full_attention",
"deepseek_sparse_attention",
"qwen_sparse_attention",
}
)
"""`layer_types` spellings that consume a full attention KV cache. Sparse
attention still caches every token, so it counts as attention here."""
_RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
"generate": [],
"pooling": ["embed", "classify"],
@@ -1718,7 +1728,8 @@ class ModelConfig:
if layer_types_value is not None:
if block_type == "attention":
return sum(
t == "full_attention" for t in layer_types_value[start:end]
t in _ATTENTION_LAYER_TYPES
for t in layer_types_value[start:end]
)
elif block_type == "linear_attention":
return sum(
vllm/config/speculative.py
+1 -1
View File
@@ -1059,7 +1059,7 @@ class SpeculativeConfig:
)
if hf_config.model_type == "glm5_next":
hf_config.model_type = "glm5_next_mtp"
n_predict = hf_config.num_nextn_predict_layers
n_predict = hf_config.get_text_config().num_nextn_predict_layers
hf_config.update(
{"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]}
)
vllm/models/glm5next/common/attention.py
+18 -14
View File
@@ -4,6 +4,7 @@
import torch
import torch.nn.functional as F
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.config import (
CacheConfig,
@@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
@@ -221,7 +221,7 @@ class Indexer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Glm5NextConfig,
config: Glm5NextTextConfig,
hidden_size: int,
q_lora_rank: int,
quant_config: QuantizationConfig | None,
@@ -236,7 +236,7 @@ class Indexer(nn.Module):
# self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
# Indexer is only constructed for v32 configs, where these sparse-indexer
# fields are guaranteed populated; narrow away the `int | None` declared
# on Glm5NextConfig for the optional-indexer case.
# on Glm5NextTextConfig for the optional-indexer case.
assert config.index_topk is not None
assert config.index_n_heads is not None
assert config.index_head_dim is not None
@@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Glm5NextConfig,
config: Glm5NextTextConfig,
hidden_size: int,
num_heads: int,
qk_nope_head_dim: int,
@@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module):
prefix=f"{prefix}.o_proj",
)
# `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`;
# a checkpoint may still ship one.
rope_parameters = getattr(config, "rope_parameters", None)
if not skip_rope:
assert config.rope_parameters is not None
if config.rope_parameters["rope_type"] != "default":
config.rope_parameters["rope_type"] = (
assert rope_parameters is not None
if rope_parameters["rope_type"] != "default":
rope_parameters["rope_type"] = (
"deepseek_llama_scaling"
if config.rope_parameters.get("attention_factor") == 1.0
if rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
self.rotary_emb: RotaryEmbedding | None = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
rope_parameters=rope_parameters,
is_neox_style=False,
)
if (
config.rope_parameters["rope_type"] != "default"
and config.rope_parameters["rope_type"] == "deepseek_yarn"
rope_parameters["rope_type"] != "default"
and rope_parameters["rope_type"] == "deepseek_yarn"
):
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
scaling_factor = config.rope_parameters["factor"]
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
scaling_factor = rope_parameters["factor"]
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
self.scaling = self.scaling * mscale * mscale
else:
@@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module):
self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
rope_parameters=rope_parameters,
is_neox_style=not config.indexer_rope_interleave,
)
# The sparse indexer projects from the MLA q-lora rank, which is
vllm/models/glm5next/common/kda.py
+2 -2
View File
@@ -6,6 +6,7 @@ from dataclasses import replace
import torch
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig, get_current_vllm_config
@@ -44,7 +45,6 @@ from vllm.model_executor.utils import (
)
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.worker.workspace import current_workspace_manager
@@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
def __init__(
self,
config: Glm5NextConfig,
config: Glm5NextTextConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
vllm/models/glm5next/common/model.py
+74 -26
View File
@@ -6,6 +6,7 @@ from typing import ClassVar, Literal
import torch
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.config import ParallelConfig, VllmConfig
from vllm.distributed import (
@@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import (
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
from .attention import Glm5NextMLAAttention
from .kda import Glm5NextLinearAttention
@@ -95,6 +95,57 @@ from .multimodal import (
logger = init_logger(__name__)
_MHC_TAU = 0.05
"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the
checkpoint nor `Glm5NextTextConfig` carries."""
_MHC_POST_MULT_VALUE = 2.0
"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the
checkpoint nor `Glm5NextTextConfig` carries."""
_LOGIT_SCALE = 1.0
"""Output logit scale. A GLM-5.3-Flash trained value that neither the
checkpoint nor `Glm5NextTextConfig` carries."""
_VISION_RMS_NORM_EPS = 1e-6
"""Vision tower RMSNorm epsilon.
GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the
vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and
produces repetitive/degraded image descriptions, so force the trained value
regardless of the checkpoint field.
"""
def _is_moe(config: Glm5NextTextConfig) -> bool:
return config.n_routed_experts is not None
def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool:
layer_types = config.layer_types
return (
layer_types is not None
and layer_idx < len(layer_types)
and layer_types[layer_idx] == "linear_attention"
)
def _is_linear_attn(config: Glm5NextTextConfig) -> bool:
layer_types = config.layer_types
return layer_types is not None and "linear_attention" in layer_types
def _validate_supported_config(config: Glm5NextTextConfig) -> None:
"""Reject checkpoints using config options this implementation lacks.
The kpool indexer kernels always keep the incomplete trailing pool, so a
checkpoint asking otherwise would be served silently wrong.
"""
if config.index_topk is not None and not config.index_kpool_always_select_tail:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
)
class Glm5NextMLP(nn.Module):
def __init__(
@@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module):
class Glm5NextMoE(nn.Module):
def __init__(
self,
config: Glm5NextConfig,
config: Glm5NextTextConfig,
parallel_config: ParallelConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module):
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_token,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
renormalize=config.moe_renormalize,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
use_grouped_topk=True,
num_expert_group=config.n_group,
@@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Glm5NextConfig,
config: Glm5NextTextConfig,
layer_idx: int,
prefix: str = "",
topk_indices_buffer: torch.Tensor | None = None,
@@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
self.layer_idx = layer_idx
self.is_moe = config.is_moe
self.is_moe = _is_moe(config)
self.num_hidden_layers = config.num_hidden_layers
self.rms_norm_eps = config.rms_norm_eps
self.num_experts = config.n_routed_experts
self.is_mtp_layer = is_mtp_layer
self.mhc = config.mhc
is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx)
is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx)
self.layer_kind = "kda" if is_kda_layer else "mla"
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
@@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module):
quant_config=None, # MLA projections are BF16 in checkpoint
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
skip_rope=config.mla_nope,
skip_rope=config.mla_use_nope,
)
# MTP layers sit past the base model's hidden layers (layer_idx >=
@@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module):
if self.mhc and not is_mtp_layer:
# mhc config
self.mhc_num_residual_streams = config.mhc_num_residual_streams
self.mhc_tau = config.mhc_tau
self.mhc_num_residual_streams = config.hc_mult
self.mhc_tau = _MHC_TAU
self.hc_eps = config.hc_eps
self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations
self.mhc_post_mult_value = config.mhc_post_mult_value
self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters
self.mhc_post_mult_value = _MHC_POST_MULT_VALUE
n = config.mhc_num_residual_streams
n = config.hc_mult
d_model = n * self.hidden_size
mix_hc = (2 + n) * n
@@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
config = vllm_config.model_config.hf_text_config
_validate_supported_config(config)
self.config = config
self.vocab_size = config.vocab_size
@@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module):
(".in_proj_qkvbfg_a", ".f_a_proj", 4),
(".in_proj_qkvbfg_a", ".g_a_proj", 5),
]
if self.config.is_moe:
if _is_moe(self.config):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
# EPLB: the mapping enumerates physical experts, so it must cover
@@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module):
# GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
# ``kv_a_proj_with_mqa``; pad them with zeros for the model shape.
kv_a_pad_size = 0
if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
kv_a_pad_size = self.config.qk_rope_head_dim
_pending_wk_fp8: dict = {}
@@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module):
if (
name.endswith(".bias")
and name not in params_dict
and not self.config.is_linear_attn
and not _is_linear_attn(self.config)
): # noqa: E501
continue
# Remapping the name of FP8 kv-scale.
@@ -939,7 +991,7 @@ class Glm5NextForCausalLM(
super().__init__()
self.model_config = vllm_config.model_config
self.vllm_config = vllm_config
self.config = self.model_config.hf_config
self.config = self.model_config.hf_text_config
quant_config = vllm_config.quant_config
self.quant_config = quant_config
self.model = Glm5NextModel(
@@ -955,7 +1007,7 @@ class Glm5NextForCausalLM(
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(
self.config.vocab_size, scale=self.config.logit_scale
self.config.vocab_size, scale=_LOGIT_SCALE
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -988,7 +1040,7 @@ class Glm5NextForCausalLM(
cls, vllm_config: "VllmConfig"
) -> tuple[tuple[int, int], tuple[int, int, int]]:
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
hf_config = vllm_config.model_config.hf_text_config
tp_size = parallel_config.tensor_parallel_size
num_spec = (
vllm_config.speculative_config.num_speculative_tokens
@@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration(
self.visual = Glm5NextVisionTransformer(
config.text_config,
config.vision_config,
# Read eps from the VISION sub-config, not the top-level
# `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors
# the latter onto text_config (1e-5), silently ignoring the
# vision tower's own (1e-6) rms_norm_eps.
norm_eps=config.vision_config.rms_norm_eps,
norm_eps=_VISION_RMS_NORM_EPS,
# Vision tower ships BF16 weights in this fp8 checkpoint (no
# weight_scale_inv for visual.*), so it must NOT inherit the
# global fp8 quant_config -- doing so incorrectly quantizes
@@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration(
def get_spec_layer_idx_from_weight_name(
config: Glm5NextConfig, weight_name: str
config: Glm5NextTextConfig, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
vllm/models/glm5next/common/mtp.py
+4 -4
View File
@@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
config = vllm_config.speculative_config.draft_model_config.hf_text_config
self.config = config
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
class Glm5NextMultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
config = vllm_config.model_config.hf_text_config
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = config.num_nextn_predict_layers
self.layers = torch.nn.ModuleDict(
@@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module):
class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.config = vllm_config.model_config.hf_text_config
self.quant_config = vllm_config.quant_config
self.model = Glm5NextMultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
@@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
# GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
# ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model.
kv_a_pad_size = 0
if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
kv_a_pad_size = self.config.qk_rope_head_dim
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
vllm/models/glm5next/common/multimodal.py
+69 -53
View File
@@ -3,13 +3,14 @@
"""GLM-5.3-Flash vision tower and multimodal processor."""
from collections.abc import Mapping
from functools import cached_property, partial
from functools import partial
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from transformers.video_utils import VideoMetadata
from vllm.distributed import (
get_tensor_model_parallel_world_size,
@@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import (
)
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
"""Wires up the vLLM-native processor for the multimodal checkpoint.
_MAX_VIDEO_TOKENS = 30000
The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
and stores its image/video processor configs inline (no standalone
``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
config. We bypass it and build our own ``Glm5NextProcessor``
(``vllm/transformers_utils/processors/glm5next.py``), a port of the
training-side pipeline that no longer imports transformers' GLM processor
classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
inside ``smart_resize``'s spatial factor.
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
"""Token-budget geometry for the multimodal checkpoint.
This checkpoint's ``processor_config.json`` ships the token-budget style
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
inherited Glm4v ``size.longest_edge`` path does not apply.
"""
@cached_property
def _glm5_hf_processor(self):
from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
def get_hf_processor(self, **kwargs: object):
return self._glm5_hf_processor
processor = super().get_hf_processor(**kwargs)
# Cap video inputs to keep encoder profiling from starving the KV cache.
video_processor = processor.video_processor
video_processor.max_image_tokens = min(
video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
)
return processor
@staticmethod
def _alignment_factor(proc) -> int:
"""Spatial factor canvases align to, as ``smart_resize`` takes it."""
return proc.patch_size * proc.merge_size * proc.patch_expand_factor
@classmethod
def _pixels_per_token(cls, proc) -> int:
"""Pixels one vision token covers, matching ``smart_resize``'s own
``temporal_factor * factor ** 2``."""
return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
from vllm.transformers_utils.processors.glm5next import _pixel_budget
return _pixel_budget(
proc.min_image_tokens,
proc.max_image_tokens,
proc.patch_size,
proc.merge_size,
proc.temporal_patch_size,
"""(min_pixels, max_pixels) from the processor's token bounds."""
if proc.min_image_tokens is None or proc.max_image_tokens is None:
raise ValueError(
"min_image_tokens and max_image_tokens must be provided by "
"processor_config.json (or per-call kwargs)."
)
pixels_per_token = self._pixels_per_token(proc)
return (
proc.min_image_tokens * pixels_per_token,
proc.max_image_tokens * pixels_per_token,
)
def _get_image_max_pixels(self) -> int:
@@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
else:
frame_indices = [
int(idx)
for idx in glm_sample_frame_indices(
int(metadata.get("total_num_frames", total_frames)),
video_fps,
float(metadata.get("duration") or 0),
target_fps=video_processor.fps_interval,
max_frame_count=video_processor.max_frame_count_dynamic,
temporal_patch_size=temporal_patch_size,
for idx in video_processor.sample_frames(
VideoMetadata(
total_num_frames=int(
metadata.get("total_num_frames", total_frames)
),
fps=video_fps,
duration=metadata.get("duration"),
)
)
]
num_frames = len(frame_indices)
if not frame_indices or num_frames <= 0:
# ``duration * fps_interval < 1`` selects no frames at all; the
# ``duration * fps < 1`` selects no frames at all; the
# pixel path raises an opaque IndexError on the same input a moment
# later, so say what happened while there is still a request.
raise ValueError(
@@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
The inherited Glm4v path resolves the pixel budget from
``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
checkpoint's ``processor_config.json`` ships the token-budget style
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
the alignment factor carries ``patch_expand_factor`` — resolve both
from the vLLM-native processor so profiling matches runtime geometry.
checkpoint ships token bounds instead, and the alignment factor carries
``patch_expand_factor`` — resolve both from the processor so profiling
matches runtime geometry.
"""
from vllm.transformers_utils.processors.glm5next import smart_resize
from transformers.models.glm5_next.image_processing_glm5_next import (
smart_resize,
)
vision_config = self.get_hf_config().vision_config
patch_size = vision_config.patch_size
@@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
temporal_patch_size = vision_config.temporal_patch_size
image_processor = self.get_hf_processor().image_processor
factor = patch_size * merge_size * image_processor.patch_expand_factor
# Keep the profiling search viable when the caller's budget is below
# one aligned canvas of the requested duration.
max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
factor = self._alignment_factor(image_processor)
temporal_factor = image_processor.temporal_patch_size
# `smart_resize` denominates its bounds in vision tokens. Round down,
# but keep the profiling search viable when the caller's budget is
# below one aligned canvas of the requested duration.
max_image_tokens = max(
max_image_pixels // self._pixels_per_token(image_processor), 1
)
if do_resize:
t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
t = num_frames if num_frames > temporal_factor else temporal_factor
resized_height, resized_width = smart_resize(
t=t,
h=image_height,
w=image_width,
t_factor=temporal_patch_size,
h_factor=factor,
w_factor=factor,
num_frames=t,
height=image_height,
width=image_width,
temporal_factor=temporal_factor,
factor=factor,
min_pixels=1,
max_pixels=max_image_pixels,
max_pixels=max_image_tokens,
)
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
else:
@@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
"""The vLLM-native ``Glm5NextProcessor`` extracts image/video features
"""``Glm5NextProcessor`` extracts image/video features
only and passes the prompt text through unchanged, so prompt expansion
(image token repeat, video frame/timestamp structure) is owned by vLLM's
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
vllm/models/qwen4_exp/amd/indexer_qsa.py
+1 -3
View File
@@ -8,6 +8,7 @@ from typing import cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context
@@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from ..common.qsa_cache import (
QSACompressedKeyCache,
vllm/models/qwen4_exp/amd/model.py
+10 -37
View File
@@ -7,6 +7,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import (
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextAttention,
Qwen3NextMLP,
Qwen3NextSparseMoeBlock,
)
from vllm.model_executor.models.qwen3_vl import (
@@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ple_layer import Qwen4ExpPLELayer
@@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
) -> None:
super().__init__()
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
@@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
)
elif layer_type in ATTENTION_LAYER_TYPES:
use_qsa = (
layer_type == QSA_LAYER_TYPE
or getattr(config, "indexer_n_heads", None) is not None
)
elif layer_type == "qwen_sparse_attention":
use_qsa = getattr(config, "indexer_n_heads", None) is not None
if not use_qsa:
self.self_attn = Qwen3NextAttention(
config,
model_config=model_config,
cache_config=cache_config,
model_config=vllm_config.model_config,
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
@@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid layer_type {layer_type}")
mlp_only_layers = getattr(config, "mlp_only_layers", [])
num_experts = getattr(config, "num_experts", 0) or 0
absolute_layer_id = self.layer_idx + 1
is_moe_layer = self.layer_idx not in mlp_only_layers and (
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
if is_moe_layer:
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
else:
self.mlp = Qwen3NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
@@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in ATTENTION_LAYER_TYPES:
elif self.layer_type == "qwen_sparse_attention":
attn_out = self.self_attn(
hidden_states=block_input,
positions=positions,
@@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module):
self._qsa_layer_ids = frozenset(
layer_idx
for layer_idx, layer_type in enumerate(config.layer_types)
if layer_type == QSA_LAYER_TYPE
or (
layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
if layer_type == "qwen_sparse_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
vllm/models/qwen4_exp/amd/mtp.py
+2 -4
View File
@@ -18,6 +18,7 @@ from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
@@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="full_attention",
layer_type="qwen_sparse_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
vllm/models/qwen4_exp/amd/ple_layer.py
+1 -3
View File
@@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence
import torch
import torch.nn.functional as F
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
vllm/models/qwen4_exp/amd/qsa.py
+1 -3
View File
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
vllm/models/qwen4_exp/config.py
-263
View File
@@ -1,263 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp model configuration."""
from typing import Any, ClassVar, cast
from transformers import PreTrainedConfig
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
Qwen3VLVisionConfig,
)
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
_QSA_CONFIG_FIELDS = (
"indexer_n_heads",
"indexer_kv_heads",
"indexer_head_dim",
"indexer_budget",
"indexer_compress_ratio",
)
# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
# Older checkpoints label them `full_attention` and mark QSA by setting the
# indexer fields for the whole model.
QSA_LAYER_TYPE = "qwen_sparse_attention"
ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
model_type = "qwen4_exp"
base_config_key = "vision_config"
class Qwen4ExpTextConfig(Qwen3NextConfig):
model_type = "qwen4_exp_text"
base_config_key = "text_config"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
hc_count: int = 4,
hc_lowrank: int = 320,
ple_layer_ids: list[int] | None = None,
ple_embed_dim: int | None = None,
ple_conv_kernel_size: int = 4,
ngram_size: int = 3,
heads_per_ngram: int = 8,
ngram_vocab_size_base: int = 20_000_000,
make_ngram_vocab_size_divisible_by: int = 128,
output_gate_type: str = "sigmoid",
rope_parameters: dict[str, Any] | None = None,
layer_types: list[str] | None = None,
**kwargs: Any,
) -> None:
if hc_count <= 1:
raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
if rope_parameters is not None:
if kwargs.get("rope_scaling") is None:
kwargs["rope_scaling"] = rope_parameters
if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
kwargs["rope_theta"] = rope_parameters["rope_theta"]
if (
kwargs.get("partial_rotary_factor") is None
and "partial_rotary_factor" in rope_parameters
):
kwargs["partial_rotary_factor"] = rope_parameters[
"partial_rotary_factor"
]
rope_scaling = kwargs.get("rope_scaling")
rope_theta = kwargs.get("rope_theta", 10_000.0)
super().__init__(layer_types=layer_types, **kwargs)
normalized_rope_parameters = self.rope_parameters
self.rope_scaling = (
rope_scaling or rope_parameters or normalized_rope_parameters
)
self.rope_parameters = rope_parameters or normalized_rope_parameters
self.rope_theta = rope_theta
self.hc_count = hc_count
self.hc_lowrank = hc_lowrank
self.ple_layer_ids = ple_layer_ids or []
self.ple_embed_dim = (
self.hidden_size if ple_embed_dim is None else ple_embed_dim
)
self.ple_conv_kernel_size = ple_conv_kernel_size
self.ngram_size = ngram_size
self.heads_per_ngram = heads_per_ngram
self.ngram_vocab_size_base = ngram_vocab_size_base
self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
self.output_gate_type = output_gate_type
self._validate_ple_config()
self._validate_ple_layer_ids()
self._validate_qsa_config()
def _validate_ple_config(self) -> None:
if self.hc_lowrank <= 0:
raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
if self.ngram_size < 2:
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
if self.heads_per_ngram <= 0:
raise ValueError(
f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
)
if self.ple_embed_dim <= 0:
raise ValueError(
f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
)
ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
if self.ple_embed_dim % ngram_heads:
raise ValueError(
"ple_embed_dim must be divisible by total ngram heads: "
f"{self.ple_embed_dim} % {ngram_heads} != 0"
)
if self.ple_conv_kernel_size <= 0:
raise ValueError(
"ple_conv_kernel_size must be positive, got "
f"{self.ple_conv_kernel_size}"
)
if self.ngram_vocab_size_base <= 0:
raise ValueError("ngram_vocab_size_base must be positive")
if self.make_ngram_vocab_size_divisible_by <= 0:
raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
def _validate_ple_layer_ids(self) -> None:
invalid = [
layer_id
for layer_id in self.ple_layer_ids
if not 1 <= int(layer_id) <= self.num_hidden_layers
]
if invalid:
raise ValueError(
"ple_layer_ids are 1-based and must refer to an existing layer; "
f"got {invalid} for {self.num_hidden_layers} layers"
)
def _validate_qsa_config(self) -> None:
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
if all(value is None for value in configured.values()):
if QSA_LAYER_TYPE in (self.layer_types or ()):
raise ValueError(
f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
f"{list(_QSA_CONFIG_FIELDS)}"
)
return
missing = [name for name, value in configured.items() if value is None]
if missing:
raise ValueError(f"QSA config is missing required fields: {missing}")
values = {name: int(cast(int, value)) for name, value in configured.items()}
if any(value <= 0 for value in values.values()):
raise ValueError(f"QSA config values must be positive: {values}")
if values["indexer_kv_heads"] != 1:
raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
raise ValueError(
"indexer_budget must be divisible by indexer_compress_ratio"
)
block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
if block_topk not in (512, 2048):
raise ValueError(
"QSA requires indexer_budget / indexer_compress_ratio "
f"to be 512 or 2048, got {block_topk}"
)
rotary_dim = int(self.head_dim * self.partial_rotary_factor)
if rotary_dim > values["indexer_head_dim"]:
raise ValueError(
"QSA indexer_head_dim must cover the attention rotary "
f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
)
@property
def layers_block_type(self) -> list[str]:
return [
"attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
for layer_type in self.layer_types
]
@property
def short_conv_layer_ids(self) -> list[int]:
if not self.ple_layer_ids:
return []
return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
@property
def short_conv_state_shape(self) -> tuple[int, int] | None:
if not self.short_conv_layer_ids:
return None
ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
ple_channels = self.hidden_size * self.hc_count
return ple_channels, ple_state_len
@property
def ngram_context_len(self) -> int:
if not self.ple_layer_ids:
return 0
return max(int(self.ngram_size) - 1, 0)
class Qwen4ExpConfig(PreTrainedConfig):
model_type = "qwen4_exp"
sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
"vision_config": Qwen4ExpVisionConfig,
"text_config": Qwen4ExpTextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
image_token_id: int = 248056,
video_token_id: int = 248057,
vision_start_token_id: int = 248053,
vision_end_token_id: int = 248054,
tie_word_embeddings: bool = False,
rope_parameters: dict[str, Any] | None = None,
**kwargs: Any,
) -> None:
if text_config is not None:
kwargs.pop("split_ngram_parts", None)
text_kwargs = (
dict(kwargs)
if text_config is None
and "hidden_size" in kwargs
and "num_hidden_layers" in kwargs
else {}
)
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
self.text_config = self.sub_configs["text_config"](**text_kwargs)
else:
self.text_config = text_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
self.rope_parameters = rope_parameters or getattr(
self.text_config, "rope_parameters", {}
)
super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
__all__ = [
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
]
vllm/models/qwen4_exp/nvidia/indexer_qsa.py
+1 -3
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from ..common.qsa_cache import (
QSACompressedKeyCache,
vllm/models/qwen4_exp/nvidia/model.py
+10 -37
View File
@@ -7,6 +7,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
@@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import (
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextAttention,
Qwen3NextMLP,
Qwen3NextSparseMoeBlock,
)
from vllm.model_executor.models.qwen3_vl import (
@@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
@@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
) -> None:
super().__init__()
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
@@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
)
elif layer_type in ATTENTION_LAYER_TYPES:
use_qsa = (
layer_type == QSA_LAYER_TYPE
or getattr(config, "indexer_n_heads", None) is not None
)
elif layer_type == "qwen_sparse_attention":
use_qsa = getattr(config, "indexer_n_heads", None) is not None
if not use_qsa:
self.self_attn = Qwen3NextAttention(
config,
model_config=model_config,
cache_config=cache_config,
model_config=vllm_config.model_config,
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
@@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid layer_type {layer_type}")
mlp_only_layers = getattr(config, "mlp_only_layers", [])
num_experts = getattr(config, "num_experts", 0) or 0
absolute_layer_id = self.layer_idx + 1
is_moe_layer = self.layer_idx not in mlp_only_layers and (
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
if is_moe_layer:
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
else:
self.mlp = Qwen3NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
@@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in ATTENTION_LAYER_TYPES:
elif self.layer_type == "qwen_sparse_attention":
attn_out = self.self_attn(
hidden_states=block_input,
positions=positions,
@@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module):
self._qsa_layer_ids = frozenset(
layer_idx
for layer_idx, layer_type in enumerate(config.layer_types)
if layer_type == QSA_LAYER_TYPE
or (
layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
if layer_type == "qwen_sparse_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
vllm/models/qwen4_exp/nvidia/mtp.py
+2 -4
View File
@@ -18,6 +18,7 @@ from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
@@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="full_attention",
layer_type="qwen_sparse_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
vllm/models/qwen4_exp/nvidia/ngram_embedding.py
+1 -3
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from ..common.ngram_embedding import (
Qwen4ExpPLEDeviceEmbedding,
vllm/models/qwen4_exp/nvidia/ple_layer.py
+1 -3
View File
@@ -6,6 +6,7 @@ from collections.abc import Sequence
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
@@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
vllm/models/qwen4_exp/nvidia/qsa.py
+1 -3
View File
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
@@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import (
kv_cache_dtype_str_to_dtype,
)
vllm/multimodal/video.py
+83 -12
View File
@@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend):
class Glm5NextVideoBackend(VideoBackend):
"""GLM-5.3-Flash fps-interval video backend.
Selects frames with the same ``glm_sample_frame_indices`` sampler the
processor falls back to, so only the sampled frames are
materialized. ``fps_interval`` semantics (default 2.0) with a
temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
pairs kept even. Request overrides: ``fps`` -> fps interval,
Selects frames with the same sampler the processor falls back to, so only
the sampled frames are materialized. ``fps_interval`` semantics (default
2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048,
temporal pairs kept even. Request overrides: ``fps`` -> fps interval,
``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
"""
_SEEK_GAP_THRESHOLD: ClassVar[int] = 64
_DEFAULT_FPS: ClassVar[float] = 2.0
_DEFAULT_MAX_FRAMES: ClassVar[int] = 2048
@classmethod
def _sample_frame_indices(
cls,
total_frames: int,
fps: float,
duration: float,
*,
target_fps: float | None = None,
max_frame_count: int | None = None,
temporal_patch_size: int = 2,
) -> list[int]:
"""GLM video frame sampling (training-reference parity).
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
frame-dense sources it collects more candidates than ``extract_t`` and
the ``> extract_t`` fixup re-spreads the picks uniformly with
``np.linspace`` -- that fallback is the intended reference behavior, not
an accident. Short clips (fewer frames than ``extract_t``) are spread at
evenly spaced timestamps (``floor`` sampling; the linspace variant
samples frames unevenly and cost 4 points on video grounding evals).
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
-> frame cap.
"""
max_frame_idx = total_frames - 1
if not duration:
duration = (round(max_frame_idx / fps) + 1) if fps else 0
if max_frame_count is None:
max_frame_count = cls._DEFAULT_MAX_FRAMES
if target_fps is None:
target_fps = cls._DEFAULT_FPS
extract_t = int(duration * target_fps)
extract_t = min(extract_t, int(max_frame_count))
duration_per_frame = 1 / fps
timestamps = [i * duration_per_frame for i in range(total_frames)]
max_second = int(duration)
if total_frames < extract_t:
frame_indices = [
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
]
else:
frame_indices = []
current_second = 0.0
inv_fps = 1 / (temporal_patch_size * target_fps)
for frame_index in range(total_frames):
if timestamps[frame_index] >= current_second:
current_second += inv_fps
frame_indices.append(frame_index)
if current_second >= max_second:
break
if len(frame_indices) < extract_t:
if len(frame_indices) == 0:
start, end = 0, max(total_frames - 1, 0)
else:
start, end = frame_indices[0], frame_indices[-1]
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
elif len(frame_indices) > extract_t:
frame_indices = np.linspace(
0, total_frames - 1, extract_t, dtype=int
).tolist()
seen, uniq = set(), []
for idx in frame_indices:
if idx not in seen:
seen.add(idx)
uniq.append(int(idx))
if len(uniq) & 1:
uniq.append(uniq[-1])
return uniq
@classmethod
def compute_frames_index_to_sample(
@@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend):
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
# Lazy import: the processor module sits behind the
# transformers_utils package init, which multimodal must not pull in.
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
return glm_sample_frame_indices(
return cls._sample_frame_indices(
source.total_frames_num,
source.original_fps,
source.duration or 0,
vllm/transformers_utils/config.py
-5
View File
@@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
k3_dspark="K3DSparkConfig",
funaudiochat="FunAudioChatConfig",
granite4_vision="Granite4VisionConfig",
glm5_next="Glm5NextConfig",
glm5_next_text="Glm5NextTextConfig",
glm5_next_vision="Glm5NextVisionConfig",
hyperclovax="HyperCLOVAXConfig",
hy_v3="HYV3Config",
hy_v4="HYV4Config",
@@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
qianfan_ocr="QianfanOCRConfig",
qwen3_asr="Qwen3ASRConfig",
qwen3_next="Qwen3NextConfig",
qwen4_exp="Qwen4ExpConfig",
qwen4_exp_text="Qwen4ExpTextConfig",
qwen3_5="Qwen3_5Config",
qwen3_5_text="Qwen3_5TextConfig",
qwen3_5_moe="Qwen3_5MoeConfig",
vllm/transformers_utils/configs/__init__.py
-12
View File
@@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
"Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next",
"Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next",
"Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
@@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
"Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
@@ -140,9 +134,6 @@ __all__ = [
"FunAudioChatConfig",
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
"Glm5NextConfig",
"Glm5NextTextConfig",
"Glm5NextVisionConfig",
"HYV3Config",
"HYV4Config",
"HyperCLOVAXConfig",
@@ -187,9 +178,6 @@ __all__ = [
"QianfanOCRVisionConfig",
"Qwen3ASRConfig",
"Qwen3NextConfig",
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
"Qwen3_5Config",
"Qwen3_5TextConfig",
"Qwen3_5MoeConfig",
vllm/transformers_utils/configs/glm5_next.py
-420
View File
@@ -1,420 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PreTrainedConfig
from vllm.logger import init_logger
logger = init_logger(__name__)
class Glm5NextTextConfig(PreTrainedConfig):
model_type = "glm5_next_text"
base_config_key = "text_config"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
model_type="glm5_next_text",
vocab_size: int = 154880,
hidden_size: int = 4096,
head_dim: int | None = None,
intermediate_size: int = 12288,
num_hidden_layers: int = 45,
num_attention_heads: int = 64,
num_key_value_heads: int | None = None,
hidden_act: str = "silu",
rms_norm_eps: float = 1e-5,
pad_token_id: int | None = 151329,
bos_token_id: int | None = None,
eos_token_id: int | list[int] | None = None,
rope_parameters: dict | None = None,
max_position_embeddings: int = 1048576,
tie_word_embeddings: bool = False,
moe_intermediate_size: int = 2048,
moe_renormalize: bool = True,
scoring_func: str = "sigmoid",
n_routed_experts: int | None = 288,
num_experts_per_token: int = 7,
n_shared_experts: int = 1,
routed_scaling_factor: float = 2.5,
topk_method: str | None = None,
first_k_dense_replace: int = 0,
moe_layer_freq: int = 1,
use_grouped_topk: bool = True,
n_group: int = 1,
topk_group: int = 1,
mla: bool = True,
q_lora_rank: int | None = 1536,
kv_lora_rank: int | None = 512,
qk_nope_head_dim: int = 256,
qk_rope_head_dim: int = 0,
v_head_dim: int | None = 256,
mla_nope: bool | None = True,
num_nextn_predict_layers: int = 1,
# Per-layer layout: "linear_attention" | "deepseek_sparse_attention"
layer_types: list[str] | None = None,
# Per-layer MLP: "dense" | "sparse"
mlp_layer_types: list[str] | None = None,
# Linear-attention (KDA) head config (flattened from the old
# linear_attn_config dict).
linear_head_dim: int = 128,
linear_num_heads: int = 64,
linear_conv_kernel_dim: int = 4,
linear_lower_bound: float = -5.0,
index_head_dim: int | None = None,
index_topk: int | None = None,
index_n_heads: int | None = None,
index_dsa_use_layernorm: bool = True,
index_kpool_compress: bool = True,
# Every ``index_kpool`` indexer K entries pool into one stored entry
# (compress_ratio). The 300B checkpoint ships 4; topk runs at pool
# granularity (select_k = index_topk // index_kpool).
index_kpool: int | None = 4,
index_kpool_always_select_tail: bool = True,
indexer_rope_interleave: bool = False,
mhc: bool | None = True,
mhc_num_residual_streams: int = 4,
hc_eps: float | None = 1e-06,
mhc_tau: float = 0.05,
hres_vwnstyle: bool | None = True,
mhc_no_norm_weight: bool | None = False,
mhc_sinkhorn_iterations: int | None = 20,
mhc_post_mult_value: float | None = 2.0,
swiglu_limit: float | None = None,
logit_scale: float = 1.0,
**kwargs,
):
# Preserve checkpoint field names and local aliases because their
# consumers use different spellings.
num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token)
moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize)
mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams)
mhc_sinkhorn_iterations = kwargs.get(
"hc_sinkhorn_iters", mhc_sinkhorn_iterations
)
# Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this
# alias self.mla_nope silently stays at the param default.
mla_nope = kwargs.get("mla_use_nope", mla_nope)
# Checkpoints ship the KDA head config as the ``linear_attn_config``
# dict (head_dim / num_heads / short_conv_kernel_size /
# gate_lower_bound) rather than the flattened top-level fields; fold it
# in so the trained values are read instead of the param defaults
# (which only match this checkpoint by coincidence).
linear_cfg = kwargs.get("linear_attn_config") or {}
if linear_cfg:
linear_head_dim = linear_cfg.get("head_dim", linear_head_dim)
linear_num_heads = linear_cfg.get("num_heads", linear_num_heads)
linear_conv_kernel_dim = linear_cfg.get(
"short_conv_kernel_size", linear_conv_kernel_dim
)
linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound)
if index_topk is not None:
if index_dsa_use_layernorm is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True"
)
if index_kpool_compress is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires index_kpool_compress=True"
)
if index_kpool_always_select_tail is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires "
"index_kpool_always_select_tail=True"
)
if mhc:
if hres_vwnstyle is not True:
raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True")
if mhc_no_norm_weight not in (False, None):
raise NotImplementedError(
"GLM-5.3 mHC requires mhc_no_norm_weight=False"
)
self.model_type = model_type
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.head_dim = (
head_dim if head_dim is not None else hidden_size // num_attention_heads
)
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.rms_norm_eps = rms_norm_eps
self.max_position_embeddings = max_position_embeddings
self.rope_parameters = rope_parameters
# mla config
self.mla = mla
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.mla_nope = mla_nope
# moe config
self.n_routed_experts = n_routed_experts
self.num_experts_per_token = num_experts_per_token
self.moe_renormalize = moe_renormalize
self.n_shared_experts = n_shared_experts
self.routed_scaling_factor = routed_scaling_factor
self.topk_method = topk_method
self.scoring_func = scoring_func
assert self.scoring_func in ("softmax", "sigmoid")
self.moe_intermediate_size = moe_intermediate_size
self.first_k_dense_replace = first_k_dense_replace
self.moe_layer_freq = moe_layer_freq
self.use_grouped_topk = use_grouped_topk
self.n_group = n_group
self.topk_group = topk_group
self.num_nextn_predict_layers = num_nextn_predict_layers
# Per-layer attention / MLP layout. Normalize mlp_layer_types from
# first_k_dense_replace when the new-schema field is absent so layer
# construction sees a consistent layout (mirrors cohere2_moe).
self.layer_types = layer_types
if mlp_layer_types is None:
n = self.num_hidden_layers
if first_k_dense_replace is not None:
mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * (
n - first_k_dense_replace
)
else:
mlp_layer_types = ["sparse"] * n
self.mlp_layer_types = mlp_layer_types
# Linear-attention (KDA) head config.
self.linear_head_dim = linear_head_dim
self.linear_num_heads = linear_num_heads
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_lower_bound = linear_lower_bound
# dsa index config
self.index_head_dim = index_head_dim
self.index_topk = index_topk
self.index_n_heads = index_n_heads
self.index_dsa_use_layernorm = index_dsa_use_layernorm
self.index_kpool_compress = index_kpool_compress
self.index_kpool = index_kpool
self.index_kpool_always_select_tail = index_kpool_always_select_tail
self.indexer_rope_interleave = indexer_rope_interleave
# mhc config
self.mhc = mhc
self.mhc_num_residual_streams = mhc_num_residual_streams
self.mhc_tau = mhc_tau
self.hres_vwnstyle = hres_vwnstyle
self.hc_eps = hc_eps
self.mhc_no_norm_weight = mhc_no_norm_weight
self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
self.mhc_post_mult_value = mhc_post_mult_value
self.swiglu_limit = swiglu_limit
self.logit_scale = logit_scale
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@property
def is_mla(self):
return (
self.q_lora_rank is not None
or self.kv_lora_rank is not None
or self.qk_nope_head_dim is not None
or self.qk_rope_head_dim is not None
or self.v_head_dim is not None
or self.mla_nope is True
)
@property
def is_moe(self):
return self.n_routed_experts is not None
@property
def is_linear_attn(self) -> bool:
return self.layer_types is not None and any(
t == "linear_attention" for t in self.layer_types
)
def is_kda_layer(self, layer_idx: int):
return (
self.layer_types is not None
and layer_idx < len(self.layer_types)
and self.layer_types[layer_idx] == "linear_attention"
)
@property
def layers_block_type(self):
# Map the schema's per-layer types onto the block strings vLLM's hybrid
# accounting (get_num_layers_by_block_type) recognizes: linear-attention
# layers stay "linear_attention"; every other attention variant collapses
# to "attention".
if self.layer_types is None:
return ["attention"] * self.num_hidden_layers
return [
"linear_attention" if t == "linear_attention" else "attention"
for t in self.layer_types
]
class Glm5NextVisionConfig(PreTrainedConfig):
model_type = "glm5_next_vision"
base_config_key = "vision_config"
def __init__(
self,
depth: int = 24,
hidden_size: int = 1024,
hidden_act: str = "silu",
image_size: int = 448,
intermediate_size: int = 4096,
num_heads: int = 16,
out_hidden_size: int = 4096,
projection_intermediate_size: int = 10240,
in_channels: int = 3,
initializer_range: float = 0.02,
patch_size: int = 14,
rms_norm_eps: float = 1e-5,
spatial_merge_size: int = 2,
temporal_patch_size: int = 2,
attention_dropout: float = 0.0,
attention_bias: bool = True,
swiglu_limit: float | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.depth = depth
self.hidden_size = hidden_size
self.hidden_act = hidden_act
self.image_size = image_size
self.intermediate_size = intermediate_size
self.num_heads = num_heads
self.out_hidden_size = out_hidden_size
# GLM-5.3-Flash merger bottleneck width (absent from the generic
# GLM-OCR vision config); the tower uses it as the PatchMerger
# context_dim instead of text_config.intermediate_size.
self.projection_intermediate_size = projection_intermediate_size
self.in_channels = in_channels
self.initializer_range = initializer_range
self.patch_size = patch_size
# GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5,
# but the vision tower was trained with 1e-6. Serving with 1e-5 drifts
# the RMSNorm and produces repetitive/degraded image descriptions, so
# force the trained value regardless of the checkpoint field.
self.rms_norm_eps = 1e-6
self.spatial_merge_size = spatial_merge_size
self.temporal_patch_size = temporal_patch_size
self.attention_dropout = attention_dropout
self.attention_bias = attention_bias
self.swiglu_limit = swiglu_limit
class Glm5NextConfig(PreTrainedConfig):
model_type = "glm5_next"
sub_configs = {
"vision_config": Glm5NextVisionConfig,
"text_config": Glm5NextTextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config=None,
vision_config=None,
image_token_id: int = 154854,
video_token_id: int = 154855,
image_start_token_id: int = 154830,
image_end_token_id: int = 154831,
video_start_token_id: int = 154832,
video_end_token_id: int = 154833,
**kwargs,
):
# Init super() first so base-class defaults don't clobber text-config
# values set below (PreTrainedConfig has many text-related defaults
# that differ from Glm5NextTextConfig).
super().__init__(**kwargs)
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
# Backward compatibility: a flat top-level checkpoint (no nested
# text_config) folds its text fields into Glm5NextTextConfig.
self.text_config = self.sub_configs["text_config"](**kwargs)
else:
self.text_config = text_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.image_start_token_id = image_start_token_id
self.image_end_token_id = image_end_token_id
self.video_start_token_id = video_start_token_id
self.video_end_token_id = video_end_token_id
# Mirror attention implementation recursively onto sub-configs.
self._attn_implementation = kwargs.pop("attn_implementation", None)
# Config-metadata fields that belong to the top-level (multimodal) config
# and must NOT be mirrored onto text_config: ``architectures`` /
# ``torch_dtype`` differ between the top-level config and the text
# sub-config, and mirroring them makes the top-level ``architectures``
# silently read back as None (PreTrainedConfig initializes both to None),
# which then fails model-class resolution ("No model architectures are
# specified").
_UNMIRRORED_KEYS = [
"_name_or_path",
"model_type",
"dtype",
"torch_dtype",
"architectures",
"_attn_implementation_internal",
]
def __setattr__(self, key, value):
unmirrored = type(self)._UNMIRRORED_KEYS
if (
(text_config := super().__getattribute__("__dict__").get("text_config"))
is not None
and key not in unmirrored
and key in text_config.__dict__
):
setattr(text_config, key, value)
else:
super().__setattr__(key, value)
def __getattribute__(self, key):
unmirrored = type(self)._UNMIRRORED_KEYS
if (
"text_config" in super().__getattribute__("__dict__")
and key not in unmirrored
):
text_config = super().__getattribute__("text_config")
# Forward both instance attributes AND class-defined properties/
# methods of the text config, so a flat text-only checkpoint
# (model_type "glm5_next", no nested text_config) sees is_moe /
# is_kda_layer / layers_block_type like a Glm5NextTextConfig.
if key in text_config.__dict__ or key in type(text_config).__dict__:
return getattr(text_config, key)
return super().__getattribute__(key)
vllm/transformers_utils/configs/qwen4_exp.py
-15
View File
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Exports for the package-owned Qwen4Exp configs."""
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
Qwen4ExpVisionConfig,
)
__all__ = [
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
]
vllm/transformers_utils/processors/__init__.py
-2
View File
@@ -17,7 +17,6 @@ __all__ = [
"FireRedASR2Processor",
"FunASRProcessor",
"GLM4VProcessor",
"Glm5NextProcessor",
"Granite4VisionProcessor",
"H2OVLProcessor",
"Moondream3Processor",
@@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
vllm/transformers_utils/processors/glm5next.py
-953
View File
@@ -1,953 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-native multimodal processor for GLM-5.3-Flash."""
import math
import numpy as np
import torch
from torchvision.transforms.v2 import functional as tvF
from transformers.image_processing_backends import (
TorchvisionBackend,
group_images_by_shape,
reorder_images,
)
from transformers.image_processing_utils import BatchFeature
from transformers.image_utils import (
OPENAI_CLIP_MEAN,
OPENAI_CLIP_STD,
ChannelDimension,
ImageInput,
PILImageResampling,
SizeDict,
get_image_size,
)
from transformers.models.auto.image_processing_auto import get_image_processor_config
from transformers.processing_utils import (
ImagesKwargs,
MultiModalData,
ProcessingKwargs,
ProcessorMixin,
Unpack,
VideosKwargs,
)
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.utils import TensorType, logging
from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import (
VideoInput,
VideoMetadata,
group_videos_by_shape,
reorder_videos,
)
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
logger = logging.get_logger(__name__)
# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
# starving the KV cache. Image inputs retain their checkpoint-defined budget.
_MAX_VIDEO_TOKENS = 30000
# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
# max_frame_count_dynamic=2048); used when neither the request nor the
# processor config overrides them.
GLM_VIDEO_DEFAULT_FPS = 2.0
GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
def glm_sample_frame_indices(
total_frames: int,
fps: float,
duration: float,
*,
target_fps: float | None = None,
max_frame_count: int | None = None,
temporal_patch_size: int = 2,
) -> list[int]:
"""GLM video frame sampling (training-reference parity).
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
frame-dense sources it collects more candidates than ``extract_t`` and
the ``> extract_t`` fixup re-spreads the picks uniformly with
``np.linspace`` -- that fallback is the intended reference behavior, not
an accident. Short clips (fewer frames than ``extract_t``) are spread at
evenly spaced timestamps (``floor`` sampling; the linspace variant
samples frames unevenly and cost 4 points on video grounding evals).
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
-> frame cap.
"""
max_frame_idx = total_frames - 1
if not duration:
duration = (round(max_frame_idx / fps) + 1) if fps else 0
if max_frame_count is None:
max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
if target_fps is None:
target_fps = GLM_VIDEO_DEFAULT_FPS
extract_t = int(duration * target_fps)
extract_t = min(extract_t, int(max_frame_count))
duration_per_frame = 1 / fps
timestamps = [i * duration_per_frame for i in range(total_frames)]
max_second = int(duration)
if total_frames < extract_t:
frame_indices = [
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
]
else:
frame_indices = []
current_second = 0.0
inv_fps = 1 / (temporal_patch_size * target_fps)
for frame_index in range(total_frames):
if timestamps[frame_index] >= current_second:
current_second += inv_fps
frame_indices.append(frame_index)
if current_second >= max_second:
break
if len(frame_indices) < extract_t:
if len(frame_indices) == 0:
start, end = 0, max(total_frames - 1, 0)
else:
start, end = frame_indices[0], frame_indices[-1]
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
elif len(frame_indices) > extract_t:
frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
seen, uniq = set(), []
for idx in frame_indices:
if idx not in seen:
seen.add(idx)
uniq.append(int(idx))
if len(uniq) & 1:
uniq.append(uniq[-1])
return uniq
def _ceil_to_factor(value: int, factor: int) -> int:
"""Round a positive integer upward to the nearest multiple of factor."""
return math.ceil(value / factor) * factor
def _fit_aligned_size_within_budget(
t: int,
h: int,
w: int,
h_factor: int,
w_factor: int,
max_pixels: int,
) -> tuple[int, int]:
"""Largest proportional size whose upward-aligned canvas fits the budget.
Binary search on the unaligned content height; each candidate is rounded
upward to h_factor/w_factor, so the returned canvas always satisfies
``t * aligned_h * aligned_w <= max_pixels``.
"""
minimum_pixels = t * h_factor * w_factor
if max_pixels < minimum_pixels:
raise ValueError(
f"max_pixels={max_pixels} is too small. At least "
f"{minimum_pixels} pixels are required for one aligned patch."
)
low, high = 1, h
best_h, best_w = h_factor, w_factor
while low <= high:
content_h = (low + high) // 2
content_w = max(1, math.floor(w * content_h / h))
aligned_h = _ceil_to_factor(content_h, h_factor)
aligned_w = _ceil_to_factor(content_w, w_factor)
if t * aligned_h * aligned_w <= max_pixels:
best_h, best_w = aligned_h, aligned_w
low = content_h + 1
else:
high = content_h - 1
return best_h, best_w
def smart_resize(
t: int,
h: int,
w: int,
t_factor: int = 1,
h_factor: int = 28,
w_factor: int = 28,
min_pixels: int = 56 * 56,
max_pixels: int = 14 * 14 * 4 * 1280,
) -> tuple[int, int]:
"""GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
``t_bar * h_bar * w_bar`` pixel budget.
Height/width always round UP to their factors (content is then padded,
never cropped or distorted); an over-budget canvas is refit by binary
search instead of one-shot square-root scaling. ``h_factor`` /
``w_factor`` carry ``patch_expand_factor`` on top of
``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
temporal_patch_size``.
"""
if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
raise ValueError("Image dimensions and alignment factors must be positive.")
if min_pixels <= 0 or max_pixels <= 0:
raise ValueError("min_pixels and max_pixels must be positive.")
if min_pixels > max_pixels:
raise ValueError("min_pixels must be less than or equal to max_pixels.")
t_bar = max(t_factor, round(t / t_factor) * t_factor)
h_bar = _ceil_to_factor(h, h_factor)
w_bar = _ceil_to_factor(w, w_factor)
if t_bar * h_bar * w_bar > max_pixels:
h_bar, w_bar = _fit_aligned_size_within_budget(
t=t_bar,
h=h,
w=w,
h_factor=h_factor,
w_factor=w_factor,
max_pixels=max_pixels,
)
elif t_bar * h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (t * h * w))
h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
# Alignment can push a candidate slightly over a tight max_pixels
# budget. Refit it when that happens.
if t_bar * h_bar * w_bar > max_pixels:
h_bar, w_bar = _fit_aligned_size_within_budget(
t=t_bar,
h=h,
w=w,
h_factor=h_factor,
w_factor=w_factor,
max_pixels=max_pixels,
)
return h_bar, w_bar
def _get_pad_content_size(
image_height: int,
image_width: int,
canvas_height: int,
canvas_width: int,
allow_upscale: bool = False,
) -> tuple[int, int]:
"""Aspect-ratio-preserving content size that fits the canvas.
Oversized images are shrunk proportionally. Small images are enlarged
only when ``allow_upscale``. Padding is applied after the resize.
"""
scale = min(canvas_height / image_height, canvas_width / image_width)
if not allow_upscale:
scale = min(1.0, scale)
content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
return content_height, content_width
def _resize_or_pad(
stacked_images: torch.Tensor,
target_height: int,
target_width: int,
resize_mode: str,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
resize,
allow_upscale: bool = False,
) -> torch.Tensor:
"""Resize onto the aligned canvas, or keep the aspect ratio and
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
height, width = stacked_images.shape[-2:]
if resize_mode == "resize":
return resize(
stacked_images,
size=SizeDict(height=target_height, width=target_width),
resample=resample,
)
if resize_mode != "pad":
raise ValueError("resize_mode must be either 'resize' or 'pad'.")
content_height, content_width = _get_pad_content_size(
image_height=height,
image_width=width,
canvas_height=target_height,
canvas_width=target_width,
allow_upscale=allow_upscale,
)
if (content_height, content_width) != (height, width):
stacked_images = resize(
stacked_images,
size=SizeDict(height=content_height, width=content_width),
resample=resample,
)
# torchvision padding order: [left, top, right, bottom] -> pad only the
# right and bottom sides.
return tvF.pad(
stacked_images,
padding=[0, 0, target_width - content_width, target_height - content_height],
fill=0,
)
def _pixel_budget(
min_image_tokens: int | None,
max_image_tokens: int | None,
patch_size: int,
merge_size: int,
temporal_patch_size: int,
) -> tuple[int, int]:
"""(min_pixels, max_pixels) from the token bounds of
``processor_config.json``; one vision token covers
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
if min_image_tokens is None or max_image_tokens is None:
raise ValueError(
"min_image_tokens and max_image_tokens must be provided by "
"processor_config.json (or per-call kwargs)."
)
factor = temporal_patch_size * (patch_size * merge_size) ** 2
return min_image_tokens * factor, max_image_tokens * factor
class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
patch_size: int | None
temporal_patch_size: int | None
merge_size: int | None
patch_expand_factor: int | None
resize_mode: str | None
min_image_tokens: int | None
max_image_tokens: int | None
class Glm5NextImageProcessor(TorchvisionBackend):
"""Fast torchvision image processor for GLM-5.3-Flash.
``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
``"pad"`` (default) preserves the aspect ratio and zero-pads the
right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
it. Defaults mirror the checkpoint's ``image_processor`` config.
"""
do_resize = True
resample = PILImageResampling.BICUBIC
size = {"longest_edge": 1} # unused: budgets come from the token bounds
do_rescale = True
do_normalize = True
image_mean = OPENAI_CLIP_MEAN
image_std = OPENAI_CLIP_STD
do_convert_rgb = True
patch_size = 14
temporal_patch_size = 2
merge_size = 2
patch_expand_factor = 1
resize_mode = "pad"
min_image_tokens = 16
max_image_tokens = 8000
valid_kwargs = Glm5NextImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
def _preprocess(
self,
images: list[torch.Tensor],
do_resize: bool,
size: SizeDict,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: float | list[float] | None,
image_std: float | list[float] | None,
patch_size: int,
temporal_patch_size: int,
merge_size: int,
patch_expand_factor: int,
resize_mode: str | None,
min_image_tokens: int | None,
max_image_tokens: int | None,
disable_grouping: bool | None,
return_tensors: str | TensorType | None,
**kwargs,
) -> BatchFeature:
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
min_pixels, max_pixels = _pixel_budget(
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
patch_size,
merge_size,
temporal_patch_size,
)
grouped_images, grouped_images_index = group_images_by_shape(
images, disable_grouping=disable_grouping
)
resized_images_grouped = {}
for shape, stacked_images in grouped_images.items():
height, width = stacked_images.shape[-2:]
if do_resize:
resized_height, resized_width = smart_resize(
t=temporal_patch_size,
h=height,
w=width,
t_factor=temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
stacked_images = _resize_or_pad(
stacked_images,
target_height=resized_height,
target_width=resized_width,
resize_mode=resize_mode,
resample=resample,
resize=self.resize,
allow_upscale=(temporal_patch_size * height * width < min_pixels),
)
resized_images_grouped[shape] = stacked_images
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
grouped_images, grouped_images_index = group_images_by_shape(
resized_images, disable_grouping=disable_grouping
)
processed_images_grouped = {}
processed_grids = {}
for shape, stacked_images in grouped_images.items():
resized_height, resized_width = stacked_images.shape[-2:]
patches = self.rescale_and_normalize(
stacked_images,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
if patches.ndim == 4: # (B, C, H, W)
patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
if patches.shape[1] % temporal_patch_size != 0:
repeats = patches[:, -1:].repeat(
1,
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
1,
1,
1,
)
patches = torch.cat([patches, repeats], dim=1)
batch_size, t_len, channel = patches.shape[:3]
grid_t = t_len // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
# (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
processed_images_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_images = reorder_images(
processed_images_grouped, grouped_images_index
)
processed_grids = reorder_images(processed_grids, grouped_images_index)
pixel_values = torch.cat(processed_images, dim=0)
image_grid_thw = torch.tensor(processed_grids)
return BatchFeature(
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
tensor_type=return_tensors,
)
def preprocess(
self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
) -> BatchFeature:
return super().preprocess(images, **kwargs)
def get_number_of_image_patches(
self, height: int, width: int, images_kwargs: dict | None = None
) -> int:
"""Number of image patches (pre-merge) for a given (height, width)."""
images_kwargs = images_kwargs or {}
patch_size = images_kwargs.get("patch_size", self.patch_size)
merge_size = images_kwargs.get("merge_size", self.merge_size)
patch_expand_factor = images_kwargs.get(
"patch_expand_factor", self.patch_expand_factor
)
min_pixels, max_pixels = _pixel_budget(
images_kwargs.get("min_image_tokens", self.min_image_tokens),
images_kwargs.get("max_image_tokens", self.max_image_tokens),
patch_size,
merge_size,
self.temporal_patch_size,
)
resized_height, resized_width = smart_resize(
t=self.temporal_patch_size,
h=height,
w=width,
t_factor=self.temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
return grid_h * grid_w
class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
fps: list[float] | float
patch_size: int
temporal_patch_size: int
merge_size: int
patch_expand_factor: int
resize_mode: str | None
target_fps: float | None
max_frames: int | None
fps_interval: int | None
max_frame_count_dynamic: int | None
min_image_tokens: int | None
max_image_tokens: int | None
class Glm5NextVideoProcessor(BaseVideoProcessor):
"""Fast video processor for GLM-5.3-Flash.
Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
image processor, and adds GLM-5.3-Flash frame sampling
(``glm_sample_frame_indices``: ``fps_interval`` semantics with a
temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
``video_processor`` config.
"""
resample = PILImageResampling.BICUBIC
size = {"longest_edge": 1} # unused: budgets come from the token bounds
image_mean = OPENAI_CLIP_MEAN
image_std = OPENAI_CLIP_STD
do_resize = True
do_rescale = True
do_normalize = True
do_convert_rgb = True
do_sample_frames = True
patch_size = 14
temporal_patch_size = 2
patch_expand_factor = 1
merge_size = 2
valid_kwargs = Glm5NextVideoProcessorKwargs
num_frames = 16
fps = 2
fps_interval = 2.0
max_frame_count_dynamic = 2048
resize_mode = "pad"
min_image_tokens = 16
max_image_tokens = 240000
model_input_names = ["pixel_values_videos", "video_grid_thw"]
def sample_frames(
self,
metadata: VideoMetadata,
fps: int | float | None = None,
**kwargs,
) -> np.ndarray:
"""Sample frame indices with GLM's fps-interval policy.
``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
``max_frame_count_dynamic`` are the overrides described in
:func:`glm_sample_frame_indices`.
"""
if metadata is None or getattr(metadata, "fps", None) is None:
raise ValueError(
"Asked to sample frames per second but no video metadata was "
"provided which is required when sampling in GLM-5.3-Flash. Please "
"pass in `VideoMetadata` object or set `do_sample_frames=False`."
)
target_fps = fps if fps is not None else kwargs.get("target_fps")
if target_fps is None:
target_fps = self.fps_interval
indices = glm_sample_frame_indices(
metadata.total_num_frames,
metadata.fps,
metadata.duration or 0,
target_fps=target_fps,
max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
temporal_patch_size=self.temporal_patch_size,
)
return np.array(indices)
def _preprocess(
self,
videos: list[torch.Tensor],
do_convert_rgb: bool = True,
do_resize: bool = True,
size: SizeDict | None = None,
resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
do_rescale: bool = True,
rescale_factor: float = 1 / 255.0,
do_normalize: bool = True,
image_mean: float | list[float] | None = None,
image_std: float | list[float] | None = None,
patch_size: int | None = None,
temporal_patch_size: int | None = None,
patch_expand_factor: int | None = None,
merge_size: int | None = None,
resize_mode: str | None = None,
min_image_tokens: int | None = None,
max_image_tokens: int | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
patch_expand_factor = self.patch_expand_factor
patch_size = patch_size if patch_size is not None else self.patch_size
temporal_patch_size = (
temporal_patch_size
if temporal_patch_size is not None
else self.temporal_patch_size
)
merge_size = merge_size if merge_size is not None else self.merge_size
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
min_pixels, max_pixels = _pixel_budget(
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
patch_size,
merge_size,
temporal_patch_size,
)
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
resized_videos_grouped = {}
for shape, stacked_videos in grouped_videos.items():
if do_convert_rgb:
stacked_videos = self.convert_to_rgb(stacked_videos)
b, t_len, c, h, w = stacked_videos.shape
num_frames, height, width = t_len, h, w
if do_resize:
resized_height, resized_width = smart_resize(
t=num_frames,
h=height,
w=width,
t_factor=temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
stacked_videos = stacked_videos.view(b * t_len, c, h, w)
stacked_videos = _resize_or_pad(
stacked_videos,
target_height=resized_height,
target_width=resized_width,
resize_mode=resize_mode,
resample=resample,
resize=self.resize,
allow_upscale=(num_frames * height * width < min_pixels),
)
stacked_videos = stacked_videos.view(
b, t_len, c, resized_height, resized_width
)
resized_videos_grouped[shape] = stacked_videos
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
processed_videos_grouped = {}
processed_grids = {}
for shape, stacked_videos in grouped_videos.items():
resized_height, resized_width = get_image_size(
stacked_videos[0], channel_dim=ChannelDimension.FIRST
)
stacked_videos = self.rescale_and_normalize(
stacked_videos,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
patches = stacked_videos
if pad := -patches.shape[1] % temporal_patch_size:
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
patches = torch.cat((patches, repeats), dim=1)
batch_size, grid_t, channel = patches.shape[:3]
grid_t = grid_t // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
processed_videos_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_videos = reorder_videos(
processed_videos_grouped, grouped_videos_index
)
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
pixel_values_videos = torch.cat(processed_videos, dim=0)
video_grid_thw = torch.tensor(processed_grids)
return BatchFeature(
data={
"pixel_values_videos": pixel_values_videos,
"video_grid_thw": video_grid_thw,
},
tensor_type=return_tensors,
)
class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
images_kwargs: Glm5NextImageProcessorKwargs
videos_kwargs: Glm5NextVideoProcessorKwargs
_defaults = {
"text_kwargs": {
"padding": False,
"return_token_type_ids": False,
"return_mm_token_type_ids": False,
},
"videos_kwargs": {"return_metadata": True},
}
class Glm5NextProcessor(ProcessorMixin):
"""Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
structure (mrope timestamps).
"""
attributes = ["image_processor", "tokenizer", "video_processor"]
image_processor_class = "AutoImageProcessor"
video_processor_class = "AutoVideoProcessor"
tokenizer_class = ("PythonBackend", "TokenizersBackend")
def __init__(
self,
image_processor=None,
tokenizer=None,
video_processor=None,
chat_template=None,
**kwargs,
) -> None:
super().__init__(
image_processor, tokenizer, video_processor, chat_template=chat_template
)
self.image_token = (
"<|image|>"
if not hasattr(tokenizer, "image_token")
else tokenizer.image_token
)
self.video_token = (
"<|video|>"
if not hasattr(tokenizer, "video_token")
else tokenizer.video_token
)
self.image_token_id = (
tokenizer.image_token_id
if getattr(tokenizer, "image_token_id", None)
else tokenizer.convert_tokens_to_ids(self.image_token)
)
self.video_token_id = (
tokenizer.video_token_id
if getattr(tokenizer, "video_token_id", None)
else tokenizer.convert_tokens_to_ids(self.video_token)
)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
"""Build the processor directly from the checkpoint config.
GLM-5.3-Flash stores nested image/video configs in
``processor_config.json`` and declares a custom processor class. This
method reads those configs directly and caps only the video token budget.
"""
from transformers import AutoTokenizer
model_path = pretrained_model_name_or_path
tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
# Video keeps a serving token cap (the checkpoint's 240k-token
# budget would starve the KV cache at startup profiling); images
# follow the checkpoint budget verbatim so preprocessing matches
# the HF reference exactly.
if is_video and cfg.get("max_image_tokens") is not None:
cfg["max_image_tokens"] = min(
cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
)
return cfg
ip_cfg = _cap_cfg(
dict(get_image_processor_config(model_path, **kwargs)), is_video=False
)
image_processor = Glm5NextImageProcessor(
**{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
)
processor_config = get_hf_file_to_dict(
"processor_config.json",
model_path,
revision=kwargs.get("revision", "main"),
)
if processor_config is None:
raise ValueError(f"Missing processor_config.json for {model_path}")
vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
video_processor = Glm5NextVideoProcessor(
**{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
)
return cls(
image_processor=image_processor,
tokenizer=tokenizer,
video_processor=video_processor,
)
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
videos: VideoInput | None = None,
**kwargs: Unpack[Glm5NextProcessorKwargs],
) -> BatchFeature:
output_kwargs = self._merge_kwargs(
Glm5NextProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
image_inputs = self.image_processor(
images=images, **output_kwargs["images_kwargs"]
)
else:
image_inputs = {}
if videos is not None:
videos_inputs = self.video_processor(
videos=videos, **output_kwargs["videos_kwargs"]
)
if "return_metadata" not in kwargs:
videos_inputs.pop("video_metadata")
else:
videos_inputs = {}
if not isinstance(text, list):
text = [text]
# Prompt updates expand the unchanged image/video markers after this call.
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
"return_mm_token_type_ids", False
)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
if return_mm_token_type_ids:
array_ids = np.array(text_inputs["input_ids"])
mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
mm_token_type_ids[array_ids == self.image_token_id] = 1
text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
return BatchFeature(
data={**text_inputs, **image_inputs, **videos_inputs},
tensor_type=return_tensors,
)
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
vision_data = {}
if image_sizes is not None:
images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
images_kwargs.update(kwargs)
merge_size = (
images_kwargs.get("merge_size", None) or self.image_processor.merge_size
)
num_image_patches = [
self.image_processor.get_number_of_image_patches(
*image_size, images_kwargs
)
for image_size in image_sizes
]
num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
vision_data.update(
{
"num_image_tokens": num_image_tokens,
"num_image_patches": num_image_patches,
}
)
if video_sizes is not None:
videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
videos_kwargs.update(kwargs)
num_video_patches = [
self.video_processor.get_number_of_video_patches(
*video_size, videos_kwargs
)
for video_size in video_sizes
]
num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
vision_data["num_video_tokens"] = num_video_tokens
return MultiModalData(**vision_data)
def post_process_image_text_to_text(
self,
generated_outputs,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
**kwargs,
):
return self.tokenizer.batch_decode(
generated_outputs,
skip_special_tokens=skip_special_tokens,
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
**kwargs,
)
__all__ = [
"Glm5NextImageProcessor",
"Glm5NextVideoProcessor",
"Glm5NextProcessor",
"smart_resize",
]