[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -15,11 +15,11 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.layers import linear
|
||||
from vllm.models.glm5next.common import model as glm_model
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
|
||||
|
||||
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
|
||||
|
||||
|
||||
@@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU.
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.models.glm5_next.video_processing_glm5_next import (
|
||||
Glm5NextVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
Glm5NextVideoProcessor,
|
||||
_pixel_budget,
|
||||
glm_sample_frame_indices,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -48,27 +47,16 @@ def _pixel_path_grid(
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
video_processor.min_image_tokens,
|
||||
video_processor.max_image_tokens,
|
||||
video_processor.patch_size,
|
||||
video_processor.merge_size,
|
||||
video_processor.temporal_patch_size,
|
||||
)
|
||||
factor = (
|
||||
video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor
|
||||
)
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=num_frames,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=video_processor.temporal_patch_size,
|
||||
h_factor=factor,
|
||||
w_factor=factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
temporal_factor=video_processor.temporal_patch_size,
|
||||
factor=video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor,
|
||||
min_pixels=video_processor.min_image_tokens,
|
||||
max_pixels=video_processor.max_image_tokens,
|
||||
)
|
||||
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
|
||||
return (
|
||||
@@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows(
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
frame_indices = glm_sample_frame_indices(
|
||||
total_num_frames,
|
||||
fps,
|
||||
duration,
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=video_processor.temporal_patch_size,
|
||||
frame_indices = video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
|
||||
)
|
||||
grid_t, grid_h, grid_w = _pixel_path_grid(
|
||||
video_processor, len(frame_indices), height, width
|
||||
|
||||
@@ -7,16 +7,13 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
@@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
"linear_num_value_heads": 2,
|
||||
"linear_key_head_dim": 8,
|
||||
"linear_value_head_dim": 8,
|
||||
"num_experts": 0,
|
||||
"num_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
|
||||
"eos_token_id": 1,
|
||||
"hc_count": 2,
|
||||
"hc_lowrank": 4,
|
||||
"ple_layer_ids": [1],
|
||||
|
||||
@@ -10,6 +10,7 @@ import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
@@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import (
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
|
||||
Reference in New Issue
Block a user