[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 02:25:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 866f02db8a
commit e3cae8d2ac
36 changed files with 366 additions and 2385 deletions
+83 -12
View File
@@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend):
class Glm5NextVideoBackend(VideoBackend):
"""GLM-5.3-Flash fps-interval video backend.
Selects frames with the same ``glm_sample_frame_indices`` sampler the
processor falls back to, so only the sampled frames are
materialized. ``fps_interval`` semantics (default 2.0) with a
temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
pairs kept even. Request overrides: ``fps`` -> fps interval,
Selects frames with the same sampler the processor falls back to, so only
the sampled frames are materialized. ``fps_interval`` semantics (default
2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048,
temporal pairs kept even. Request overrides: ``fps`` -> fps interval,
``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
"""
_SEEK_GAP_THRESHOLD: ClassVar[int] = 64
_DEFAULT_FPS: ClassVar[float] = 2.0
_DEFAULT_MAX_FRAMES: ClassVar[int] = 2048
@classmethod
def _sample_frame_indices(
cls,
total_frames: int,
fps: float,
duration: float,
*,
target_fps: float | None = None,
max_frame_count: int | None = None,
temporal_patch_size: int = 2,
) -> list[int]:
"""GLM video frame sampling (training-reference parity).
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
frame-dense sources it collects more candidates than ``extract_t`` and
the ``> extract_t`` fixup re-spreads the picks uniformly with
``np.linspace`` -- that fallback is the intended reference behavior, not
an accident. Short clips (fewer frames than ``extract_t``) are spread at
evenly spaced timestamps (``floor`` sampling; the linspace variant
samples frames unevenly and cost 4 points on video grounding evals).
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
-> frame cap.
"""
max_frame_idx = total_frames - 1
if not duration:
duration = (round(max_frame_idx / fps) + 1) if fps else 0
if max_frame_count is None:
max_frame_count = cls._DEFAULT_MAX_FRAMES
if target_fps is None:
target_fps = cls._DEFAULT_FPS
extract_t = int(duration * target_fps)
extract_t = min(extract_t, int(max_frame_count))
duration_per_frame = 1 / fps
timestamps = [i * duration_per_frame for i in range(total_frames)]
max_second = int(duration)
if total_frames < extract_t:
frame_indices = [
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
]
else:
frame_indices = []
current_second = 0.0
inv_fps = 1 / (temporal_patch_size * target_fps)
for frame_index in range(total_frames):
if timestamps[frame_index] >= current_second:
current_second += inv_fps
frame_indices.append(frame_index)
if current_second >= max_second:
break
if len(frame_indices) < extract_t:
if len(frame_indices) == 0:
start, end = 0, max(total_frames - 1, 0)
else:
start, end = frame_indices[0], frame_indices[-1]
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
elif len(frame_indices) > extract_t:
frame_indices = np.linspace(
0, total_frames - 1, extract_t, dtype=int
).tolist()
seen, uniq = set(), []
for idx in frame_indices:
if idx not in seen:
seen.add(idx)
uniq.append(int(idx))
if len(uniq) & 1:
uniq.append(uniq[-1])
return uniq
@classmethod
def compute_frames_index_to_sample(
@@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend):
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
# Lazy import: the processor module sits behind the
# transformers_utils package init, which multimodal must not pull in.
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
return glm_sample_frame_indices(
return cls._sample_frame_indices(
source.total_frames_num,
source.original_fps,
source.duration or 0,