[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
+83
-12
@@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend):
|
||||
class Glm5NextVideoBackend(VideoBackend):
|
||||
"""GLM-5.3-Flash fps-interval video backend.
|
||||
|
||||
Selects frames with the same ``glm_sample_frame_indices`` sampler the
|
||||
processor falls back to, so only the sampled frames are
|
||||
materialized. ``fps_interval`` semantics (default 2.0) with a
|
||||
temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
|
||||
pairs kept even. Request overrides: ``fps`` -> fps interval,
|
||||
Selects frames with the same sampler the processor falls back to, so only
|
||||
the sampled frames are materialized. ``fps_interval`` semantics (default
|
||||
2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048,
|
||||
temporal pairs kept even. Request overrides: ``fps`` -> fps interval,
|
||||
``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
|
||||
"""
|
||||
|
||||
_SEEK_GAP_THRESHOLD: ClassVar[int] = 64
|
||||
_DEFAULT_FPS: ClassVar[float] = 2.0
|
||||
_DEFAULT_MAX_FRAMES: ClassVar[int] = 2048
|
||||
|
||||
@classmethod
|
||||
def _sample_frame_indices(
|
||||
cls,
|
||||
total_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
*,
|
||||
target_fps: float | None = None,
|
||||
max_frame_count: int | None = None,
|
||||
temporal_patch_size: int = 2,
|
||||
) -> list[int]:
|
||||
"""GLM video frame sampling (training-reference parity).
|
||||
|
||||
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
|
||||
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
|
||||
frame-dense sources it collects more candidates than ``extract_t`` and
|
||||
the ``> extract_t`` fixup re-spreads the picks uniformly with
|
||||
``np.linspace`` -- that fallback is the intended reference behavior, not
|
||||
an accident. Short clips (fewer frames than ``extract_t``) are spread at
|
||||
evenly spaced timestamps (``floor`` sampling; the linspace variant
|
||||
samples frames unevenly and cost 4 points on video grounding evals).
|
||||
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
|
||||
-> frame cap.
|
||||
"""
|
||||
max_frame_idx = total_frames - 1
|
||||
if not duration:
|
||||
duration = (round(max_frame_idx / fps) + 1) if fps else 0
|
||||
if max_frame_count is None:
|
||||
max_frame_count = cls._DEFAULT_MAX_FRAMES
|
||||
if target_fps is None:
|
||||
target_fps = cls._DEFAULT_FPS
|
||||
|
||||
extract_t = int(duration * target_fps)
|
||||
extract_t = min(extract_t, int(max_frame_count))
|
||||
|
||||
duration_per_frame = 1 / fps
|
||||
timestamps = [i * duration_per_frame for i in range(total_frames)]
|
||||
max_second = int(duration)
|
||||
|
||||
if total_frames < extract_t:
|
||||
frame_indices = [
|
||||
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
|
||||
]
|
||||
else:
|
||||
frame_indices = []
|
||||
current_second = 0.0
|
||||
inv_fps = 1 / (temporal_patch_size * target_fps)
|
||||
for frame_index in range(total_frames):
|
||||
if timestamps[frame_index] >= current_second:
|
||||
current_second += inv_fps
|
||||
frame_indices.append(frame_index)
|
||||
if current_second >= max_second:
|
||||
break
|
||||
|
||||
if len(frame_indices) < extract_t:
|
||||
if len(frame_indices) == 0:
|
||||
start, end = 0, max(total_frames - 1, 0)
|
||||
else:
|
||||
start, end = frame_indices[0], frame_indices[-1]
|
||||
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||
elif len(frame_indices) > extract_t:
|
||||
frame_indices = np.linspace(
|
||||
0, total_frames - 1, extract_t, dtype=int
|
||||
).tolist()
|
||||
|
||||
seen, uniq = set(), []
|
||||
for idx in frame_indices:
|
||||
if idx not in seen:
|
||||
seen.add(idx)
|
||||
uniq.append(int(idx))
|
||||
|
||||
if len(uniq) & 1:
|
||||
uniq.append(uniq[-1])
|
||||
|
||||
return uniq
|
||||
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
@@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend):
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
# Lazy import: the processor module sits behind the
|
||||
# transformers_utils package init, which multimodal must not pull in.
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
return glm_sample_frame_indices(
|
||||
return cls._sample_frame_indices(
|
||||
source.total_frames_num,
|
||||
source.original_fps,
|
||||
source.duration or 0,
|
||||
|
||||
Reference in New Issue
Block a user