[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Isotr0py
parent
314ab8cffd
commit
092c35dd15
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
Reference in New Issue
Block a user