Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -65,6 +65,8 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor:
|
||||
processor.supports_score_template = False
|
||||
processor.use_sep_token = False
|
||||
processor.model = None
|
||||
processor.saved_chat_template = None
|
||||
processor.saved_chat_template_kwargs = {}
|
||||
return processor
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,14 @@ RERANK_MODELS = [
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
# Declares its score template in its Sentence Transformers config, so no
|
||||
# chat template is passed.
|
||||
RerankModelInfo(
|
||||
"nvidia/llama-nemotron-rerank-vl-1b-v2",
|
||||
architecture="LlamaNemotronVLForSequenceClassification",
|
||||
mteb_score=0.33919,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
|
||||
|
||||
from collections.abc import Sequence
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
@@ -202,13 +201,6 @@ def test_models(
|
||||
|
||||
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
|
||||
|
||||
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
|
||||
# APIs (it's inherited from the base LLM). We must use the provided override.
|
||||
_RERANKER_SCORE_TEMPLATE = (
|
||||
Path(__file__).parents[4]
|
||||
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
|
||||
).read_text()
|
||||
|
||||
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
|
||||
RERANKER_TEXT_DOCS = [
|
||||
"AI enables robots to perceive, plan, and act autonomously.",
|
||||
@@ -281,7 +273,11 @@ def _run_vllm_reranker(
|
||||
dtype: str,
|
||||
input_cases: Sequence[RerankerCase],
|
||||
) -> list[list[float]]:
|
||||
"""Run all vLLM reranker cases in one model lifecycle."""
|
||||
"""Run all vLLM reranker cases in one model lifecycle.
|
||||
|
||||
The checkpoint declares its score template in its Sentence Transformers
|
||||
config, so no chat template is passed.
|
||||
"""
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
@@ -298,11 +294,7 @@ def _run_vllm_reranker(
|
||||
if not has_images:
|
||||
queries = [query] * len(docs)
|
||||
doc_texts = [doc_text for doc_text, _ in docs]
|
||||
outputs = vllm_model.score(
|
||||
queries,
|
||||
doc_texts,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
outputs = vllm_model.score(queries, doc_texts)
|
||||
else:
|
||||
query_params = [
|
||||
ScoreMultiModalParam(
|
||||
@@ -334,11 +326,7 @@ def _run_vllm_reranker(
|
||||
)
|
||||
doc_params.append(ScoreMultiModalParam(content=content))
|
||||
|
||||
raw_outputs = vllm_model.llm.score(
|
||||
query_params,
|
||||
doc_params,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
raw_outputs = vllm_model.llm.score(query_params, doc_params)
|
||||
outputs = [output.outputs.score for output in raw_outputs]
|
||||
|
||||
scores_per_case.append(outputs)
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
import mmap
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.quantization import QuantizationArgs
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
@@ -25,6 +26,10 @@ from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -39,8 +44,7 @@ from vllm.models.qwen4_exp.amd.ple_layer import (
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
PLEVocabParallelEmbedding,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
@@ -217,12 +221,23 @@ def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
return module
|
||||
|
||||
|
||||
def _make_test_ple_embedding(tp_start: int, tp_end: int) -> PLEVocabParallelEmbedding:
|
||||
"""Create an embedding with only the state needed for shard loading."""
|
||||
embedding = PLEVocabParallelEmbedding.__new__(PLEVocabParallelEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.shard_indices = SimpleNamespace(
|
||||
org_vocab_start_index=tp_start,
|
||||
org_vocab_end_index=tp_end,
|
||||
)
|
||||
return embedding
|
||||
|
||||
|
||||
def _set_test_embedding_weight_loader(embedding) -> None:
|
||||
embedding.weight.weight_loader = partial(
|
||||
copy_ple_embedding_shard_,
|
||||
loader = _make_test_ple_embedding(
|
||||
tp_start=embedding.shard_indices.org_vocab_start_index,
|
||||
tp_end=embedding.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
embedding.weight.weight_loader = loader.weight_loader
|
||||
|
||||
|
||||
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
@@ -251,19 +266,16 @@ def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
|
||||
|
||||
def test_ple_shard_overlap_and_copy() -> None:
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
|
||||
)
|
||||
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
|
||||
overlap = embedding._compute_shard_overlap(checkpoint_start=2, checkpoint_rows=5)
|
||||
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
|
||||
|
||||
destination = torch.full((4, 2), -1.0)
|
||||
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
destination,
|
||||
loaded,
|
||||
checkpoint_start=2,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 3
|
||||
@@ -272,13 +284,12 @@ def test_ple_shard_overlap_and_copy() -> None:
|
||||
|
||||
|
||||
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
|
||||
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
|
||||
destination = torch.ones(4, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
destination,
|
||||
torch.zeros(2, 2),
|
||||
checkpoint_start=10,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 0
|
||||
@@ -711,17 +722,44 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"quant_config",
|
||||
[
|
||||
pytest.param(ModelOptNvFp4Config(exclude_modules=[]), id="modelopt-nvfp4"),
|
||||
pytest.param(
|
||||
OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")),
|
||||
id="online-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
CompressedTensorsConfig(
|
||||
target_scheme_map={
|
||||
"re:.*ngram_embedding": {
|
||||
"weights": QuantizationArgs(num_bits=8, type="int"),
|
||||
},
|
||||
},
|
||||
ignore=[],
|
||||
quant_format="int-quantized",
|
||||
),
|
||||
id="compressed-tensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ple_embedding_falls_back_for_unsupported_quantization_configs(
|
||||
quant_config: QuantizationConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Warn when global quantization falls back to an unquantized PLE table."""
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
|
||||
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
|
||||
|
||||
online_fp8_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="fp8_per_tensor")
|
||||
test_logger = logging.getLogger("test.ple_fallback")
|
||||
monkeypatch.setattr(ngram_embedding_module, "logger", test_logger)
|
||||
caplog.set_level(logging.WARNING, logger=test_logger.name)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
|
||||
assert caplog.records[0].levelno == logging.WARNING
|
||||
assert "Falling back to unquantized PLE" in caplog.text
|
||||
|
||||
|
||||
def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
@@ -748,8 +786,10 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
|
||||
)
|
||||
|
||||
quant_config.extra_config = None
|
||||
with pytest.raises(NotImplementedError, match="INCConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_is_unquantized_under_quark() -> None:
|
||||
@@ -933,12 +973,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
loaded_weight = (
|
||||
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
|
||||
)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = embedding._lookup(input_ids)
|
||||
@@ -995,12 +1033,10 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
|
||||
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||
loaded_weight = loaded_weight.to(torch.bfloat16)
|
||||
copy_ple_embedding_shard_(
|
||||
embedding._copy_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
@@ -1019,8 +1055,18 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"exclude_pattern",
|
||||
[
|
||||
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding",
|
||||
"*.ngram_embedding",
|
||||
],
|
||||
)
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config(
|
||||
exclude_pattern: str,
|
||||
) -> None:
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_prefix = "model.language_model.layers.2.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
@@ -1029,6 +1075,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
"group_size": 16,
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "FP8"},
|
||||
nvfp4_prefix: {"quant_algo": "NVFP4"},
|
||||
"model.language_model.layers.2.moe.gate_proj": {
|
||||
"quant_algo": "MXFP8"
|
||||
},
|
||||
@@ -1041,13 +1088,30 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
# A quant_algo the PLE table cannot load must not fall back to unquantized.
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.3.ple.ple_embedding.ngram_embedding",
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, nvfp4_prefix),
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
)
|
||||
# An explicitly unsupported PLE format must not fall back to unquantized.
|
||||
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
)
|
||||
|
||||
quant_config.exclude_modules = [exclude_pattern]
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def _patch_ple_construction(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import aux_stream
|
||||
|
||||
HIDDEN = 64
|
||||
INTER = 64
|
||||
@@ -156,7 +157,10 @@ def test_shared_expert_route_appended(monkeypatch):
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
@pytest.mark.parametrize("hash_routing", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 37])
|
||||
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
|
||||
@pytest.mark.parametrize("shared_stream", [False, True])
|
||||
def test_mega_moe_routing_matches_reference(
|
||||
monkeypatch, hash_routing, num_tokens, shared_stream
|
||||
):
|
||||
num_experts, top_k, vocab = 16, 4, 64
|
||||
scaling = 2.5
|
||||
device = "cuda"
|
||||
@@ -195,6 +199,10 @@ def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_token
|
||||
moe.gate = Gate()
|
||||
moe.experts = experts
|
||||
moe.shared_experts = lambda x: 3 * x
|
||||
moe.shared_experts_stream = aux_stream() if shared_stream else None
|
||||
moe.shared_experts_events = (
|
||||
(torch.cuda.Event(), torch.cuda.Event()) if shared_stream else None
|
||||
)
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = top_k
|
||||
moe.renormalize = True
|
||||
|
||||
@@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_
|
||||
|
||||
from .utils import (
|
||||
create_edit_list_trimmed_video,
|
||||
create_live_webm_video,
|
||||
create_long_gop_video,
|
||||
create_raw_h264_video,
|
||||
create_video_from_image,
|
||||
)
|
||||
|
||||
@@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf(
|
||||
the metadata carries the fps HF assumes, which the frame timestamps use."""
|
||||
get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata
|
||||
|
||||
def unknown_fps(cap) -> VideoSourceMetadata:
|
||||
return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0)
|
||||
def unknown_fps(data) -> VideoSourceMetadata:
|
||||
return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0)
|
||||
|
||||
monkeypatch.setattr(
|
||||
OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps)
|
||||
@@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video(
|
||||
assert qwen_frames.shape[0] >= 4
|
||||
|
||||
|
||||
def test_opencv_demux_counts_frames_when_video_header_lacks_metadata(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Streamed WebM and raw H.264 carry no frame count/duration in their
|
||||
headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``.
|
||||
The loader must count the frames via a demux-only pass, without
|
||||
decoding.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
|
||||
webm_data = create_live_webm_video(num_frames=20, fps=10)
|
||||
frames, metadata = loader.load_bytes(webm_data, backend="opencv")
|
||||
assert frames.shape[0] == 20
|
||||
assert metadata["total_num_frames"] == 20
|
||||
assert metadata["duration"] == pytest.approx(2.0)
|
||||
|
||||
h264_data = create_raw_h264_video(num_frames=20, fps=25)
|
||||
frames, metadata = loader.load_bytes(h264_data, backend="opencv")
|
||||
assert frames.shape[0] == 20
|
||||
assert metadata["total_num_frames"] == 20
|
||||
assert metadata["duration"] == pytest.approx(0.8)
|
||||
|
||||
# The Qwen samplers used to raise on the bogus negative frame count.
|
||||
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data)
|
||||
assert qwen_metadata["total_num_frames"] == 20
|
||||
assert qwen_frames.shape[0] >= 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Frame Recovery Tests
|
||||
# ============================================================================
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -77,8 +79,6 @@ def create_long_gop_video(
|
||||
The marker lets a test recover which frame the decoder actually returned,
|
||||
independent of any metadata label.
|
||||
"""
|
||||
import io
|
||||
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
@@ -103,6 +103,63 @@ def create_long_gop_video(
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_live_webm_video(
|
||||
num_frames: int = 20,
|
||||
fps: int = 10,
|
||||
width: int = 64,
|
||||
height: int = 64,
|
||||
) -> bytes:
|
||||
"""Encode a WebM stream without container duration or frame-count metadata.
|
||||
|
||||
The ``live`` muxing option mimics writing to a pipe: the Segment size is
|
||||
unknown and no Duration or Cues elements are written, so decoder backends
|
||||
cannot learn the frame count from the header.
|
||||
"""
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container:
|
||||
stream = container.add_stream("libvpx", rate=fps)
|
||||
stream.width = width
|
||||
stream.height = height
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_raw_h264_video(
|
||||
num_frames: int = 20,
|
||||
fps: int = 25,
|
||||
width: int = 64,
|
||||
height: int = 64,
|
||||
) -> bytes:
|
||||
"""Encode a raw H.264 Annex B elementary stream (no container metadata)."""
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
with av.open(buf, mode="w", format="h264") as container:
|
||||
stream = container.add_stream("h264", rate=fps)
|
||||
stream.width = width
|
||||
stream.height = height
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_edit_list_trimmed_video(
|
||||
num_frames: int = 90,
|
||||
trim_start_frame: int = 60,
|
||||
@@ -117,8 +174,6 @@ def create_edit_list_trimmed_video(
|
||||
trailing frames are presentable — the shape produced by lossless trims
|
||||
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
|
||||
"""
|
||||
import io
|
||||
|
||||
import av
|
||||
|
||||
src = create_long_gop_video(num_frames=num_frames, fps=fps)
|
||||
|
||||
@@ -99,6 +99,7 @@ def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
|
||||
|
||||
llm = LLM.__new__(LLM)
|
||||
llm.llm_engine = Mock()
|
||||
llm.llm_engine.input_processor.generation_config_fields = {}
|
||||
tokenizer = SimpleNamespace(
|
||||
eos_token_id=0,
|
||||
decode=lambda tokens, skip_special_tokens=False: " ".join(
|
||||
|
||||
@@ -67,6 +67,7 @@ class _VllmConfig:
|
||||
|
||||
class _InputProcessor:
|
||||
vllm_config = _VllmConfig()
|
||||
generation_config_fields: dict[str, Any] = {}
|
||||
|
||||
def resolve_watermarking(self, params):
|
||||
return self.vllm_config._check_supports_watermarking(params)
|
||||
@@ -128,7 +129,7 @@ class _OfflineServing(BeamSearchOfflineMixin):
|
||||
self,
|
||||
instances_batch: list[BeamSearchInstance],
|
||||
base_sampling_params: SamplingParams,
|
||||
eos_token_id: int | None,
|
||||
eos_token_ids: set[int],
|
||||
ignore_eos: bool,
|
||||
beam_width: int,
|
||||
sort_beams_key: Callable[..., Any],
|
||||
@@ -140,6 +141,74 @@ class _OfflineServing(BeamSearchOfflineMixin):
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("offline", [False, True])
|
||||
@pytest.mark.parametrize("config_eos", [None, 11, [0, 11]])
|
||||
@pytest.mark.parametrize("ignore_eos", [False, True])
|
||||
async def test_beam_search_respects_generation_config_eos(
|
||||
monkeypatch, offline: bool, config_eos: int | list[int] | None, ignore_eos: bool
|
||||
) -> None:
|
||||
"""Model-specific end-of-turn tokens must finish beams like tokenizer EOS."""
|
||||
monkeypatch.setattr(
|
||||
_InputProcessor, "generation_config_fields", {"eos_token_id": config_eos}
|
||||
)
|
||||
prompt: TokensInput = {
|
||||
"type": "token",
|
||||
"prompt_token_ids": [1],
|
||||
}
|
||||
params = BeamSearchParams(
|
||||
beam_width=1,
|
||||
max_tokens=3,
|
||||
ignore_eos=ignore_eos,
|
||||
include_stop_str_in_output=True,
|
||||
watermarking=False,
|
||||
)
|
||||
result = await anext(
|
||||
_EngineClient().generate(prompt, SamplingParams(watermarking=False))
|
||||
)
|
||||
result.outputs[0].logprobs = [{11: Logprob(-0.1)}]
|
||||
should_stop = config_eos is not None and not ignore_eos
|
||||
expected_tokens = [11] if should_stop else [11, 11, 11]
|
||||
engine_params: list[SamplingParams] = []
|
||||
|
||||
if offline:
|
||||
offline_serving = _OfflineServing()
|
||||
monkeypatch.setattr(
|
||||
_OfflineServing,
|
||||
"_beam_search_step",
|
||||
BeamSearchOfflineMixin._beam_search_step,
|
||||
)
|
||||
|
||||
def render_and_run_requests(*, params, **kwargs):
|
||||
engine_params.extend(params)
|
||||
return [result]
|
||||
|
||||
monkeypatch.setattr(
|
||||
offline_serving, "_render_and_run_requests", render_and_run_requests
|
||||
)
|
||||
beam = offline_serving.beam_search(cast(list, [prompt]), params)[0].sequences[0]
|
||||
assert beam.tokens == [1, *expected_tokens]
|
||||
else:
|
||||
|
||||
async def generate(engine_prompt, sampling_params, *args, **kwargs):
|
||||
engine_params.append(sampling_params)
|
||||
yield result
|
||||
|
||||
online_serving = _AsyncServing()
|
||||
monkeypatch.setattr(online_serving.engine_client, "generate", generate)
|
||||
output = (
|
||||
await anext(online_serving.beam_search(prompt, "request", params))
|
||||
).outputs[0]
|
||||
assert output.token_ids == expected_tokens
|
||||
assert output.finish_reason == ("stop" if should_stop else "length")
|
||||
assert output.stop_reason == (11 if should_stop else None)
|
||||
|
||||
# The engine validates stop_token_ids against the vocab before it applies
|
||||
# the generation config, so the config EOS ids must not be copied into them.
|
||||
assert engine_params
|
||||
assert all(not p.stop_token_ids for p in engine_params)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_beam_search_handles_extra_logprob_candidates() -> None:
|
||||
prompt: TokensInput = {
|
||||
|
||||
Reference in New Issue
Block a user