Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 18:09:11 +00:00
43 changed files with 1290 additions and 380 deletions
@@ -65,6 +65,8 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor:
processor.supports_score_template = False
processor.use_sep_token = False
processor.model = None
processor.saved_chat_template = None
processor.saved_chat_template_kwargs = {}
return processor
@@ -39,6 +39,14 @@ RERANK_MODELS = [
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
# Declares its score template in its Sentence Transformers config, so no
# chat template is passed.
RerankModelInfo(
"nvidia/llama-nemotron-rerank-vl-1b-v2",
architecture="LlamaNemotronVLForSequenceClassification",
mteb_score=0.33919,
mteb_tol=1e-2,
),
]
@@ -10,7 +10,6 @@ Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
import pybase64 as base64
import pytest
@@ -202,13 +201,6 @@ def test_models(
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
# APIs (it's inherited from the base LLM). We must use the provided override.
_RERANKER_SCORE_TEMPLATE = (
Path(__file__).parents[4]
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
).read_text()
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
RERANKER_TEXT_DOCS = [
"AI enables robots to perceive, plan, and act autonomously.",
@@ -281,7 +273,11 @@ def _run_vllm_reranker(
dtype: str,
input_cases: Sequence[RerankerCase],
) -> list[list[float]]:
"""Run all vLLM reranker cases in one model lifecycle."""
"""Run all vLLM reranker cases in one model lifecycle.
The checkpoint declares its score template in its Sentence Transformers
config, so no chat template is passed.
"""
with vllm_runner(
model,
runner="pooling",
@@ -298,11 +294,7 @@ def _run_vllm_reranker(
if not has_images:
queries = [query] * len(docs)
doc_texts = [doc_text for doc_text, _ in docs]
outputs = vllm_model.score(
queries,
doc_texts,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
outputs = vllm_model.score(queries, doc_texts)
else:
query_params = [
ScoreMultiModalParam(
@@ -334,11 +326,7 @@ def _run_vllm_reranker(
)
doc_params.append(ScoreMultiModalParam(content=content))
raw_outputs = vllm_model.llm.score(
query_params,
doc_params,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
raw_outputs = vllm_model.llm.score(query_params, doc_params)
outputs = [output.outputs.score for output in raw_outputs]
scores_per_case.append(outputs)
+97 -33
View File
@@ -1,16 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import mmap
import os
from dataclasses import dataclass
from functools import partial
from itertools import accumulate
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import QuantizationArgs
from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
@@ -25,6 +26,10 @@ from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.modelopt import (
@@ -39,8 +44,7 @@ from vllm.models.qwen4_exp.amd.ple_layer import (
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
PLEVocabParallelEmbedding,
)
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
@@ -217,12 +221,23 @@ def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
return module
def _make_test_ple_embedding(tp_start: int, tp_end: int) -> PLEVocabParallelEmbedding:
"""Create an embedding with only the state needed for shard loading."""
embedding = PLEVocabParallelEmbedding.__new__(PLEVocabParallelEmbedding)
nn.Module.__init__(embedding)
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=tp_start,
org_vocab_end_index=tp_end,
)
return embedding
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
loader = _make_test_ple_embedding(
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
embedding.weight.weight_loader = loader.weight_loader
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
@@ -251,19 +266,16 @@ def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
overlap = embedding._compute_shard_overlap(checkpoint_start=2, checkpoint_rows=5)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
@@ -272,13 +284,12 @@ def test_ple_shard_overlap_and_copy() -> None:
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
@@ -711,17 +722,44 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
)
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
@pytest.mark.parametrize(
"quant_config",
[
pytest.param(ModelOptNvFp4Config(exclude_modules=[]), id="modelopt-nvfp4"),
pytest.param(
OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")),
id="online-fp8",
),
pytest.param(
CompressedTensorsConfig(
target_scheme_map={
"re:.*ngram_embedding": {
"weights": QuantizationArgs(num_bits=8, type="int"),
},
},
ignore=[],
quant_format="int-quantized",
),
id="compressed-tensors",
),
],
)
def test_ple_embedding_falls_back_for_unsupported_quantization_configs(
quant_config: QuantizationConfig,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Warn when global quantization falls back to an unquantized PLE table."""
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
online_fp8_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="fp8_per_tensor")
test_logger = logging.getLogger("test.ple_fallback")
monkeypatch.setattr(ngram_embedding_module, "logger", test_logger)
caplog.set_level(logging.WARNING, logger=test_logger.name)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
assert caplog.records[0].levelno == logging.WARNING
assert "Falling back to unquantized PLE" in caplog.text
def test_ple_embedding_respects_modelopt_exclusion() -> None:
@@ -748,8 +786,10 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
)
quant_config.extra_config = None
with pytest.raises(NotImplementedError, match="INCConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_is_unquantized_under_quark() -> None:
@@ -933,12 +973,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
loaded_weight = (
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = embedding._lookup(input_ids)
@@ -995,12 +1033,10 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
loaded_weight = loaded_weight.to(torch.bfloat16)
copy_ple_embedding_shard_(
embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
@@ -1019,8 +1055,18 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
@pytest.mark.parametrize(
"exclude_pattern",
[
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding",
"*.ngram_embedding",
],
)
def test_ple_fp8_embedding_supports_mixed_precision_config(
exclude_pattern: str,
) -> None:
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_prefix = "model.language_model.layers.2.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptMixedPrecisionConfig.from_config(
{
"quantization": {
@@ -1029,6 +1075,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
"group_size": 16,
"quantized_layers": {
prefix: {"quant_algo": "FP8"},
nvfp4_prefix: {"quant_algo": "NVFP4"},
"model.language_model.layers.2.moe.gate_proj": {
"quant_algo": "MXFP8"
},
@@ -1041,13 +1088,30 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
# A quant_algo the PLE table cannot load must not fall back to unquantized.
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.3.ple.ple_embedding.ngram_embedding",
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, nvfp4_prefix),
Qwen4ExpPLENvFp4EmbeddingMethod,
)
# An explicitly unsupported PLE format must not fall back to unquantized.
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
)
quant_config.exclude_modules = [exclude_pattern]
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def _patch_ple_construction(
monkeypatch: pytest.MonkeyPatch,
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.amd.mega_moe import (
)
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
from vllm.platforms import current_platform
from vllm.utils.torch_utils import aux_stream
HIDDEN = 64
INTER = 64
@@ -156,7 +157,10 @@ def test_shared_expert_route_appended(monkeypatch):
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
@pytest.mark.parametrize("hash_routing", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 37])
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
@pytest.mark.parametrize("shared_stream", [False, True])
def test_mega_moe_routing_matches_reference(
monkeypatch, hash_routing, num_tokens, shared_stream
):
num_experts, top_k, vocab = 16, 4, 64
scaling = 2.5
device = "cuda"
@@ -195,6 +199,10 @@ def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_token
moe.gate = Gate()
moe.experts = experts
moe.shared_experts = lambda x: 3 * x
moe.shared_experts_stream = aux_stream() if shared_stream else None
moe.shared_experts_events = (
(torch.cuda.Event(), torch.cuda.Event()) if shared_stream else None
)
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = top_k
moe.renormalize = True
+34 -2
View File
@@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_
from .utils import (
create_edit_list_trimmed_video,
create_live_webm_video,
create_long_gop_video,
create_raw_h264_video,
create_video_from_image,
)
@@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf(
the metadata carries the fps HF assumes, which the frame timestamps use."""
get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata
def unknown_fps(cap) -> VideoSourceMetadata:
return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0)
def unknown_fps(data) -> VideoSourceMetadata:
return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0)
monkeypatch.setattr(
OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps)
@@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video(
assert qwen_frames.shape[0] >= 4
def test_opencv_demux_counts_frames_when_video_header_lacks_metadata(
monkeypatch: pytest.MonkeyPatch,
):
"""Streamed WebM and raw H.264 carry no frame count/duration in their
headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``.
The loader must count the frames via a demux-only pass, without
decoding.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
loader = VIDEO_LOADER_REGISTRY.load("opencv")
webm_data = create_live_webm_video(num_frames=20, fps=10)
frames, metadata = loader.load_bytes(webm_data, backend="opencv")
assert frames.shape[0] == 20
assert metadata["total_num_frames"] == 20
assert metadata["duration"] == pytest.approx(2.0)
h264_data = create_raw_h264_video(num_frames=20, fps=25)
frames, metadata = loader.load_bytes(h264_data, backend="opencv")
assert frames.shape[0] == 20
assert metadata["total_num_frames"] == 20
assert metadata["duration"] == pytest.approx(0.8)
# The Qwen samplers used to raise on the bogus negative frame count.
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data)
assert qwen_metadata["total_num_frames"] == 20
assert qwen_frames.shape[0] >= 4
# ============================================================================
# Frame Recovery Tests
# ============================================================================
+59 -4
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import cv2
import numpy as np
import numpy.typing as npt
@@ -77,8 +79,6 @@ def create_long_gop_video(
The marker lets a test recover which frame the decoder actually returned,
independent of any metadata label.
"""
import io
import av
buf = io.BytesIO()
@@ -103,6 +103,63 @@ def create_long_gop_video(
return buf.getvalue()
def create_live_webm_video(
num_frames: int = 20,
fps: int = 10,
width: int = 64,
height: int = 64,
) -> bytes:
"""Encode a WebM stream without container duration or frame-count metadata.
The ``live`` muxing option mimics writing to a pipe: the Segment size is
unknown and no Duration or Cues elements are written, so decoder backends
cannot learn the frame count from the header.
"""
import av
buf = io.BytesIO()
with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container:
stream = container.add_stream("libvpx", rate=fps)
stream.width = width
stream.height = height
stream.pix_fmt = "yuv420p"
for i in range(num_frames):
img = np.zeros((height, width, 3), dtype=np.uint8)
img[:, :, 1] = i % 256
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
for packet in stream.encode(frame):
container.mux(packet)
for packet in stream.encode():
container.mux(packet)
return buf.getvalue()
def create_raw_h264_video(
num_frames: int = 20,
fps: int = 25,
width: int = 64,
height: int = 64,
) -> bytes:
"""Encode a raw H.264 Annex B elementary stream (no container metadata)."""
import av
buf = io.BytesIO()
with av.open(buf, mode="w", format="h264") as container:
stream = container.add_stream("h264", rate=fps)
stream.width = width
stream.height = height
stream.pix_fmt = "yuv420p"
for i in range(num_frames):
img = np.zeros((height, width, 3), dtype=np.uint8)
img[:, :, 1] = i % 256
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
for packet in stream.encode(frame):
container.mux(packet)
for packet in stream.encode():
container.mux(packet)
return buf.getvalue()
def create_edit_list_trimmed_video(
num_frames: int = 90,
trim_start_frame: int = 60,
@@ -117,8 +174,6 @@ def create_edit_list_trimmed_video(
trailing frames are presentable — the shape produced by lossless trims
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
"""
import io
import av
src = create_long_gop_video(num_frames=num_frames, fps=fps)
+1
View File
@@ -99,6 +99,7 @@ def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
llm = LLM.__new__(LLM)
llm.llm_engine = Mock()
llm.llm_engine.input_processor.generation_config_fields = {}
tokenizer = SimpleNamespace(
eos_token_id=0,
decode=lambda tokens, skip_special_tokens=False: " ".join(
+70 -1
View File
@@ -67,6 +67,7 @@ class _VllmConfig:
class _InputProcessor:
vllm_config = _VllmConfig()
generation_config_fields: dict[str, Any] = {}
def resolve_watermarking(self, params):
return self.vllm_config._check_supports_watermarking(params)
@@ -128,7 +129,7 @@ class _OfflineServing(BeamSearchOfflineMixin):
self,
instances_batch: list[BeamSearchInstance],
base_sampling_params: SamplingParams,
eos_token_id: int | None,
eos_token_ids: set[int],
ignore_eos: bool,
beam_width: int,
sort_beams_key: Callable[..., Any],
@@ -140,6 +141,74 @@ class _OfflineServing(BeamSearchOfflineMixin):
return True
@pytest.mark.asyncio
@pytest.mark.parametrize("offline", [False, True])
@pytest.mark.parametrize("config_eos", [None, 11, [0, 11]])
@pytest.mark.parametrize("ignore_eos", [False, True])
async def test_beam_search_respects_generation_config_eos(
monkeypatch, offline: bool, config_eos: int | list[int] | None, ignore_eos: bool
) -> None:
"""Model-specific end-of-turn tokens must finish beams like tokenizer EOS."""
monkeypatch.setattr(
_InputProcessor, "generation_config_fields", {"eos_token_id": config_eos}
)
prompt: TokensInput = {
"type": "token",
"prompt_token_ids": [1],
}
params = BeamSearchParams(
beam_width=1,
max_tokens=3,
ignore_eos=ignore_eos,
include_stop_str_in_output=True,
watermarking=False,
)
result = await anext(
_EngineClient().generate(prompt, SamplingParams(watermarking=False))
)
result.outputs[0].logprobs = [{11: Logprob(-0.1)}]
should_stop = config_eos is not None and not ignore_eos
expected_tokens = [11] if should_stop else [11, 11, 11]
engine_params: list[SamplingParams] = []
if offline:
offline_serving = _OfflineServing()
monkeypatch.setattr(
_OfflineServing,
"_beam_search_step",
BeamSearchOfflineMixin._beam_search_step,
)
def render_and_run_requests(*, params, **kwargs):
engine_params.extend(params)
return [result]
monkeypatch.setattr(
offline_serving, "_render_and_run_requests", render_and_run_requests
)
beam = offline_serving.beam_search(cast(list, [prompt]), params)[0].sequences[0]
assert beam.tokens == [1, *expected_tokens]
else:
async def generate(engine_prompt, sampling_params, *args, **kwargs):
engine_params.append(sampling_params)
yield result
online_serving = _AsyncServing()
monkeypatch.setattr(online_serving.engine_client, "generate", generate)
output = (
await anext(online_serving.beam_search(prompt, "request", params))
).outputs[0]
assert output.token_ids == expected_tokens
assert output.finish_reason == ("stop" if should_stop else "length")
assert output.stop_reason == (11 if should_stop else None)
# The engine validates stop_token_ids against the vocab before it applies
# the generation config, so the config EOS ids must not be copied into them.
assert engine_params
assert all(not p.stop_token_ids for p in engine_params)
@pytest.mark.asyncio
async def test_beam_search_handles_extra_logprob_candidates() -> None:
prompt: TokensInput = {