Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 18:09:11 +00:00
43 changed files with 1290 additions and 380 deletions
@@ -39,6 +39,14 @@ RERANK_MODELS = [
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
# Declares its score template in its Sentence Transformers config, so no
# chat template is passed.
RerankModelInfo(
"nvidia/llama-nemotron-rerank-vl-1b-v2",
architecture="LlamaNemotronVLForSequenceClassification",
mteb_score=0.33919,
mteb_tol=1e-2,
),
]
@@ -10,7 +10,6 @@ Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
import pybase64 as base64
import pytest
@@ -202,13 +201,6 @@ def test_models(
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
# APIs (it's inherited from the base LLM). We must use the provided override.
_RERANKER_SCORE_TEMPLATE = (
Path(__file__).parents[4]
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
).read_text()
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
RERANKER_TEXT_DOCS = [
"AI enables robots to perceive, plan, and act autonomously.",
@@ -281,7 +273,11 @@ def _run_vllm_reranker(
dtype: str,
input_cases: Sequence[RerankerCase],
) -> list[list[float]]:
"""Run all vLLM reranker cases in one model lifecycle."""
"""Run all vLLM reranker cases in one model lifecycle.
The checkpoint declares its score template in its Sentence Transformers
config, so no chat template is passed.
"""
with vllm_runner(
model,
runner="pooling",
@@ -298,11 +294,7 @@ def _run_vllm_reranker(
if not has_images:
queries = [query] * len(docs)
doc_texts = [doc_text for doc_text, _ in docs]
outputs = vllm_model.score(
queries,
doc_texts,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
outputs = vllm_model.score(queries, doc_texts)
else:
query_params = [
ScoreMultiModalParam(
@@ -334,11 +326,7 @@ def _run_vllm_reranker(
)
doc_params.append(ScoreMultiModalParam(content=content))
raw_outputs = vllm_model.llm.score(
query_params,
doc_params,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
raw_outputs = vllm_model.llm.score(query_params, doc_params)
outputs = [output.outputs.score for output in raw_outputs]
scores_per_case.append(outputs)
+97 -33
View File
@@ -1,16 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import mmap
import os
from dataclasses import dataclass
from functools import partial
from itertools import accumulate
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import QuantizationArgs
from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
@@ -25,6 +26,10 @@ from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.modelopt import (
@@ -39,8 +44,7 @@ from vllm.models.qwen4_exp.amd.ple_layer import (
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
PLEVocabParallelEmbedding,
)
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
@@ -217,12 +221,23 @@ def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
return module
def _make_test_ple_embedding(tp_start: int, tp_end: int) -> PLEVocabParallelEmbedding:
"""Create an embedding with only the state needed for shard loading."""
embedding = PLEVocabParallelEmbedding.__new__(PLEVocabParallelEmbedding)
nn.Module.__init__(embedding)
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=tp_start,
org_vocab_end_index=tp_end,
)
return embedding
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
loader = _make_test_ple_embedding(
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
embedding.weight.weight_loader = loader.weight_loader
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
@@ -251,19 +266,16 @@ def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
overlap = embedding._compute_shard_overlap(checkpoint_start=2, checkpoint_rows=5)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
@@ -272,13 +284,12 @@ def test_ple_shard_overlap_and_copy() -> None:
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
@@ -711,17 +722,44 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
)
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
@pytest.mark.parametrize(
"quant_config",
[
pytest.param(ModelOptNvFp4Config(exclude_modules=[]), id="modelopt-nvfp4"),
pytest.param(
OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")),
id="online-fp8",
),
pytest.param(
CompressedTensorsConfig(
target_scheme_map={
"re:.*ngram_embedding": {
"weights": QuantizationArgs(num_bits=8, type="int"),
},
},
ignore=[],
quant_format="int-quantized",
),
id="compressed-tensors",
),
],
)
def test_ple_embedding_falls_back_for_unsupported_quantization_configs(
quant_config: QuantizationConfig,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Warn when global quantization falls back to an unquantized PLE table."""
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
online_fp8_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="fp8_per_tensor")
test_logger = logging.getLogger("test.ple_fallback")
monkeypatch.setattr(ngram_embedding_module, "logger", test_logger)
caplog.set_level(logging.WARNING, logger=test_logger.name)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
assert caplog.records[0].levelno == logging.WARNING
assert "Falling back to unquantized PLE" in caplog.text
def test_ple_embedding_respects_modelopt_exclusion() -> None:
@@ -748,8 +786,10 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
)
quant_config.extra_config = None
with pytest.raises(NotImplementedError, match="INCConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_is_unquantized_under_quark() -> None:
@@ -933,12 +973,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
loaded_weight = (
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = embedding._lookup(input_ids)
@@ -995,12 +1033,10 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
loaded_weight = loaded_weight.to(torch.bfloat16)
copy_ple_embedding_shard_(
embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
@@ -1019,8 +1055,18 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
@pytest.mark.parametrize(
"exclude_pattern",
[
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding",
"*.ngram_embedding",
],
)
def test_ple_fp8_embedding_supports_mixed_precision_config(
exclude_pattern: str,
) -> None:
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_prefix = "model.language_model.layers.2.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptMixedPrecisionConfig.from_config(
{
"quantization": {
@@ -1029,6 +1075,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
"group_size": 16,
"quantized_layers": {
prefix: {"quant_algo": "FP8"},
nvfp4_prefix: {"quant_algo": "NVFP4"},
"model.language_model.layers.2.moe.gate_proj": {
"quant_algo": "MXFP8"
},
@@ -1041,13 +1088,30 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
# A quant_algo the PLE table cannot load must not fall back to unquantized.
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.3.ple.ple_embedding.ngram_embedding",
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, nvfp4_prefix),
Qwen4ExpPLENvFp4EmbeddingMethod,
)
# An explicitly unsupported PLE format must not fall back to unquantized.
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
)
quant_config.exclude_modules = [exclude_pattern]
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def _patch_ple_construction(
monkeypatch: pytest.MonkeyPatch,
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.amd.mega_moe import (
)
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
from vllm.platforms import current_platform
from vllm.utils.torch_utils import aux_stream
HIDDEN = 64
INTER = 64
@@ -156,7 +157,10 @@ def test_shared_expert_route_appended(monkeypatch):
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
@pytest.mark.parametrize("hash_routing", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 37])
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
@pytest.mark.parametrize("shared_stream", [False, True])
def test_mega_moe_routing_matches_reference(
monkeypatch, hash_routing, num_tokens, shared_stream
):
num_experts, top_k, vocab = 16, 4, 64
scaling = 2.5
device = "cuda"
@@ -195,6 +199,10 @@ def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_token
moe.gate = Gate()
moe.experts = experts
moe.shared_experts = lambda x: 3 * x
moe.shared_experts_stream = aux_stream() if shared_stream else None
moe.shared_experts_events = (
(torch.cuda.Event(), torch.cuda.Event()) if shared_stream else None
)
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = top_k
moe.renormalize = True