Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -39,6 +39,14 @@ RERANK_MODELS = [
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
# Declares its score template in its Sentence Transformers config, so no
|
||||
# chat template is passed.
|
||||
RerankModelInfo(
|
||||
"nvidia/llama-nemotron-rerank-vl-1b-v2",
|
||||
architecture="LlamaNemotronVLForSequenceClassification",
|
||||
mteb_score=0.33919,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
|
||||
|
||||
from collections.abc import Sequence
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
@@ -202,13 +201,6 @@ def test_models(
|
||||
|
||||
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
|
||||
|
||||
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
|
||||
# APIs (it's inherited from the base LLM). We must use the provided override.
|
||||
_RERANKER_SCORE_TEMPLATE = (
|
||||
Path(__file__).parents[4]
|
||||
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
|
||||
).read_text()
|
||||
|
||||
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
|
||||
RERANKER_TEXT_DOCS = [
|
||||
"AI enables robots to perceive, plan, and act autonomously.",
|
||||
@@ -281,7 +273,11 @@ def _run_vllm_reranker(
|
||||
dtype: str,
|
||||
input_cases: Sequence[RerankerCase],
|
||||
) -> list[list[float]]:
|
||||
"""Run all vLLM reranker cases in one model lifecycle."""
|
||||
"""Run all vLLM reranker cases in one model lifecycle.
|
||||
|
||||
The checkpoint declares its score template in its Sentence Transformers
|
||||
config, so no chat template is passed.
|
||||
"""
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
@@ -298,11 +294,7 @@ def _run_vllm_reranker(
|
||||
if not has_images:
|
||||
queries = [query] * len(docs)
|
||||
doc_texts = [doc_text for doc_text, _ in docs]
|
||||
outputs = vllm_model.score(
|
||||
queries,
|
||||
doc_texts,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
outputs = vllm_model.score(queries, doc_texts)
|
||||
else:
|
||||
query_params = [
|
||||
ScoreMultiModalParam(
|
||||
@@ -334,11 +326,7 @@ def _run_vllm_reranker(
|
||||
)
|
||||
doc_params.append(ScoreMultiModalParam(content=content))
|
||||
|
||||
raw_outputs = vllm_model.llm.score(
|
||||
query_params,
|
||||
doc_params,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
raw_outputs = vllm_model.llm.score(query_params, doc_params)
|
||||
outputs = [output.outputs.score for output in raw_outputs]
|
||||
|
||||
scores_per_case.append(outputs)
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
import mmap
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.quantization import QuantizationArgs
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
@@ -25,6 +26,10 @@ from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -39,8 +44,7 @@ from vllm.models.qwen4_exp.amd.ple_layer import (
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
PLEVocabParallelEmbedding,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
@@ -217,12 +221,23 @@ def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
return module
|
||||
|
||||
|
||||
def _make_test_ple_embedding(tp_start: int, tp_end: int) -> PLEVocabParallelEmbedding:
|
||||
"""Create an embedding with only the state needed for shard loading."""
|
||||
embedding = PLEVocabParallelEmbedding.__new__(PLEVocabParallelEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.shard_indices = SimpleNamespace(
|
||||
org_vocab_start_index=tp_start,
|
||||
org_vocab_end_index=tp_end,
|
||||
)
|
||||
return embedding
|
||||
|
||||
|
||||
def _set_test_embedding_weight_loader(embedding) -> None:
|
||||
embedding.weight.weight_loader = partial(
|
||||
copy_ple_embedding_shard_,
|
||||
loader = _make_test_ple_embedding(
|
||||
tp_start=embedding.shard_indices.org_vocab_start_index,
|
||||
tp_end=embedding.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
embedding.weight.weight_loader = loader.weight_loader
|
||||
|
||||
|
||||
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
@@ -251,19 +266,16 @@ def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
|
||||
|
||||
def test_ple_shard_overlap_and_copy() -> None:
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
|
||||
)
|
||||
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
|
||||
overlap = embedding._compute_shard_overlap(checkpoint_start=2, checkpoint_rows=5)
|
||||
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
|
||||
|
||||
destination = torch.full((4, 2), -1.0)
|
||||
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
destination,
|
||||
loaded,
|
||||
checkpoint_start=2,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 3
|
||||
@@ -272,13 +284,12 @@ def test_ple_shard_overlap_and_copy() -> None:
|
||||
|
||||
|
||||
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
|
||||
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
|
||||
destination = torch.ones(4, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
destination,
|
||||
torch.zeros(2, 2),
|
||||
checkpoint_start=10,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 0
|
||||
@@ -711,17 +722,44 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"quant_config",
|
||||
[
|
||||
pytest.param(ModelOptNvFp4Config(exclude_modules=[]), id="modelopt-nvfp4"),
|
||||
pytest.param(
|
||||
OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")),
|
||||
id="online-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
CompressedTensorsConfig(
|
||||
target_scheme_map={
|
||||
"re:.*ngram_embedding": {
|
||||
"weights": QuantizationArgs(num_bits=8, type="int"),
|
||||
},
|
||||
},
|
||||
ignore=[],
|
||||
quant_format="int-quantized",
|
||||
),
|
||||
id="compressed-tensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ple_embedding_falls_back_for_unsupported_quantization_configs(
|
||||
quant_config: QuantizationConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Warn when global quantization falls back to an unquantized PLE table."""
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
|
||||
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
|
||||
|
||||
online_fp8_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="fp8_per_tensor")
|
||||
test_logger = logging.getLogger("test.ple_fallback")
|
||||
monkeypatch.setattr(ngram_embedding_module, "logger", test_logger)
|
||||
caplog.set_level(logging.WARNING, logger=test_logger.name)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
|
||||
assert caplog.records[0].levelno == logging.WARNING
|
||||
assert "Falling back to unquantized PLE" in caplog.text
|
||||
|
||||
|
||||
def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
@@ -748,8 +786,10 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
|
||||
)
|
||||
|
||||
quant_config.extra_config = None
|
||||
with pytest.raises(NotImplementedError, match="INCConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_is_unquantized_under_quark() -> None:
|
||||
@@ -933,12 +973,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
loaded_weight = (
|
||||
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
|
||||
)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
copied = embedding._copy_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = embedding._lookup(input_ids)
|
||||
@@ -995,12 +1033,10 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
|
||||
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||
loaded_weight = loaded_weight.to(torch.bfloat16)
|
||||
copy_ple_embedding_shard_(
|
||||
embedding._copy_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
@@ -1019,8 +1055,18 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"exclude_pattern",
|
||||
[
|
||||
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding",
|
||||
"*.ngram_embedding",
|
||||
],
|
||||
)
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config(
|
||||
exclude_pattern: str,
|
||||
) -> None:
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_prefix = "model.language_model.layers.2.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
@@ -1029,6 +1075,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
"group_size": 16,
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "FP8"},
|
||||
nvfp4_prefix: {"quant_algo": "NVFP4"},
|
||||
"model.language_model.layers.2.moe.gate_proj": {
|
||||
"quant_algo": "MXFP8"
|
||||
},
|
||||
@@ -1041,13 +1088,30 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
# A quant_algo the PLE table cannot load must not fall back to unquantized.
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.3.ple.ple_embedding.ngram_embedding",
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, nvfp4_prefix),
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
)
|
||||
# An explicitly unsupported PLE format must not fall back to unquantized.
|
||||
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
)
|
||||
|
||||
quant_config.exclude_modules = [exclude_pattern]
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def _patch_ple_construction(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import aux_stream
|
||||
|
||||
HIDDEN = 64
|
||||
INTER = 64
|
||||
@@ -156,7 +157,10 @@ def test_shared_expert_route_appended(monkeypatch):
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
@pytest.mark.parametrize("hash_routing", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 37])
|
||||
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
|
||||
@pytest.mark.parametrize("shared_stream", [False, True])
|
||||
def test_mega_moe_routing_matches_reference(
|
||||
monkeypatch, hash_routing, num_tokens, shared_stream
|
||||
):
|
||||
num_experts, top_k, vocab = 16, 4, 64
|
||||
scaling = 2.5
|
||||
device = "cuda"
|
||||
@@ -195,6 +199,10 @@ def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_token
|
||||
moe.gate = Gate()
|
||||
moe.experts = experts
|
||||
moe.shared_experts = lambda x: 3 * x
|
||||
moe.shared_experts_stream = aux_stream() if shared_stream else None
|
||||
moe.shared_experts_events = (
|
||||
(torch.cuda.Event(), torch.cuda.Event()) if shared_stream else None
|
||||
)
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = top_k
|
||||
moe.renormalize = True
|
||||
|
||||
Reference in New Issue
Block a user