Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 18:09:11 +00:00
43 changed files with 1290 additions and 380 deletions
+97 -33
View File
@@ -1,16 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import mmap
import os
from dataclasses import dataclass
from functools import partial
from itertools import accumulate
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import QuantizationArgs
from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
@@ -25,6 +26,10 @@ from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.modelopt import (
@@ -39,8 +44,7 @@ from vllm.models.qwen4_exp.amd.ple_layer import (
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
PLEVocabParallelEmbedding,
)
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
@@ -217,12 +221,23 @@ def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
return module
def _make_test_ple_embedding(tp_start: int, tp_end: int) -> PLEVocabParallelEmbedding:
"""Create an embedding with only the state needed for shard loading."""
embedding = PLEVocabParallelEmbedding.__new__(PLEVocabParallelEmbedding)
nn.Module.__init__(embedding)
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=tp_start,
org_vocab_end_index=tp_end,
)
return embedding
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
loader = _make_test_ple_embedding(
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
embedding.weight.weight_loader = loader.weight_loader
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
@@ -251,19 +266,16 @@ def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
overlap = embedding._compute_shard_overlap(checkpoint_start=2, checkpoint_rows=5)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
@@ -272,13 +284,12 @@ def test_ple_shard_overlap_and_copy() -> None:
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
embedding = _make_test_ple_embedding(tp_start=4, tp_end=8)
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
@@ -711,17 +722,44 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
)
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
@pytest.mark.parametrize(
"quant_config",
[
pytest.param(ModelOptNvFp4Config(exclude_modules=[]), id="modelopt-nvfp4"),
pytest.param(
OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")),
id="online-fp8",
),
pytest.param(
CompressedTensorsConfig(
target_scheme_map={
"re:.*ngram_embedding": {
"weights": QuantizationArgs(num_bits=8, type="int"),
},
},
ignore=[],
quant_format="int-quantized",
),
id="compressed-tensors",
),
],
)
def test_ple_embedding_falls_back_for_unsupported_quantization_configs(
quant_config: QuantizationConfig,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Warn when global quantization falls back to an unquantized PLE table."""
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
online_fp8_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="fp8_per_tensor")
test_logger = logging.getLogger("test.ple_fallback")
monkeypatch.setattr(ngram_embedding_module, "logger", test_logger)
caplog.set_level(logging.WARNING, logger=test_logger.name)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
assert caplog.records[0].levelno == logging.WARNING
assert "Falling back to unquantized PLE" in caplog.text
def test_ple_embedding_respects_modelopt_exclusion() -> None:
@@ -748,8 +786,10 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
)
quant_config.extra_config = None
with pytest.raises(NotImplementedError, match="INCConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_is_unquantized_under_quark() -> None:
@@ -933,12 +973,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
loaded_weight = (
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
)
copied = copy_ple_embedding_shard_(
copied = embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = embedding._lookup(input_ids)
@@ -995,12 +1033,10 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
loaded_weight = loaded_weight.to(torch.bfloat16)
copy_ple_embedding_shard_(
embedding._copy_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
@@ -1019,8 +1055,18 @@ def test_pinned_embedding_start_prefetch_allocates_lazily(
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
@pytest.mark.parametrize(
"exclude_pattern",
[
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding",
"*.ngram_embedding",
],
)
def test_ple_fp8_embedding_supports_mixed_precision_config(
exclude_pattern: str,
) -> None:
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_prefix = "model.language_model.layers.2.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptMixedPrecisionConfig.from_config(
{
"quantization": {
@@ -1029,6 +1075,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
"group_size": 16,
"quantized_layers": {
prefix: {"quant_algo": "FP8"},
nvfp4_prefix: {"quant_algo": "NVFP4"},
"model.language_model.layers.2.moe.gate_proj": {
"quant_algo": "MXFP8"
},
@@ -1041,13 +1088,30 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
# A quant_algo the PLE table cannot load must not fall back to unquantized.
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.3.ple.ple_embedding.ngram_embedding",
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, nvfp4_prefix),
Qwen4ExpPLENvFp4EmbeddingMethod,
)
# An explicitly unsupported PLE format must not fall back to unquantized.
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
)
quant_config.exclude_modules = [exclude_pattern]
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def _patch_ple_construction(
monkeypatch: pytest.MonkeyPatch,