diff --git a/tests/models/quantization/conftest.py b/tests/models/quantization/conftest.py new file mode 100644 index 0000000000..261f7934f4 --- /dev/null +++ b/tests/models/quantization/conftest.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared BF16 baselines for the quantization comparison tests. + +Several tests here quantize the same model and compare against an +unquantized run of it. Computing that reference once per session avoids +booting an identical engine for every quantization scheme under test. + +Only the outputs are cached; each engine is still torn down as usual, so +tests keep the isolation ``VllmRunner.__exit__`` provides. +""" + +import pytest + + +@pytest.fixture(scope="session") +def quant_baseline_logprobs(vllm_runner): + """Greedy logprobs for an unquantized model, computed once per config.""" + cache: dict = {} + + def baseline( + model: str, + prompts: list[str], + *, + max_model_len: int, + max_tokens: int, + num_logprobs: int, + ): + key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts)) + if key not in cache: + with vllm_runner( + model, + max_model_len=max_model_len, + enforce_eager=True, + ) as vllm_model: + cache[key] = vllm_model.generate_greedy_logprobs( + prompts, max_tokens, num_logprobs + ) + return cache[key] + + return baseline diff --git a/tests/models/quantization/test_fp8_per_channel.py b/tests/models/quantization/test_fp8_per_channel.py index f003ccd5f8..ef11b17968 100644 --- a/tests/models/quantization/test_fp8_per_channel.py +++ b/tests/models/quantization/test_fp8_per_channel.py @@ -37,6 +37,7 @@ NUM_LOG_PROBS = 8 @pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"]) def test_fp8_per_channel_logprobs( vllm_runner, + quant_baseline_logprobs, example_prompts, model: str, monkeypatch: pytest.MonkeyPatch, @@ -52,14 +53,13 @@ def test_fp8_per_channel_logprobs( with monkeypatch.context() as m: m.setenv("TOKENIZERS_PARALLELISM", "true") - with vllm_runner( + baseline_outputs = quant_baseline_logprobs( model, + example_prompts, max_model_len=MAX_MODEL_LEN, - enforce_eager=True, - ) as vllm_model: - baseline_outputs = vllm_model.generate_greedy_logprobs( - example_prompts, MAX_TOKENS, NUM_LOG_PROBS - ) + max_tokens=MAX_TOKENS, + num_logprobs=NUM_LOG_PROBS, + ) with vllm_runner( model, diff --git a/tests/models/quantization/test_mxfp8.py b/tests/models/quantization/test_mxfp8.py index 0463468eae..06b2b3cccd 100644 --- a/tests/models/quantization/test_mxfp8.py +++ b/tests/models/quantization/test_mxfp8.py @@ -42,6 +42,7 @@ NUM_LOG_PROBS = 8 @pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"]) def test_mxfp8_logprobs( vllm_runner, + quant_baseline_logprobs, example_prompts, model: str, monkeypatch: pytest.MonkeyPatch, @@ -56,14 +57,13 @@ def test_mxfp8_logprobs( with monkeypatch.context() as m: m.setenv("TOKENIZERS_PARALLELISM", "true") - with vllm_runner( + baseline_outputs = quant_baseline_logprobs( model, + example_prompts, max_model_len=MAX_MODEL_LEN, - enforce_eager=True, - ) as vllm_model: - baseline_outputs = vllm_model.generate_greedy_logprobs( - example_prompts, MAX_TOKENS, NUM_LOG_PROBS - ) + max_tokens=MAX_TOKENS, + num_logprobs=NUM_LOG_PROBS, + ) with vllm_runner( model,