[CI] Share BF16 baselines across quantization comparison tests (#58469)

Signed-off-by: Aarushi Jain <Aarushi.Jain2@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Aarushi Jain
2026-09-23 19:26:09 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent 7ef0d69ea0
commit e0f03eb4f4
3 changed files with 53 additions and 12 deletions
+41
View File
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared BF16 baselines for the quantization comparison tests.
Several tests here quantize the same model and compare against an
unquantized run of it. Computing that reference once per session avoids
booting an identical engine for every quantization scheme under test.
Only the outputs are cached; each engine is still torn down as usual, so
tests keep the isolation ``VllmRunner.__exit__`` provides.
"""
import pytest
@pytest.fixture(scope="session")
def quant_baseline_logprobs(vllm_runner):
"""Greedy logprobs for an unquantized model, computed once per config."""
cache: dict = {}
def baseline(
model: str,
prompts: list[str],
*,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
):
key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts))
if key not in cache:
with vllm_runner(
model,
max_model_len=max_model_len,
enforce_eager=True,
) as vllm_model:
cache[key] = vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
return cache[key]
return baseline
@@ -37,6 +37,7 @@ NUM_LOG_PROBS = 8
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_fp8_per_channel_logprobs(
vllm_runner,
quant_baseline_logprobs,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
@@ -52,14 +53,13 @@ def test_fp8_per_channel_logprobs(
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
with vllm_runner(
baseline_outputs = quant_baseline_logprobs(
model,
example_prompts,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
max_tokens=MAX_TOKENS,
num_logprobs=NUM_LOG_PROBS,
)
with vllm_runner(
model,
+6 -6
View File
@@ -42,6 +42,7 @@ NUM_LOG_PROBS = 8
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_mxfp8_logprobs(
vllm_runner,
quant_baseline_logprobs,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
@@ -56,14 +57,13 @@ def test_mxfp8_logprobs(
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
with vllm_runner(
baseline_outputs = quant_baseline_logprobs(
model,
example_prompts,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
max_tokens=MAX_TOKENS,
num_logprobs=NUM_LOG_PROBS,
)
with vllm_runner(
model,