[CI] Explicitly tear down speculative decode runners (#49910)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-27 05:32:46 +00:00
committed by GitHub
parent 53397fbfac
commit 5f89a03dcb
+43 -33
View File
@@ -5,7 +5,7 @@
import pytest
from tests.utils import get_attn_backend_list_based_on_platform
from vllm import LLM, SamplingParams
from vllm import SamplingParams
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
from vllm.platforms import current_platform
from vllm.sampling_params import StructuredOutputsParams
@@ -18,10 +18,12 @@ _PROMPTS = [
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
def test_ngram_max_len(num_speculative_tokens: int):
llm = LLM(
model="facebook/opt-125m",
def test_ngram_max_len(num_speculative_tokens: int, vllm_runner):
with vllm_runner(
"facebook/opt-125m",
trust_remote_code=False,
max_model_len=100,
enable_chunked_prefill=None,
enforce_eager=True, # For faster initialization.
speculative_config={
"method": "ngram",
@@ -29,21 +31,26 @@ def test_ngram_max_len(num_speculative_tokens: int):
"prompt_lookup_min": 3,
"num_speculative_tokens": num_speculative_tokens,
},
)
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
llm.generate(_PROMPTS, sampling_params)
) as runner:
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
runner.llm.generate(_PROMPTS, sampling_params)
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
@pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform())
def test_eagle_max_len(
monkeypatch: pytest.MonkeyPatch, num_speculative_tokens: int, attn_backend: str
monkeypatch: pytest.MonkeyPatch,
num_speculative_tokens: int,
attn_backend: str,
vllm_runner,
):
if attn_backend == "ROCM_AITER_FA" and current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
llm = LLM(
model="meta-llama/Meta-Llama-3-8B-Instruct",
with vllm_runner(
"meta-llama/Meta-Llama-3-8B-Instruct",
trust_remote_code=False,
enable_chunked_prefill=None,
enforce_eager=True, # For faster initialization.
speculative_config={
"method": "eagle",
@@ -53,31 +60,34 @@ def test_eagle_max_len(
},
max_model_len=200,
attention_config={"backend": attn_backend},
)
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
outputs = llm.generate(_PROMPTS, sampling_params)
for o in outputs:
assert o.outputs[0].finish_reason == "length", (
"This test is only meaningful if the output is truncated due to max length"
)
) as runner:
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
outputs = runner.llm.generate(_PROMPTS, sampling_params)
for o in outputs:
assert o.outputs[0].finish_reason == "length", (
"This test is only meaningful if the output is truncated "
"due to max length"
)
sampling_params = SamplingParams(
max_tokens=200,
structured_outputs=StructuredOutputsParams(regex="^" + "a b c d e " * 15 + "$"),
)
output = llm.generate(_PROMPTS, sampling_params)
for o in output:
assert o.prompt_token_ids is not None
assert (
len(o.prompt_token_ids)
< 80
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
<= 200
), (
"This test is only meaningful if the output "
"is longer than the eagle max length"
sampling_params = SamplingParams(
max_tokens=200,
structured_outputs=StructuredOutputsParams(
regex="^" + "a b c d e " * 15 + "$"
),
)
assert o.outputs[0].text == "a b c d e " * 15
output = runner.llm.generate(_PROMPTS, sampling_params)
for o in output:
assert o.prompt_token_ids is not None
assert (
len(o.prompt_token_ids)
< 80
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
<= 200
), (
"This test is only meaningful if the output "
"is longer than the eagle max length"
)
assert o.outputs[0].text == "a b c d e " * 15
@pytest.mark.parametrize("spec_max_model_len", [80, 150])