[CI] Explicitly tear down speculative decode runners (#49910)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
import pytest
|
||||
|
||||
from tests.utils import get_attn_backend_list_based_on_platform
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
@@ -18,10 +18,12 @@ _PROMPTS = [
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
def test_ngram_max_len(num_speculative_tokens: int):
|
||||
llm = LLM(
|
||||
model="facebook/opt-125m",
|
||||
def test_ngram_max_len(num_speculative_tokens: int, vllm_runner):
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
trust_remote_code=False,
|
||||
max_model_len=100,
|
||||
enable_chunked_prefill=None,
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
@@ -29,21 +31,26 @@ def test_ngram_max_len(num_speculative_tokens: int):
|
||||
"prompt_lookup_min": 3,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
},
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
|
||||
llm.generate(_PROMPTS, sampling_params)
|
||||
) as runner:
|
||||
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
|
||||
runner.llm.generate(_PROMPTS, sampling_params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
@pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform())
|
||||
def test_eagle_max_len(
|
||||
monkeypatch: pytest.MonkeyPatch, num_speculative_tokens: int, attn_backend: str
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_speculative_tokens: int,
|
||||
attn_backend: str,
|
||||
vllm_runner,
|
||||
):
|
||||
if attn_backend == "ROCM_AITER_FA" and current_platform.is_rocm():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Meta-Llama-3-8B-Instruct",
|
||||
with vllm_runner(
|
||||
"meta-llama/Meta-Llama-3-8B-Instruct",
|
||||
trust_remote_code=False,
|
||||
enable_chunked_prefill=None,
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": "eagle",
|
||||
@@ -53,31 +60,34 @@ def test_eagle_max_len(
|
||||
},
|
||||
max_model_len=200,
|
||||
attention_config={"backend": attn_backend},
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
|
||||
outputs = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in outputs:
|
||||
assert o.outputs[0].finish_reason == "length", (
|
||||
"This test is only meaningful if the output is truncated due to max length"
|
||||
)
|
||||
) as runner:
|
||||
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
|
||||
outputs = runner.llm.generate(_PROMPTS, sampling_params)
|
||||
for o in outputs:
|
||||
assert o.outputs[0].finish_reason == "length", (
|
||||
"This test is only meaningful if the output is truncated "
|
||||
"due to max length"
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=200,
|
||||
structured_outputs=StructuredOutputsParams(regex="^" + "a b c d e " * 15 + "$"),
|
||||
)
|
||||
output = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in output:
|
||||
assert o.prompt_token_ids is not None
|
||||
assert (
|
||||
len(o.prompt_token_ids)
|
||||
< 80
|
||||
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
|
||||
<= 200
|
||||
), (
|
||||
"This test is only meaningful if the output "
|
||||
"is longer than the eagle max length"
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=200,
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
regex="^" + "a b c d e " * 15 + "$"
|
||||
),
|
||||
)
|
||||
assert o.outputs[0].text == "a b c d e " * 15
|
||||
output = runner.llm.generate(_PROMPTS, sampling_params)
|
||||
for o in output:
|
||||
assert o.prompt_token_ids is not None
|
||||
assert (
|
||||
len(o.prompt_token_ids)
|
||||
< 80
|
||||
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
|
||||
<= 200
|
||||
), (
|
||||
"This test is only meaningful if the output "
|
||||
"is longer than the eagle max length"
|
||||
)
|
||||
assert o.outputs[0].text == "a b c d e " * 15
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec_max_model_len", [80, 150])
|
||||
|
||||
Reference in New Issue
Block a user