[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -6,7 +6,6 @@ import weakref
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "distilbert/distilgpt2"
|
||||
|
||||
@@ -26,22 +25,17 @@ TOKEN_IDS = [
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_num_batched_tokens=4096,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
|
||||
assert len(outputs) == 1
|
||||
|
||||
|
||||
def test_max_model_len():
|
||||
def test_max_model_len(vllm_runner):
|
||||
max_model_len = 20
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=max_model_len,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True, # reduce test time
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
||||
outputs = llm.generate(PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
num_total_tokens = len(output.prompt_token_ids) + len(
|
||||
output.outputs[0].token_ids
|
||||
)
|
||||
# Total tokens must not exceed max_model_len.
|
||||
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
||||
# before reaching the absolute model length limit.
|
||||
assert num_total_tokens <= max_model_len
|
||||
) as runner:
|
||||
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
||||
outputs = runner.llm.generate(PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
num_total_tokens = len(output.prompt_token_ids) + len(
|
||||
output.outputs[0].token_ids
|
||||
)
|
||||
# Total tokens must not exceed max_model_len.
|
||||
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
||||
# before reaching the absolute model length limit.
|
||||
assert num_total_tokens <= max_model_len
|
||||
|
||||
|
||||
def test_log_stats():
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def test_log_stats(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True, # reduce test time
|
||||
)
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None)
|
||||
) as runner:
|
||||
outputs = runner.llm.generate(PROMPTS, sampling_params=None)
|
||||
|
||||
# disable_log_stats is False, every output should have metrics
|
||||
assert all(output.metrics is not None for output in outputs)
|
||||
# disable_log_stats is False, every output should have metrics
|
||||
assert all(output.metrics is not None for output in outputs)
|
||||
|
||||
Reference in New Issue
Block a user