[torch.compile] register allreduce operations as custom ops (#8526)
This commit is contained in:
@@ -2,9 +2,20 @@ import os
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils import cuda_device_count_stateless
|
||||
|
||||
from ..utils import fork_new_process_for_each_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
|
||||
def test_full_graph(model):
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@fork_new_process_for_each_test
|
||||
def test_full_graph(model, tp_size):
|
||||
|
||||
# Skip the test if there are not enough CUDA devices.
|
||||
if cuda_device_count_stateless() < tp_size:
|
||||
pytest.skip("Not enough CUDA devices for the test.")
|
||||
|
||||
# make sure these models can be captured in full graph mode
|
||||
if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
|
||||
os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
|
||||
@@ -17,7 +28,7 @@ def test_full_graph(model):
|
||||
"The future of AI is",
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
llm = LLM(model=model, enforce_eager=True)
|
||||
llm = LLM(model=model, enforce_eager=True, tensor_parallel_size=tp_size)
|
||||
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user