[CI] Prune PyTorch Fullgraph Test (#51074)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-08-05 12:27:27 -07:00
committed by GitHub
parent 8f158d0ee2
commit e6d67fddb4
3 changed files with 25 additions and 124 deletions
+4 -37
View File
@@ -107,9 +107,9 @@ steps:
commands: commands:
- pytest -s -v compile/passes --ignore compile/passes/distributed - pytest -s -v compile/passes --ignore compile/passes/distributed
- label: PyTorch Fullgraph Smoke Test - label: PyTorch Fullgraph Test
device: h200_35gb device: h200_35gb
key: pytorch-fullgraph-smoke-test key: pytorch-fullgraph-test
timeout_in_minutes: 90 timeout_in_minutes: 90
source_file_dependencies: source_file_dependencies:
- vllm/__init__.py - vllm/__init__.py
@@ -138,11 +138,11 @@ steps:
- vllm/v1/ - vllm/v1/
- tests/compile - tests/compile
commands: commands:
# Run smoke tests under fullgraph directory, except test_full_graph.py # Run smoke tests under fullgraph directory, except test_full_cudagraph.py
# as it is a heavy test that is covered in other steps. # as it is a heavy test that is covered in other steps.
# Use `find` to launch multiple instances of pytest so that # Use `find` to launch multiple instances of pytest so that
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965 # they do not suffer from https://github.com/vllm-project/vllm/issues/28965
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
# Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model # Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
# instances and cannot fit a 35GB MIG slice. L4 retains the original coverage: # instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
@@ -179,39 +179,6 @@ steps:
commands: commands:
- pytest -s -v compile/fullgraph/test_full_cudagraph.py - pytest -s -v compile/fullgraph/test_full_cudagraph.py
- label: PyTorch Fullgraph
key: pytorch-fullgraph
timeout_in_minutes: 40
device: h200_18gb
source_file_dependencies:
- vllm/__init__.py
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
- vllm/compilation/
- vllm/config/
- vllm/distributed/
- vllm/engine/
- vllm/env_override.py
- vllm/envs.py
- vllm/forward_context.py
- vllm/inputs/
- vllm/ir/
- vllm/kernels/
- vllm/logger.py
- vllm/model_executor/
- vllm/multimodal/
- vllm/platforms/
- vllm/plugins/
- vllm/sampling_params.py
- vllm/sequence.py
- vllm/transformers_utils/
- vllm/triton_utils/
- vllm/utils/
- vllm/v1/
- tests/compile
commands:
- pytest -v -s compile/fullgraph/test_full_graph.py
- label: Pytorch Nightly Dependency Override Check # 2min - label: Pytorch Nightly Dependency Override Check # 2min
key: pytorch-nightly-dependency-override-check key: pytorch-nightly-dependency-override-check
# if this test fails, it means the nightly torch version is not compatible with some # if this test fails, it means the nightly torch version is not compatible with some
@@ -36,21 +36,12 @@ class TestSetting:
attn_backend=ATTN_BACKEND, attn_backend=ATTN_BACKEND,
method="generate", method="generate",
), ),
# llama model with quantization
TestSetting(
model="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
model_args=["--quantization", "gptq", "--max-model-len", "2048"],
pp_size=1,
tp_size=1,
attn_backend=ATTN_BACKEND,
method="generate",
),
# MoE model # MoE model
TestSetting( TestSetting(
model="ibm/PowerMoE-3b", model="ibm-granite/granite-3.0-1b-a400m-instruct",
model_args=["--max-model-len", "2048"], model_args=["--max-model-len", "2048"],
pp_size=1, pp_size=1,
tp_size=2, tp_size=1,
attn_backend=ATTN_BACKEND, attn_backend=ATTN_BACKEND,
method="generate", method="generate",
), ),
@@ -72,30 +63,6 @@ class TestSetting:
attn_backend=ATTN_BACKEND, attn_backend=ATTN_BACKEND,
method="encode", method="encode",
), ),
pytest.param(
TestSetting(
model="BAAI/bge-base-en-v1.5",
model_args=["--runner", "pooling"],
pp_size=1,
tp_size=1,
attn_backend="FLASH_ATTN",
method="encode",
),
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Encoder self-attention is not implemented for ROCm",
),
),
# vision language model
# See https://github.com/vllm-project/vllm/issues/26716.
# TestSetting(
# model="microsoft/Phi-3.5-vision-instruct",
# model_args=["--trust-remote-code", "--max-model-len", "2048"],
# pp_size=2,
# tp_size=1,
# attn_backend="FLASH_ATTN",
# method="generate_with_image",
# ),
], ],
) )
def test_compile_correctness( def test_compile_correctness(
@@ -129,29 +96,29 @@ def test_compile_correctness(
all_args: list[list[str]] = [] all_args: list[list[str]] = []
all_envs: list[dict[str, str] | None] = [] all_envs: list[dict[str, str] | None] = []
for comp_mode in [ # Test all compilation modes with inductor backend
for mode in [
CompilationMode.NONE,
CompilationMode.STOCK_TORCH_COMPILE, CompilationMode.STOCK_TORCH_COMPILE,
CompilationMode.DYNAMO_TRACE_ONCE, CompilationMode.DYNAMO_TRACE_ONCE,
CompilationMode.VLLM_COMPILE, CompilationMode.VLLM_COMPILE,
]: ]:
for mode in [CompilationMode.NONE, comp_mode]: all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"])
all_args.append( all_envs.append({})
final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"] # inductor will change the output, so we only compare if the output
) # is close, not exactly the same.
all_envs.append({}) compare_all_settings(
model,
all_args,
all_envs,
method=method if method != "generate" else "generate_close",
force_v1_runner=True,
)
# inductor will change the output, so we only compare if the output all_envs.clear()
# is close, not exactly the same. all_args.clear()
compare_all_settings(
model,
all_args,
all_envs,
method=method if method != "generate" else "generate_close",
force_v1_runner=True,
)
all_envs.clear()
all_args.clear()
# Test all compilation modes with eager backend
for mode in [ for mode in [
CompilationMode.NONE, CompilationMode.NONE,
CompilationMode.STOCK_TORCH_COMPILE, CompilationMode.STOCK_TORCH_COMPILE,
@@ -160,5 +127,4 @@ def test_compile_correctness(
]: ]:
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"]) all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
all_envs.append({}) all_envs.append({})
compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True) compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True)
+2 -34
View File
@@ -21,28 +21,12 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
TEST_MODELS: list[tuple[str, dict[str, Any]]] = [ TEST_MODELS: list[tuple[str, dict[str, Any]]] = [
("facebook/opt-125m", {}), ("facebook/opt-125m", {}),
( (
"neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic", "RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic",
{"dtype": torch.float16}, {"dtype": torch.float16},
), ),
("meta-llama/Llama-3.2-1B-Instruct", {}),
] ]
if all: if all:
TEST_MODELS.extend(
[
("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {}),
(
"nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
{"dtype": torch.float16},
),
]
)
if is_quant_method_supported("gptq"):
TEST_MODELS.append(
("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {"quantization": "gptq"})
)
if is_quant_method_supported("gptq_marlin"): if is_quant_method_supported("gptq_marlin"):
TEST_MODELS.append( TEST_MODELS.append(
( (
@@ -76,14 +60,6 @@ def test_full_graph(
model_kwargs: dict[str, Any], model_kwargs: dict[str, Any],
compilation_mode: CompilationMode, compilation_mode: CompilationMode,
): ):
if (
"w8a8" in model
or "w8w8" in model
and current_platform.has_device_capability((10, 0))
):
# int8 removed on Blackwell:
pytest.skip("int8 support removed on Blackwell")
with monkeypatch.context(): with monkeypatch.context():
print(f"MODEL={model}") print(f"MODEL={model}")
@@ -113,7 +89,7 @@ def test_full_graph(
), ),
*model_info, *model_info,
) )
for model_info in models_list(keywords=["FP8-dynamic", "quantized.w8a8"]) for model_info in models_list(keywords=["FP8-dynamic"])
] ]
+ [ + [
# Test depyf integration works # Test depyf integration works
@@ -164,14 +140,6 @@ def test_custom_compile_config(
model: str, model: str,
model_kwargs: dict[str, Any], model_kwargs: dict[str, Any],
): ):
if (
"w8a8" in model
or "w8w8" in model
and current_platform.has_device_capability((10, 0))
):
# int8 removed on Blackwell:
pytest.skip("int8 support removed on Blackwell")
if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer( if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer(
"2.9.0.dev" "2.9.0.dev"
): ):