[ROCm][CI] Stage E gating (#50920)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
d3d79ffc1e
commit
cacc429f62
@@ -83,6 +83,27 @@ steps:
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and not +quant_fp8"
|
||||
# Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Fusion E2E Quick"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
working_dir: "/vllm-workspace/"
|
||||
timeout_in_minutes: 25
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/model_executor/
|
||||
- vllm/v1/attention/
|
||||
- vllm/compilation/
|
||||
- tests/compile/fusions_e2e/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- rocm-smi
|
||||
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
|
||||
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
|
||||
|
||||
- label: ":nvidia: (H100) Fusion E2E Config Sweep"
|
||||
key: fusion-e2e-config-sweep-h100
|
||||
@@ -103,6 +124,28 @@ steps:
|
||||
- nvidia-smi
|
||||
# Run just llama3 (fp8) for all config combinations
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Fusion E2E Config Sweep"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
working_dir: "/vllm-workspace/"
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/compilation/
|
||||
- vllm/model_executor/layers/layernorm.py
|
||||
- vllm/model_executor/layers/activation.py
|
||||
- vllm/model_executor/layers/attention/attention.py
|
||||
- vllm/model_executor/layers/quantization/input_quant_fp8.py
|
||||
- tests/compile/fusions_e2e/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- rocm-smi
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
|
||||
|
||||
- label: ":nvidia: (B200) Fusion E2E Config Sweep"
|
||||
key: fusion-e2e-config-sweep-b200
|
||||
|
||||
@@ -17,6 +17,18 @@ steps:
|
||||
- pytest -v -s cuda/test_cuda_context.py
|
||||
- pytest -v -s cuda/test_platform_no_cuda_init.py
|
||||
- pytest -v -s cuda/test_cuda_compatibility_path.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) CUDA Platform"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
timeout_in_minutes: 20
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/cuda
|
||||
|
||||
- label: ":nvidia: (H200) CUDAGraph"
|
||||
device: h200_35gb
|
||||
@@ -34,3 +46,16 @@ steps:
|
||||
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
|
||||
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
|
||||
- pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) CUDAGraph"
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- tests/v1/cudagraph
|
||||
- vllm/v1/cudagraph_dispatcher.py
|
||||
- vllm/config/compilation.py
|
||||
- vllm/compilation
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
@@ -14,3 +14,19 @@ steps:
|
||||
- tests/tools/test_docker_build_metadata_args.py
|
||||
commands:
|
||||
- pytest -v -s tools/test_docker_build_metadata_args.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) Docker Build Metadata"
|
||||
device: mi250_1
|
||||
no_gpu: true
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- .buildkite/scripts/docker-build-metadata-args.sh
|
||||
- .buildkite/scripts/ci-bake-rocm.sh
|
||||
- docker/Dockerfile.rocm
|
||||
- docker/Dockerfile.rocm_base
|
||||
- docker/ci-rocm.hcl
|
||||
- docker/docker-bake-rocm.hcl
|
||||
- tests/tools/test_docker_build_metadata_args.py
|
||||
|
||||
@@ -51,3 +51,23 @@ steps:
|
||||
working_dir: "/vllm-workspace"
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
working_dir: "/vllm-workspace"
|
||||
timeout_in_minutes: 20
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/backends/mla/
|
||||
- vllm/v1/attention/selector.py
|
||||
- .buildkite/scripts/scheduled_integration_test/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
@@ -235,6 +235,13 @@ steps:
|
||||
- vllm/model_executor/layers/mamba/ops
|
||||
commands:
|
||||
- pytest -v -s kernels/mamba
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) Mamba Kernels"
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 55
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H100) DeepGEMM Kernels"
|
||||
key: kernels-deepgemm-test-h100
|
||||
@@ -393,6 +400,13 @@ steps:
|
||||
- pip install helion==1.4.0
|
||||
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 2
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) Helion Kernels Shard %N"
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
|
||||
- label: ":nvidia: (H100) FP8 MoE Kernels"
|
||||
|
||||
@@ -395,3 +395,19 @@ steps:
|
||||
- tests/evals/mrcr/
|
||||
commands:
|
||||
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) MRCR Eval Small Models"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/evals/mrcr/
|
||||
|
||||
@@ -149,3 +149,19 @@ steps:
|
||||
- >-
|
||||
pytest -v -s
|
||||
'v1/e2e/spec_decode/mtp/test_mtp.py::test_mtp_correctness[qwen3_5-hybrid]'
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Model Runner V2 Spec Decode"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
timeout_in_minutes: 75
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/v1/worker/gpu/
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- tests/v1/spec_decode/test_max_len.py
|
||||
- tests/v1/spec_decode/test_rejection_sampler_utils.py
|
||||
- tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py
|
||||
- tests/v1/e2e/spec_decode/
|
||||
|
||||
@@ -14,6 +14,14 @@ steps:
|
||||
commands:
|
||||
# Run a subset of model initialization tests
|
||||
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Basic Models (Initialization)"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
timeout_in_minutes: 25
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H200) Basic Models (Extra Initialization) Shard %N"
|
||||
device: h200_35gb
|
||||
|
||||
@@ -144,6 +144,32 @@ steps:
|
||||
- tests/models/language/generation_ppl_test
|
||||
commands:
|
||||
- pytest -v -s models/language/generation_ppl_test
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) Language Models (PPL)"
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 65
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/qwen3_5.py
|
||||
- vllm/model_executor/models/qwen3_5_mtp.py
|
||||
- vllm/transformers_utils/configs/qwen3_5.py
|
||||
- vllm/transformers_utils/configs/qwen3_5_moe.py
|
||||
- vllm/model_executor/models/qwen2.py
|
||||
- vllm/model_executor/models/qwen3.py
|
||||
- vllm/model_executor/models/qwen3_next.py
|
||||
- vllm/model_executor/models/qwen3_next_mtp.py
|
||||
- vllm/model_executor/layers/fla/ops/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/v1/attention/backends/triton_attn.py
|
||||
- vllm/v1/attention/backends/rocm_attn.py
|
||||
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
|
||||
- vllm/v1/attention/backends/rocm_aiter_fa.py
|
||||
- vllm/v1/attention/backends/flex_attention.py
|
||||
- vllm/v1/attention/ops/
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/models/language/generation_ppl_test
|
||||
|
||||
- label: ":nvidia: (H200) Language Models (Extended Pooling) Shard %N"
|
||||
device: h200_35gb
|
||||
@@ -180,3 +206,10 @@ steps:
|
||||
- tests/models/language/pooling_mteb_test
|
||||
commands:
|
||||
- pytest -v -s models/language/pooling_mteb_test
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI250) Language Models (MTEB)"
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -36,6 +36,29 @@ steps:
|
||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
|
||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Rust Frontend OpenAI Coverage"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- rust/
|
||||
- vllm/benchmarks/
|
||||
- vllm/entrypoints/openai/
|
||||
- vllm/entrypoints/serve/
|
||||
- vllm/v1/sample/
|
||||
- tests/utils.py
|
||||
- tests/benchmarks/test_serve_cli.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
||||
- tests/entrypoints/openai/test_return_token_ids.py
|
||||
- tests/entrypoints/openai/test_uds.py
|
||||
- tests/v1/sample/test_logprobs_e2e.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
|
||||
timeout_in_minutes: 25
|
||||
@@ -65,6 +88,26 @@ steps:
|
||||
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
|
||||
# /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op).
|
||||
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- rust/
|
||||
- vllm/entrypoints/openai/
|
||||
- vllm/entrypoints/serve/
|
||||
- vllm/v1/engine/
|
||||
- tests/utils.py
|
||||
- tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||
- tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
|
||||
- tests/entrypoints/serve/instrumentator/test_basic.py
|
||||
- tests/entrypoints/serve/instrumentator/test_metrics.py
|
||||
- tests/entrypoints/serve/tokenize/test_tokenization.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200) Rust Frontend Core Correctness"
|
||||
timeout_in_minutes: 20
|
||||
|
||||
Reference in New Issue
Block a user