From 42ecc8eebe701faf07e7d6f33dbf5a8218825b5b Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 7 Oct 2026 16:29:09 +0800 Subject: [PATCH] [CI][RL] Consolidate RL entrypoint tests under tests/entrypoints/rl (#59948) Signed-off-by: wang.yuqi Signed-off-by: aoshen02 Co-authored-by: wang.yuqi Co-authored-by: Claude Opus 5.5 --- .buildkite/intel_jobs/entrypoints_intel.yaml | 13 ++++++---- .buildkite/test-amd.yaml | 6 +++-- .buildkite/test_areas/entrypoints.yaml | 2 -- .buildkite/test_areas/rl.yaml | 25 +++++++++++++++++++ .buildkite/test_areas/rust_frontend.yaml | 2 -- .github/CODEOWNERS | 3 +-- .../{serve/dev/rlhf => rl}/__init__.py | 0 .../{serve/dev/rlhf => rl}/conftest.py | 0 .../return_routed_experts}/__init__.py | 0 .../return_routed_experts/test_openai_api.py} | 2 +- .../test_scale_out_api.py} | 0 .../rl/state_transitions/__init__.py | 0 .../state_transitions/test_pause_resume.py | 2 +- .../state_transitions}/test_sleep.py | 0 .../state_transitions/test_weight_checker.py | 2 +- .../test_weight_operation_metrics.py | 2 +- .../{ => rl}/weight_transfer/__init__.py | 0 .../weight_transfer/test_offline.py} | 3 +-- .../test_serving_multimodal_tokens.py | 4 +-- tools/pre_commit/mypy.py | 2 +- 20 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 .buildkite/test_areas/rl.yaml rename tests/entrypoints/{serve/dev/rlhf => rl}/__init__.py (100%) rename tests/entrypoints/{serve/dev/rlhf => rl}/conftest.py (100%) rename tests/entrypoints/{serve/dev/rlhf/state_transitions => rl/return_routed_experts}/__init__.py (100%) rename tests/entrypoints/{openai/test_return_routed_experts.py => rl/return_routed_experts/test_openai_api.py} (98%) rename tests/entrypoints/{scale_out/token_in_token_out/test_return_routed_experts.py => rl/return_routed_experts/test_scale_out_api.py} (100%) create mode 100644 tests/entrypoints/rl/state_transitions/__init__.py rename tests/entrypoints/{serve/dev/rlhf => rl}/state_transitions/test_pause_resume.py (98%) rename tests/entrypoints/{serve/dev => rl/state_transitions}/test_sleep.py (100%) rename tests/entrypoints/{serve/dev/rlhf => rl}/state_transitions/test_weight_checker.py (98%) rename tests/entrypoints/{serve/dev/rlhf => rl}/test_weight_operation_metrics.py (97%) rename tests/entrypoints/{ => rl}/weight_transfer/__init__.py (100%) rename tests/entrypoints/{weight_transfer/test_weight_transfer_llm.py => rl/weight_transfer/test_offline.py} (99%) diff --git a/.buildkite/intel_jobs/entrypoints_intel.yaml b/.buildkite/intel_jobs/entrypoints_intel.yaml index 7e823e0245..3cc9c1422d 100644 --- a/.buildkite/intel_jobs/entrypoints_intel.yaml +++ b/.buildkite/intel_jobs/entrypoints_intel.yaml @@ -19,14 +19,14 @@ steps: source_file_dependencies: - vllm/entrypoints - tests/entrypoints/unit_tests - - tests/entrypoints/weight_transfer + - tests/entrypoints/rl/weight_transfer - tests/entrypoints/launchers commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pytest -v -s entrypoints/unit_tests && - pytest -v -s entrypoints/weight_transfer' + pytest -v -s entrypoints/rl/weight_transfer' - label: ":intel: (BMG) Entrypoints Integration (LLM)" key: entrypoints-integration-llm-intel @@ -71,14 +71,17 @@ steps: - vllm/ - "!vllm/distributed/kv_transfer/" - tests/entrypoints/serve + - tests/entrypoints/rl commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - --ignore=entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py - --deselect="tests/entrypoints/serve/dev/rlhf/state_transitions/test_pause_resume.py::TestPauseResume::test_clear_cache_preserves_output_and_controls_prefix_cache[MRV1]" && + pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc && + pytest -v -s entrypoints/rl --ignore=entrypoints/rl/weight_transfer + --ignore=entrypoints/rl/return_routed_experts/test_openai_api.py + --ignore=entrypoints/rl/test_weight_operation_metrics.py + --deselect="tests/entrypoints/rl/state_transitions/test_pause_resume.py::TestPauseResume::test_clear_cache_preserves_output_and_controls_prefix_cache[MRV1]" && PYTHONPATH=/workspace/vllm pytest -v -s entrypoints/serve/dev/rpc' - label: ":intel: (BMG) Entrypoints Integration (API Server Scale Out)" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 17fe9c0ae5..8317919ce4 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1044,12 +1044,12 @@ steps: source_file_dependencies: - vllm/entrypoints - tests/entrypoints/unit_tests - - tests/entrypoints/weight_transfer + - tests/entrypoints/rl/weight_transfer - tests/entrypoints/launchers - vllm/platforms/rocm.py commands: - pytest -v -s entrypoints/unit_tests - - pytest -v -s entrypoints/weight_transfer + - pytest -v -s entrypoints/rl/weight_transfer - pytest -v -s entrypoints/launchers - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD @@ -3746,11 +3746,13 @@ steps: - vllm/ - tests/entrypoints/serve - tests/entrypoints/scale_out + - tests/entrypoints/rl commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s entrypoints/rl --ignore=entrypoints/rl/weight_transfer --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion)" # TBD timeout_in_minutes: 65 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 5f8e1d1c35..68c6243d47 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -31,11 +31,9 @@ steps: source_file_dependencies: - vllm/entrypoints - tests/entrypoints/unit_tests - - tests/entrypoints/weight_transfer - tests/entrypoints/launchers commands: - pytest -v -s entrypoints/unit_tests - - pytest -v -s entrypoints/weight_transfer - pytest -v -s entrypoints/launchers mirror: amd: diff --git a/.buildkite/test_areas/rl.yaml b/.buildkite/test_areas/rl.yaml new file mode 100644 index 0000000000..7141a81bde --- /dev/null +++ b/.buildkite/test_areas/rl.yaml @@ -0,0 +1,25 @@ +group: RL +depends_on: + - image-build +steps: +- label: ":nvidia: (H200 MIG 35GB) RL Entrypoints" + device: h200_35gb + key: rl-entrypoints-tests + timeout_in_minutes: 40 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - "!vllm/distributed/kv_transfer/" + - tests/entrypoints/rl + commands: + - pytest -v -s entrypoints/rl/weight_transfer + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/rl --ignore=entrypoints/rl/weight_transfer + mirror: + amd: + label: ":amd: (MI355 DPX) RL Entrypoints" + dind: false + device: mi355_dpx + timeout_in_minutes: 50 + depends_on: + - image-build-amd diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index c5899a9c59..59c78525c5 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -82,7 +82,6 @@ steps: - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py - tests/entrypoints/serve/instrumentator/test_basic.py - tests/entrypoints/serve/instrumentator/test_metrics.py - # - tests/entrypoints/serve/dev/test_sleep.py - tests/entrypoints/serve/tokenize/test_tokenization.py commands: - export VLLM_USE_RUST_FRONTEND=1 @@ -93,7 +92,6 @@ steps: # test_generate_logprobs expects Python-style top_logprobs truncation (dedup sampled + cap at max(k, 1)). - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist" - # - pytest -v -s entrypoints/serve/dev/test_sleep.py # /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op). - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info" mirror: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 71d501eabe..083842c932 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -231,8 +231,7 @@ mkdocs.yaml @hmellor /vllm/distributed/weight_transfer @aoshen02 /vllm/entrypoints/serve/dev/rlhf @njhill @aoshen02 /tests/distributed/test_weight_transfer.py @aoshen02 -/tests/entrypoints/weight_transfer @DarkLight1337 @robertgshaw2-redhat @aarnphm @NickLucche @AndreasKaratzas @aoshen02 -/tests/entrypoints/serve/dev/rlhf @DarkLight1337 @robertgshaw2-redhat @aarnphm @NickLucche @AndreasKaratzas @aoshen02 +/tests/entrypoints/rl @DarkLight1337 @robertgshaw2-redhat @njhill @aarnphm @NickLucche @AndreasKaratzas @aoshen02 /tests/v1/worker/test_gpu_worker_weight_transfer.py @aoshen02 /examples/rl @aoshen02 /docs/training/rlhf.md @aoshen02 diff --git a/tests/entrypoints/serve/dev/rlhf/__init__.py b/tests/entrypoints/rl/__init__.py similarity index 100% rename from tests/entrypoints/serve/dev/rlhf/__init__.py rename to tests/entrypoints/rl/__init__.py diff --git a/tests/entrypoints/serve/dev/rlhf/conftest.py b/tests/entrypoints/rl/conftest.py similarity index 100% rename from tests/entrypoints/serve/dev/rlhf/conftest.py rename to tests/entrypoints/rl/conftest.py diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/__init__.py b/tests/entrypoints/rl/return_routed_experts/__init__.py similarity index 100% rename from tests/entrypoints/serve/dev/rlhf/state_transitions/__init__.py rename to tests/entrypoints/rl/return_routed_experts/__init__.py diff --git a/tests/entrypoints/openai/test_return_routed_experts.py b/tests/entrypoints/rl/return_routed_experts/test_openai_api.py similarity index 98% rename from tests/entrypoints/openai/test_return_routed_experts.py rename to tests/entrypoints/rl/return_routed_experts/test_openai_api.py index 46eeed1795..f4fc401399 100644 --- a/tests/entrypoints/openai/test_return_routed_experts.py +++ b/tests/entrypoints/rl/return_routed_experts/test_openai_api.py @@ -7,7 +7,7 @@ import numpy as np import pybase64 as base64 import pytest -from ...utils import RemoteOpenAIServer +from tests.utils import RemoteOpenAIServer MODEL_NAME = "TitanML/tiny-mixtral" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py b/tests/entrypoints/rl/return_routed_experts/test_scale_out_api.py similarity index 100% rename from tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py rename to tests/entrypoints/rl/return_routed_experts/test_scale_out_api.py diff --git a/tests/entrypoints/rl/state_transitions/__init__.py b/tests/entrypoints/rl/state_transitions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_pause_resume.py b/tests/entrypoints/rl/state_transitions/test_pause_resume.py similarity index 98% rename from tests/entrypoints/serve/dev/rlhf/state_transitions/test_pause_resume.py rename to tests/entrypoints/rl/state_transitions/test_pause_resume.py index 3de9e00549..587b4b7319 100644 --- a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_pause_resume.py +++ b/tests/entrypoints/rl/state_transitions/test_pause_resume.py @@ -10,7 +10,7 @@ from unittest.mock import patch import pytest import requests -from tests.entrypoints.serve.dev.rlhf.conftest import ( +from tests.entrypoints.rl.conftest import ( cached_tokens, completion_with_cache_details, gen, diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/rl/state_transitions/test_sleep.py similarity index 100% rename from tests/entrypoints/serve/dev/test_sleep.py rename to tests/entrypoints/rl/state_transitions/test_sleep.py diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py b/tests/entrypoints/rl/state_transitions/test_weight_checker.py similarity index 98% rename from tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py rename to tests/entrypoints/rl/state_transitions/test_weight_checker.py index 7cb58df9d9..ec3c8016a8 100644 --- a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py +++ b/tests/entrypoints/rl/state_transitions/test_weight_checker.py @@ -9,7 +9,7 @@ import requests import torch from transformers import AutoModelForCausalLM -from tests.entrypoints.serve.dev.rlhf.conftest import ( +from tests.entrypoints.rl.conftest import ( MODEL_NAME, gen, health, diff --git a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py b/tests/entrypoints/rl/test_weight_operation_metrics.py similarity index 97% rename from tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py rename to tests/entrypoints/rl/test_weight_operation_metrics.py index d7758011f9..108dd77b2f 100644 --- a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py +++ b/tests/entrypoints/rl/test_weight_operation_metrics.py @@ -7,7 +7,7 @@ import torch from prometheus_client.parser import text_string_to_metric_families from transformers import AutoModelForCausalLM -from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME +from tests.entrypoints.rl.conftest import MODEL_NAME from tests.utils import RemoteOpenAIServer from vllm.distributed.weight_transfer import ( HTTPVLLMWeightSyncClient, diff --git a/tests/entrypoints/weight_transfer/__init__.py b/tests/entrypoints/rl/weight_transfer/__init__.py similarity index 100% rename from tests/entrypoints/weight_transfer/__init__.py rename to tests/entrypoints/rl/weight_transfer/__init__.py diff --git a/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py b/tests/entrypoints/rl/weight_transfer/test_offline.py similarity index 99% rename from tests/entrypoints/weight_transfer/test_weight_transfer_llm.py rename to tests/entrypoints/rl/weight_transfer/test_offline.py index e5c3616d6e..e4a30f17e6 100644 --- a/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py +++ b/tests/entrypoints/rl/weight_transfer/test_offline.py @@ -15,6 +15,7 @@ from unittest.mock import patch import pytest import torch +from tests.utils import create_new_process_for_each_test from vllm.config import WeightTransferConfig from vllm.distributed.weight_transfer.base import ( WeightTransferEngine, @@ -24,8 +25,6 @@ from vllm.distributed.weight_transfer.base import ( WeightTransferUpdateRequest, ) -from ...utils import create_new_process_for_each_test - # Use a tiny model for fast testing MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py index 1d017bd3cf..2e58d70972 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py @@ -5,8 +5,8 @@ Mirrors test_serving_tokens.py but exercises the multimodal piping using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render -> /generate -> /detokenize path. Intentionally avoids running the HF -processor in the pytest parent process to keep os.fork() in sibling -tests (e.g. test_weight_transfer_llm.py) deadlock-free. +processor in the pytest parent process to keep os.fork() in other +tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free. """ import httpx diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index a6c8e46419..245d0daf5e 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -38,10 +38,10 @@ SILENT_GROUPS = [ "tests/entrypoints/generate", "tests/entrypoints/multimodal", "tests/entrypoints/pooling", + "tests/entrypoints/rl", "tests/entrypoints/serve", "tests/entrypoints/tool_parsers", "tests/entrypoints/unit_tests", - "tests/entrypoints/weight_transfer", "tests/kernels/attention", "tests/kernels/core", "tests/kernels/helion",