diff --git a/.buildkite/intel_jobs/basic_correctness.yaml b/.buildkite/intel_jobs/basic_correctness.yaml index 35a1c21075..7d9011d6dc 100644 --- a/.buildkite/intel_jobs/basic_correctness.yaml +++ b/.buildkite/intel_jobs/basic_correctness.yaml @@ -18,11 +18,10 @@ steps: source_file_dependencies: - vllm/ - tests/basic_correctness/test_cumem.py - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && export VLLM_WORKER_MULTIPROC_METHOD=spawn && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s basic_correctness/test_cpu_offload.py && + pytest -v -s basic_correctness/test_cpu_offload.py && pytest -v -s basic_correctness/test_mem.py::test_end_to_end' diff --git a/.buildkite/intel_jobs/benchmarks_intel.yaml b/.buildkite/intel_jobs/benchmarks_intel.yaml index 125cbd8a26..9ce373e4f4 100644 --- a/.buildkite/intel_jobs/benchmarks_intel.yaml +++ b/.buildkite/intel_jobs/benchmarks_intel.yaml @@ -19,9 +19,8 @@ steps: source_file_dependencies: - vllm/ - tests/benchmarks/ - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s benchmarks/' + pytest -v -s benchmarks/' diff --git a/.buildkite/intel_jobs/model_executor_intel.yaml b/.buildkite/intel_jobs/model_executor_intel.yaml index ff3e59cb31..14a853544e 100644 --- a/.buildkite/intel_jobs/model_executor_intel.yaml +++ b/.buildkite/intel_jobs/model_executor_intel.yaml @@ -21,7 +21,6 @@ steps: - vllm/config/model.py - vllm/model_executor - tests/model_executor - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh @@ -31,4 +30,4 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn && export PYTHONFAULTHANDLER=1 && cd tests && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --deselect="tests/model_executor/model_loader/test_reload.py::test_kv_scale_reload"' + pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --deselect="tests/model_executor/model_loader/test_reload.py::test_kv_scale_reload"' diff --git a/.buildkite/intel_jobs/model_runner_v2_intel.yaml b/.buildkite/intel_jobs/model_runner_v2_intel.yaml index 8b13d1ecb4..1f68333e97 100644 --- a/.buildkite/intel_jobs/model_runner_v2_intel.yaml +++ b/.buildkite/intel_jobs/model_runner_v2_intel.yaml @@ -22,14 +22,13 @@ steps: - vllm/v1/attention/ - tests/v1/engine/test_llm_engine.py - tests/v1/e2e/ - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'export VLLM_USE_V2_MODEL_RUNNER=1 && cd tests && pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s v1/e2e/general/test_context_length.py && + pytest -v -s v1/e2e/general/test_context_length.py && ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" && pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" && pytest -v -s v1/e2e/general/test_min_tokens.py' diff --git a/.buildkite/intel_jobs/models_distributed_intel.yaml b/.buildkite/intel_jobs/models_distributed_intel.yaml index 6beafda8c1..5f2247eeaa 100644 --- a/.buildkite/intel_jobs/models_distributed_intel.yaml +++ b/.buildkite/intel_jobs/models_distributed_intel.yaml @@ -20,9 +20,8 @@ steps: - vllm/model_executor/model_loader/sharded_state_loader.py - vllm/model_executor/models/ - tests/model_executor/model_loader/test_sharded_state_loader.py - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"' + pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"' diff --git a/.buildkite/intel_jobs/samplers_intel.yaml b/.buildkite/intel_jobs/samplers_intel.yaml index ffc7fec631..41ec3c4df7 100644 --- a/.buildkite/intel_jobs/samplers_intel.yaml +++ b/.buildkite/intel_jobs/samplers_intel.yaml @@ -22,9 +22,8 @@ steps: - tests/samplers - tests/conftest.py - vllm/entrypoints/generate/beam_search - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - VLLM_XPU_USE_SAMPLER_KERNEL=1 VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s samplers' + VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers' diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 8622fc945d..22b6d46cb8 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -92,7 +92,6 @@ steps: source_file_dependencies: - vllm/ - .buildkite/intel_jobs/test-intel.yaml - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh @@ -103,7 +102,7 @@ steps: pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && pytest -v -s v1/structured_output && pytest -v -s v1/test_serial_utils.py && - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && + pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py && pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py' - label: "XPU server test" diff --git a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh index 1026a9ae15..2f9522b6f1 100644 --- a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh @@ -28,14 +28,14 @@ case "${test_suite}" in ;; v1) cd tests - # Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed + pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py pytest -v -s v1/structured_output pytest -v -s v1/test_serial_utils.py - VLLM_DISABLE_COMPILE_CACHE=1 pytest -v -s v1/e2e/general/test_correctness_sliding_window.py + pytest -v -s v1/e2e/general/test_correctness_sliding_window.py pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py ;; diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh index edbb771623..5acdc8f7cb 100755 --- a/.buildkite/scripts/hardware_ci/run-intel-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh @@ -18,7 +18,9 @@ if [[ "${1:-}" == "--dry-run" ]]; then fi # Export Python path +# Remove VLLM_DISABLE_COMPILE_CACHE=1 once intel/intel-xpu-backend-for-triton#7682 is fixed export PYTHONPATH=".." +export VLLM_DISABLE_COMPILE_CACHE=1 ############################################################################### # Helper Functions @@ -342,7 +344,7 @@ if [[ -z "${ZE_AFFINITY_MASK:-}" ]]; then fi export CMDS="${commands}" -export HF_TOKEN ZE_AFFINITY_MASK +export HF_TOKEN ZE_AFFINITY_MASK VLLM_DISABLE_COMPILE_CACHE { flock 9 @@ -364,6 +366,7 @@ export HF_TOKEN ZE_AFFINITY_MASK --entrypoint='' \ -e HF_TOKEN \ -e ZE_AFFINITY_MASK \ + -e VLLM_DISABLE_COMPILE_CACHE \ -e BUILDKITE_PARALLEL_JOB \ -e BUILDKITE_PARALLEL_JOB_COUNT \ -e CMDS \