diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-test.sh index eecf70ea2b..5f9355f9e9 100644 --- a/.buildkite/scripts/hardware_ci/run-cpu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test.sh @@ -19,10 +19,9 @@ DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-80} CACHE_MAX_AGE=${CACHE_MAX_AGE:-24h} # Reclaim disk only when the host is under pressure, aging out anything unused -# for less than CACHE_MAX_AGE so hot layers survive -- same `--filter -# until=h` pattern the TPU CI scripts already rely on. `docker buildx -# prune --max-used-space` is a no-op on this host's `docker` driver (BuildKit -# embedded in dockerd never enforces the size cap), so we don't use it. +# for less than CACHE_MAX_AGE so hot layers survive (`--filter until=h`). +# `docker buildx prune --max-used-space` is a no-op on this host's `docker` +# driver (BuildKit embedded in dockerd never enforces the size cap), so we don't use it. prune_if_disk_pressure() { local docker_root disk_usage docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true) diff --git a/.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh b/.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh deleted file mode 100755 index f300b7f809..0000000000 --- a/.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash - -set -xu - - -remove_docker_container() { - docker rm -f tpu-test || true; -} - -trap remove_docker_container EXIT - -# Remove the container that might not be cleaned up in the previous run. -remove_docker_container - -# Build the docker image. -docker build -f docker/Dockerfile.tpu -t vllm-tpu . - -# Set up cleanup. -cleanup_docker() { - # Get Docker's root directory - docker_root=$(docker info -f '{{.DockerRootDir}}') - if [ -z "$docker_root" ]; then - echo "Failed to determine Docker root directory." - exit 1 - fi - echo "Docker root directory: $docker_root" - # Check disk usage of the filesystem where Docker's root directory is located - disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') - # Define the threshold - threshold=70 - if [ "$disk_usage" -gt "$threshold" ]; then - echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - # Remove dangling images (those that are not tagged and not used by any container) - docker image prune -f - # Remove unused volumes / force the system prune for old images as well. - docker volume prune -f && docker system prune --force --filter "until=72h" --all - echo "Docker images and volumes cleanup completed." - else - echo "Disk usage is below $threshold%. No cleanup needed." - fi -} -cleanup_docker - -# For HF_TOKEN. -source /etc/environment - -docker run --privileged --net host --shm-size=16G -it \ - -e "HF_TOKEN=$HF_TOKEN" --name tpu-test \ - vllm-tpu /bin/bash -c ' -set -e # Exit immediately if a command exits with a non-zero status. -set -u # Treat unset variables as an error. - -echo "--- Starting script inside Docker container ---" - -# Create results directory -RESULTS_DIR=$(mktemp -d) -# If mktemp fails, set -e will cause the script to exit. -echo "Results will be stored in: $RESULTS_DIR" - -# Install dependencies -echo "--- Installing Python dependencies ---" -python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \ - && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \ - && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \ - && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0 -echo "--- Python dependencies installed ---" - -export VLLM_XLA_CHECK_RECOMPILATION=1 -export VLLM_XLA_CACHE_PATH= - -echo "--- Hardware Information ---" -# tpu-info -echo "--- Starting Tests ---" -set +e -overall_script_exit_code=0 - -# --- Test Definitions --- -# If a test fails, this function will print logs and will not cause the main script to exit. -run_test() { - local test_num=$1 - local test_name=$2 - local test_command=$3 - local log_file="$RESULTS_DIR/test_${test_num}.log" - local actual_exit_code - - echo "--- TEST_$test_num: Running $test_name ---" - - # Execute the test command. - eval "$test_command" > >(tee -a "$log_file") 2> >(tee -a "$log_file" >&2) - actual_exit_code=$? - - echo "TEST_${test_num}_COMMAND_EXIT_CODE: $actual_exit_code" # This goes to main log - echo "TEST_${test_num}_COMMAND_EXIT_CODE: $actual_exit_code" >> "$log_file" # Also to per-test log - - if [ "$actual_exit_code" -ne 0 ]; then - echo "TEST_$test_num ($test_name) FAILED with exit code $actual_exit_code." >&2 - echo "--- Log for failed TEST_$test_num ($test_name) ---" >&2 - if [ -f "$log_file" ]; then - cat "$log_file" >&2 - else - echo "Log file $log_file not found for TEST_$test_num ($test_name)." >&2 - fi - echo "--- End of log for TEST_$test_num ($test_name) ---" >&2 - return "$actual_exit_code" # Return the failure code - else - echo "TEST_$test_num ($test_name) PASSED." - return 0 # Return success - fi -} - -# Helper function to call run_test and update the overall script exit code -run_and_track_test() { - local test_num_arg="$1" - local test_name_arg="$2" - local test_command_arg="$3" - - # Run the test - run_test "$test_num_arg" "$test_name_arg" "$test_command_arg" - local test_specific_exit_code=$? - - # If the test failed, set the overall script exit code to 1 - if [ "$test_specific_exit_code" -ne 0 ]; then - # No need for extra echo here, run_test already logged the failure. - overall_script_exit_code=1 - fi -} - -# --- Actual Test Execution --- -run_and_track_test 1 "test_struct_output_generate.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\"" -run_and_track_test 2 "test_moe_pallas.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_moe_pallas.py" -run_and_track_test 3 "test_lora.py" \ - "VLLM_XLA_CHECK_RECOMPILATION=0 python3 -m pytest -s -v /workspace/vllm/tests/tpu/lora/test_lora.py" -run_and_track_test 4 "test_tpu_qkv_linear.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_tpu_qkv_linear.py" -run_and_track_test 5 "test_spmd_model_weight_loading.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_spmd_model_weight_loading.py" -run_and_track_test 6 "test_kv_cache_update_kernel.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_kv_cache_update_kernel.py" -run_and_track_test 7 "test_tpu_int8.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_tpu_int8.py" - -# After all tests have been attempted, exit with the overall status. -if [ "$overall_script_exit_code" -ne 0 ]; then - echo "--- One or more tests FAILED. Overall script exiting with failure code 1. ---" -else - echo "--- All tests have completed and PASSED. Overall script exiting with success code 0. ---" -fi -exit "$overall_script_exit_code" -' # IMPORTANT: This is the closing single quote for the bash -c "..." command. Ensure it is present and correct. - -# Capture the exit code of the docker run command -DOCKER_RUN_EXIT_CODE=$? - -# The trap will run for cleanup. -# Exit the main script with the Docker run command's exit code. -if [ "$DOCKER_RUN_EXIT_CODE" -ne 0 ]; then - echo "Docker run command failed with exit code $DOCKER_RUN_EXIT_CODE." - exit "$DOCKER_RUN_EXIT_CODE" -else - echo "Docker run command completed successfully." - exit 0 -fi -# TODO: This test fails because it uses RANDOM_SEED sampling -# pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \ diff --git a/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh b/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh deleted file mode 100755 index 6f4e0d37db..0000000000 --- a/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/bash - -set -xu - - -remove_docker_container() { - docker rm -f tpu-test || true; -} - -trap remove_docker_container EXIT - -# Remove the container that might not be cleaned up in the previous run. -remove_docker_container - -# Build the docker image. -docker build -f docker/Dockerfile.tpu -t vllm-tpu . - -# Set up cleanup. -cleanup_docker() { - # Get Docker's root directory - docker_root=$(docker info -f '{{.DockerRootDir}}') - if [ -z "$docker_root" ]; then - echo "Failed to determine Docker root directory." - exit 1 - fi - echo "Docker root directory: $docker_root" - # Check disk usage of the filesystem where Docker's root directory is located - disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') - # Define the threshold - threshold=70 - if [ "$disk_usage" -gt "$threshold" ]; then - echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - # Remove dangling images (those that are not tagged and not used by any container) - docker image prune -f - # Remove unused volumes / force the system prune for old images as well. - docker volume prune -f && docker system prune --force --filter "until=72h" --all - echo "Docker images and volumes cleanup completed." - else - echo "Disk usage is below $threshold%. No cleanup needed." - fi -} -cleanup_docker - -# For HF_TOKEN. -source /etc/environment - -docker run --privileged --net host --shm-size=16G -it \ - -e "HF_TOKEN=$HF_TOKEN" --name tpu-test \ - vllm-tpu /bin/bash -c ' -set -e # Exit immediately if a command exits with a non-zero status. -set -u # Treat unset variables as an error. - -echo "--- Starting script inside Docker container ---" - -# Create results directory -RESULTS_DIR=$(mktemp -d) -# If mktemp fails, set -e will cause the script to exit. -echo "Results will be stored in: $RESULTS_DIR" - -# Install dependencies -echo "--- Installing Python dependencies ---" -python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \ - && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \ - && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \ - && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0 -echo "--- Python dependencies installed ---" - -export VLLM_XLA_CHECK_RECOMPILATION=1 -export VLLM_XLA_CACHE_PATH= - -echo "--- Hardware Information ---" -# tpu-info -echo "--- Starting Tests ---" -set +e -overall_script_exit_code=0 - -# --- Test Definitions --- -# If a test fails, this function will print logs and will not cause the main script to exit. -run_test() { - local test_num=$1 - local test_name=$2 - local test_command=$3 - local log_file="$RESULTS_DIR/test_${test_num}.log" - local actual_exit_code - - echo "--- TEST_$test_num: Running $test_name ---" - - # Execute the test command. - eval "$test_command" > >(tee -a "$log_file") 2> >(tee -a "$log_file" >&2) - actual_exit_code=$? - - echo "TEST_${test_num}_COMMAND_EXIT_CODE: $actual_exit_code" # This goes to main log - echo "TEST_${test_num}_COMMAND_EXIT_CODE: $actual_exit_code" >> "$log_file" # Also to per-test log - - if [ "$actual_exit_code" -ne 0 ]; then - echo "TEST_$test_num ($test_name) FAILED with exit code $actual_exit_code." >&2 - echo "--- Log for failed TEST_$test_num ($test_name) ---" >&2 - if [ -f "$log_file" ]; then - cat "$log_file" >&2 - else - echo "Log file $log_file not found for TEST_$test_num ($test_name)." >&2 - fi - echo "--- End of log for TEST_$test_num ($test_name) ---" >&2 - return "$actual_exit_code" # Return the failure code - else - echo "TEST_$test_num ($test_name) PASSED." - return 0 # Return success - fi -} - -# Helper function to call run_test and update the overall script exit code -run_and_track_test() { - local test_num_arg="$1" - local test_name_arg="$2" - local test_command_arg="$3" - - # Run the test - run_test "$test_num_arg" "$test_name_arg" "$test_command_arg" - local test_specific_exit_code=$? - - # If the test failed, set the overall script exit code to 1 - if [ "$test_specific_exit_code" -ne 0 ]; then - # No need for extra echo here, run_test already logged the failure. - overall_script_exit_code=1 - fi -} - -# --- Actual Test Execution --- -run_and_track_test 0 "test_perf.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_perf.py" -run_and_track_test 1 "test_compilation.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_compilation.py" -run_and_track_test 2 "test_basic.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_basic.py" -run_and_track_test 3 "test_accuracy.py::test_lm_eval_accuracy_v1_engine" \ - "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine" -run_and_track_test 4 "test_quantization_accuracy.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py" -run_and_track_test 6 "test_tpu_model_runner.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py" -run_and_track_test 7 "test_sampler.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py" -run_and_track_test 8 "test_topk_topp_sampler.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_topk_topp_sampler.py" -run_and_track_test 9 "test_multimodal.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_multimodal.py" -run_and_track_test 10 "test_pallas.py" \ - "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/test_pallas.py" - -# After all tests have been attempted, exit with the overall status. -if [ "$overall_script_exit_code" -ne 0 ]; then - echo "--- One or more tests FAILED. Overall script exiting with failure code 1. ---" -else - echo "--- All tests have completed and PASSED. Overall script exiting with success code 0. ---" -fi -exit "$overall_script_exit_code" -' # IMPORTANT: This is the closing single quote for the bash -c "..." command. Ensure it is present and correct. - -# Capture the exit code of the docker run command -DOCKER_RUN_EXIT_CODE=$? - -# The trap will run for cleanup. -# Exit the main script with the Docker run command's exit code. -if [ "$DOCKER_RUN_EXIT_CODE" -ne 0 ]; then - echo "Docker run command failed with exit code $DOCKER_RUN_EXIT_CODE." - exit "$DOCKER_RUN_EXIT_CODE" -else - echo "Docker run command completed successfully." - exit 0 -fi -# TODO: This test fails because it uses RANDOM_SEED sampling -# pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \ diff --git a/.buildkite/scripts/tpu/cleanup_docker.sh b/.buildkite/scripts/tpu/cleanup_docker.sh deleted file mode 100755 index 740d81fb39..0000000000 --- a/.buildkite/scripts/tpu/cleanup_docker.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -docker_root=$(docker info -f '{{.DockerRootDir}}') -if [ -z "$docker_root" ]; then - echo "Failed to determine Docker root directory." - exit 1 -fi -echo "Docker root directory: $docker_root" -# Check disk usage of the filesystem where Docker's root directory is located -disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') -# Define the threshold -threshold=70 -if [ "$disk_usage" -gt "$threshold" ]; then - echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - # Remove dangling images (those that are not tagged and not used by any container) - docker image prune -f - # Remove unused volumes / force the system prune for old images as well. - docker volume prune -f && docker system prune --force --filter "until=24h" --all - echo "Docker images and volumes cleanup completed." -else - echo "Disk usage is below $threshold%. No cleanup needed." -fi diff --git a/.buildkite/scripts/tpu/config_v6e_1.env b/.buildkite/scripts/tpu/config_v6e_1.env deleted file mode 100644 index c9e3c26571..0000000000 --- a/.buildkite/scripts/tpu/config_v6e_1.env +++ /dev/null @@ -1,14 +0,0 @@ -# Environment config -TEST_NAME=llama8b -CONTAINER_NAME=tpu-test - -# vllm config -MODEL=meta-llama/Llama-3.1-8B-Instruct -MAX_NUM_SEQS=256 -MAX_NUM_BATCHED_TOKENS=1024 -TENSOR_PARALLEL_SIZE=1 -MAX_MODEL_LEN=2048 -DOWNLOAD_DIR=/mnt/disks/persist -EXPECTED_THROUGHPUT=8.0 -INPUT_LEN=1800 -OUTPUT_LEN=128 diff --git a/.buildkite/scripts/tpu/docker_run_bm.sh b/.buildkite/scripts/tpu/docker_run_bm.sh deleted file mode 100755 index efb632e0a8..0000000000 --- a/.buildkite/scripts/tpu/docker_run_bm.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -if [ ! -f "$1" ]; then - echo "Error: The env file '$1' does not exist." - exit 1 # Exit the script with a non-zero status to indicate an error -fi - -ENV_FILE=$1 - -# For testing on local vm, use `set -a` to export all variables -source /etc/environment -# shellcheck source=/dev/null -source "$ENV_FILE" - -remove_docker_container() { - docker rm -f "$CONTAINER_NAME" || true; -} - -trap remove_docker_container EXIT - -# Remove the container that might not be cleaned up in the previous run. -remove_docker_container - -LOG_ROOT=$(mktemp -d) -# If mktemp fails, set -e will cause the script to exit. -echo "Results will be stored in: $LOG_ROOT" - -if [ -z "$HF_TOKEN" ]; then - echo "Error: HF_TOKEN is not set or is empty." - exit 1 -fi - -# Make sure mounted disk or dir exists -if [ ! -d "$DOWNLOAD_DIR" ]; then - echo "Error: Folder $DOWNLOAD_DIR does not exist. This is useually a mounted drive. If no mounted drive, just create a folder." - exit 1 -fi - -echo "Run model $MODEL" -echo - -echo "starting docker...$CONTAINER_NAME" -echo -docker run \ - -v "$DOWNLOAD_DIR":"$DOWNLOAD_DIR" \ - --env-file "$ENV_FILE" \ - -e HF_TOKEN="$HF_TOKEN" \ - -e TARGET_COMMIT="$BUILDKITE_COMMIT" \ - -e MODEL="$MODEL" \ - -e WORKSPACE=/workspace \ - --name "$CONTAINER_NAME" \ - -d \ - --privileged \ - --network host \ - -v /dev/shm:/dev/shm \ - vllm/vllm-tpu-bm tail -f /dev/null - -echo "run script..." -echo -docker exec "$CONTAINER_NAME" /bin/bash -c ".buildkite/scripts/tpu/run_bm.sh" - -echo "copy result back..." -VLLM_LOG="$LOG_ROOT/$TEST_NAME"_vllm_log.txt -BM_LOG="$LOG_ROOT/$TEST_NAME"_bm_log.txt -docker cp "$CONTAINER_NAME:/workspace/vllm_log.txt" "$VLLM_LOG" -docker cp "$CONTAINER_NAME:/workspace/bm_log.txt" "$BM_LOG" - -throughput=$(grep "Request throughput (req/s):" "$BM_LOG" | sed 's/[^0-9.]//g') -echo "throughput for $TEST_NAME at $BUILDKITE_COMMIT: $throughput" - -if [ "$BUILDKITE" = "true" ]; then - echo "Running inside Buildkite" - buildkite-agent artifact upload "$VLLM_LOG" - buildkite-agent artifact upload "$BM_LOG" -else - echo "Not running inside Buildkite" -fi - -# -# compare the throughput with EXPECTED_THROUGHPUT -# and assert meeting the expectation -# -if [[ -z "$throughput" || ! "$throughput" =~ ^[0-9]+([.][0-9]+)?$ ]]; then - echo "Failed to get the throughput" - exit 1 -fi - -if (( $(echo "$throughput < $EXPECTED_THROUGHPUT" | bc -l) )); then - echo "Error: throughput($throughput) is less than expected($EXPECTED_THROUGHPUT)" - exit 1 -fi diff --git a/.buildkite/scripts/tpu/quantized_v6e_1.env b/.buildkite/scripts/tpu/quantized_v6e_1.env deleted file mode 100644 index ecb98d4516..0000000000 --- a/.buildkite/scripts/tpu/quantized_v6e_1.env +++ /dev/null @@ -1,14 +0,0 @@ -# Environment config -TEST_NAME=llama8bw8a8 -CONTAINER_NAME=tpu-test - -# vllm config -MODEL=RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8 -MAX_NUM_SEQS=128 -MAX_NUM_BATCHED_TOKENS=1024 -TENSOR_PARALLEL_SIZE=1 -MAX_MODEL_LEN=2048 -DOWNLOAD_DIR=/mnt/disks/persist -EXPECTED_THROUGHPUT=8.7 -INPUT_LEN=1800 -OUTPUT_LEN=128 diff --git a/.buildkite/scripts/tpu/run_bm.sh b/.buildkite/scripts/tpu/run_bm.sh deleted file mode 100755 index b5d001bea0..0000000000 --- a/.buildkite/scripts/tpu/run_bm.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -VLLM_LOG="$WORKSPACE/vllm_log.txt" -BM_LOG="$WORKSPACE/bm_log.txt" - -if [ -n "$TARGET_COMMIT" ]; then - head_hash=$(git rev-parse HEAD) - if [ "$TARGET_COMMIT" != "$head_hash" ]; then - echo "Error: target commit $TARGET_COMMIT does not match HEAD: $head_hash" - exit 1 - fi -fi - -echo "model: $MODEL" -echo - -# -# create a log folder -# -mkdir "$WORKSPACE/log" - -# TODO: Move to image building. -pip install pandas -pip install datasets - -# -# create sonnet_4x -# -echo "Create sonnet_4x.txt" -echo "" > benchmarks/sonnet_4x.txt -for _ in {1..4} - do - cat benchmarks/sonnet.txt >> benchmarks/sonnet_4x.txt -done - -# -# start vllm service in backend -# -echo "lanching vllm..." -echo "logging to $VLLM_LOG" -echo - -vllm serve "$MODEL" \ - --seed 42 \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ - --tensor-parallel-size "$TENSOR_PARALLEL_SIZE" \ - --no-enable-prefix-caching \ - --download_dir "$DOWNLOAD_DIR" \ - --max-model-len "$MAX_MODEL_LEN" > "$VLLM_LOG" 2>&1 & - - -echo "wait for 20 minutes.." -echo -# sleep 1200 -# wait for 10 minutes... -for _ in {1..120}; do - # TODO: detect other type of errors. - if grep -Fq "raise RuntimeError" "$VLLM_LOG"; then - echo "Detected RuntimeError, exiting." - exit 1 - elif grep -Fq "Application startup complete" "$VLLM_LOG"; then - echo "Application started" - break - else - echo "wait for 10 seconds..." - sleep 10 - fi -done - -# -# run test -# -echo "run benchmark test..." -echo "logging to $BM_LOG" -echo -vllm bench serve \ - --backend vllm \ - --model "$MODEL" \ - --dataset-name sonnet \ - --dataset-path benchmarks/sonnet_4x.txt \ - --sonnet-input-len "$INPUT_LEN" \ - --sonnet-output-len "$OUTPUT_LEN" \ - --ignore-eos > "$BM_LOG" - -echo "completed..." -echo - -throughput=$(grep "Request throughput (req/s):" "$BM_LOG" | sed 's/[^0-9.]//g') -echo "throughput: $throughput" -echo diff --git a/.github/mergify.yml b/.github/mergify.yml index 0798654d34..98c7d0327a 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -549,8 +549,6 @@ pull_request_rules: - label != stale - or: - files~=tpu.py - - files~=_tpu - - files~=tpu_ - files~=/tpu/ actions: label: @@ -564,8 +562,6 @@ pull_request_rules: - label != stale - and: - -files~=tpu.py - - -files~=_tpu - - -files~=tpu_ - -files~=/tpu/ actions: label: diff --git a/tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh deleted file mode 100644 index 9274e3c573..0000000000 --- a/tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -set -xe - -# Hosts / ports -PREFILL_HOST=${PREFILL_HOST:-"localhost"} -PREFILL_PORT=${PREFILL_PORT:-8100} -PREFILL_NIXL_SIDE_PORT=${PREFILL_NIXL_SIDE_PORT:-5577} -DECODE_HOST=${DECODE_HOST:-"localhost"} -DECODE_PORT=${DECODE_PORT:-8200} -PROXY_HOST=${PROXY_HOST:-"localhost"} -PROXY_PORT=${PROXY_PORT:-8192} -BASELINE_HOST=${BASELINE_HOST:-"localhost"} -BASELINE_PORT=${BASELINE_PORT:-9290} - - -# Model to run. -MODEL_NAME=${MODEL_NAME:-"meta-llama/Llama-3.2-3B-Instruct"} -MAX_MODEL_LEN=${MAX_MODEL_LEN:-1024} -BLOCK_SIZE=${BLOCK_SIZE:-32} - - -# execution env -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" -EXP_ROOT="${GIT_ROOT}/tests/v1/kv_connector/nixl_integration" -CONDA_PATH=${CONDA_PATH:-"/home/${USER}/anaconda3"} -CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nixl"} - -OUTPUT_FILE=${OUTPUT_FILE:-"${EXP_ROOT}/.tpu_accuracy_test_outputs.txt"} - -# Trap the SIGINT signal (triggered by Ctrl+C) -trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT - - -# Waits for vLLM server to start. -wait_for_server() { - local host=$1 - local port=$2 - timeout 1200 bash -c " - until curl -s ${host}:${port}/v1/completions > /dev/null; do - sleep 1 - done" && return 0 || return 1 -} - -# Cleanup function -cleanup() { - echo "Caught Ctrl+C, cleaning up..." - # Cleanup commands - pgrep python | xargs kill -9 || true - # pkill -f python || true - echo "Cleanup complete. Exiting." -} - -launch_baseline() { - BASELINE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - VLLM_LOGGING_LEVEL=DEBUG \ - PJRT_DEVICE=TPU \ - VLLM_WORKER_MULTIPROC_METHOD=spawn \ - VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \ - --host ${BASELINE_HOST} \ - --port ${BASELINE_PORT} \ - --max-model-len ${MAX_MODEL_LEN}\ - --seed 42 \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization 0.5 \ - --enforce-eager" - echo "${BASELINE_BASE_CMD}" - ssh -tt "${BASELINE_HOST}" "${BASELINE_BASE_CMD}" & -} - -launch_pd() { - PREFILL_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - UCX_TLS=tcp \ - VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \ - VLLM_LOGGING_LEVEL=DEBUG \ - VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \ - VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \ - PJRT_DEVICE=TPU \ - VLLM_WORKER_MULTIPROC_METHOD=spawn \ - VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \ - --host ${PREFILL_HOST} \ - --port ${PREFILL_PORT} \ - --max-model-len ${MAX_MODEL_LEN}\ - --seed 42 \ - --block-size ${BLOCK_SIZE} \ - --enforce-eager \ - --gpu-memory-utilization 0.5 \ - --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'" - - - DECODE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - UCX_TLS=tcp \ - VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \ - VLLM_LOGGING_LEVEL=DEBUG \ - PJRT_DEVICE=TPU \ - VLLM_WORKER_MULTIPROC_METHOD=spawn \ - VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \ - --host ${DECODE_HOST} \ - --port ${DECODE_PORT} \ - --max-model-len ${MAX_MODEL_LEN}\ - --seed 42 \ - --block-size ${BLOCK_SIZE} \ - --enforce-eager \ - --gpu-memory-utilization 0.5 \ - --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'" - - echo "${PREFILL_BASE_CMD}" - echo "${DECODE_BASE_CMD}" - sleep 2 - - # execute on hosts - ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" & - ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" & - sleep 1 - wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}" - sleep 1 - wait_for_server "${DECODE_HOST}" "${DECODE_PORT}" - sleep 1 -} - -launch_pd_proxy(){ - PROXY_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - python3 ${EXP_ROOT}/toy_proxy_server.py \ - --prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \ - --decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \ - --host=${PROXY_HOST} --port ${PROXY_PORT}" - echo "${PROXY_BASE_CMD}" - ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" & -} - -run_tests(){ - local service_url=$1 - local mode=$2 - python3 "${EXP_ROOT}"/test_disagg_accuracy.py --service_url="${service_url}" --model_name="${MODEL_NAME}" --mode="${mode}" --file_name="${OUTPUT_FILE}" -} - - -# run non-disagg. baseline & save outputs -launch_baseline -sleep 2 -wait_for_server "${BASELINE_HOST}" "${BASELINE_PORT}" -run_tests "http://${BASELINE_HOST}:${BASELINE_PORT}" "baseline" -cleanup -sleep 10 - - -# run disagg. & do exact-match with the outputs from baseline -launch_pd -launch_pd_proxy -sleep 10 -run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg" -echo "-----P/D success----" - -rm "${OUTPUT_FILE}" -cleanup - -exit 0 diff --git a/tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh b/tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh deleted file mode 100644 index 5969455025..0000000000 --- a/tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -set -xe - -# Hosts / ports -PREFILL_HOST=${PREFILL_HOST:-"localhost"} -PREFILL_PORT=${PREFILL_PORT:-8100} -PREFILL_NIXL_SIDE_PORT=${PREFILL_NIXL_SIDE_PORT:-5577} -DECODE_HOST=${DECODE_HOST:-"localhost"} -DECODE_PORT=${DECODE_PORT:-8200} -PROXY_HOST=${PROXY_HOST:-"localhost"} -PROXY_PORT=${PROXY_PORT:-8192} -BASELINE_HOST=${BASELINE_HOST:-"localhost"} -BASELINE_PORT=${BASELINE_PORT:-9290} - - -# Model to run. -MODEL_NAME=${MODEL_NAME:-"meta-llama/Llama-3.2-3B-Instruct"} -MAX_MODEL_LEN=${MAX_MODEL_LEN:-1024} -BLOCK_SIZE=${BLOCK_SIZE:-32} - - -# execution env -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" -EXP_ROOT="${GIT_ROOT}/tests/v1/kv_connector/nixl_integration" -CONDA_PATH=${CONDA_PATH:-"/home/${USER}/anaconda3"} -CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nixl"} - -OUTPUT_FILE=${OUTPUT_FILE:-"${EXP_ROOT}/.tpu_accuracy_test_outputs.txt"} - -# Trap the SIGINT signal (triggered by Ctrl+C) -trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT - -# Waits for vLLM server to start. -wait_for_server() { - local host=$1 - local port=$2 - timeout 1200 bash -c " - until curl -s ${host}:${port}/v1/completions > /dev/null; do - sleep 1 - done" && return 0 || return 1 -} - -# Cleanup function -cleanup() { - echo "Caught Ctrl+C, cleaning up..." - # Cleanup commands - pgrep python | xargs kill -9 || true - # pkill -f python || true - echo "Cleanup complete. Exiting." -} - - -launch_pd() { - PREFILL_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - UCX_TLS=tcp \ - VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \ - VLLM_LOGGING_LEVEL=DEBUG \ - VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \ - VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \ - PJRT_DEVICE=TPU \ - VLLM_WORKER_MULTIPROC_METHOD=spawn \ - VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \ - --host ${PREFILL_HOST} \ - --port ${PREFILL_PORT} \ - --max-model-len ${MAX_MODEL_LEN}\ - --seed 42 \ - --block-size ${BLOCK_SIZE} \ - --enforce-eager \ - --gpu-memory-utilization 0.5 \ - --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'" - - - DECODE_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - UCX_TLS=tcp \ - VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \ - VLLM_LOGGING_LEVEL=DEBUG \ - PJRT_DEVICE=TPU \ - VLLM_WORKER_MULTIPROC_METHOD=spawn \ - VLLM_ENABLE_V1_MULTIPROCESSING=0 vllm serve $MODEL_NAME \ - --host ${DECODE_HOST} \ - --port ${DECODE_PORT} \ - --max-model-len ${MAX_MODEL_LEN}\ - --seed 42 \ - --block-size ${BLOCK_SIZE} \ - --enforce-eager \ - --gpu-memory-utilization 0.5 \ - --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'" - - echo "${PREFILL_BASE_CMD}" - echo "${DECODE_BASE_CMD}" - sleep 2 - - # execute on hosts - ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" & - ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" & - sleep 1 - wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}" - sleep 1 - wait_for_server "${DECODE_HOST}" "${DECODE_PORT}" - sleep 1 -} - -launch_pd_proxy(){ - PROXY_BASE_CMD="source ${CONDA_PATH}/bin/activate ${CONDA_ENV_NAME}; - python3 ${EXP_ROOT}/toy_proxy_server.py \ - --prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \ - --decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \ - --host=${PROXY_HOST} --port ${PROXY_PORT}" - echo "${PROXY_BASE_CMD}" - ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" & -} - - -# run disagg. & do exact-match with the outputs from baseline -launch_pd -launch_pd_proxy -sleep 10 - -PREFILL_HOST=${PREFILL_HOST} \ -PREFILL_PORT=${PREFILL_PORT} \ -DECODE_HOST=${DECODE_HOST} \ -DECODE_PORT=${DECODE_PORT} \ -PROXY_HOST=${PROXY_HOST} \ -PROXY_PORT=${PROXY_PORT} python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py