Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-27 18:25:21 +09:00
1165 changed files with 51759 additions and 14579 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ run_all_patterns:
- "requirements/test/rocm.txt"
- "pyproject.toml"
- "setup.py"
- "build_rust.sh"
- "tools/build_rust.sh"
- "rust/"
- "rust-toolchain.toml"
- "tools/build_rust.py"
@@ -49,7 +49,16 @@ echo "Image not found, proceeding with build..."
# Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9
NIGHTLY_CUDA_VERSION="13.0.2"
NIGHTLY_BUILD_BASE_IMAGE="pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359"
NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04"
# Must track ARG UBUNTU_VERSION in docker/Dockerfile. The nightly lane exists to
# run the normal pipeline against nightly torch, so it has to differ from the
# regular image in torch only -- not in the OS underneath it. This was pinned to
# 22.04 while everything else moved to 24.04, and the runtime dependency list in
# tools/install_snapshot_runtime.sh is 24.04-only (libgnutls30t64 is the 64-bit
# time_t rename and does not exist on jammy), so the nightly build failed with
# "E: Unable to locate package libgnutls30t64" while regular CI stayed green on
# a cached layer.
NIGHTLY_UBUNTU_VERSION="24.04"
NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu${NIGHTLY_UBUNTU_VERSION}"
echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})"
docker buildx build --file docker/Dockerfile \
@@ -18,11 +18,12 @@ steps:
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/basic_correctness/test_cumem.py
- tests/basic_correctness/memory/
- tests/basic_correctness/cpu_offload/
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
pytest -v -s basic_correctness/test_cpu_offload.py &&
pytest -v -s basic_correctness/test_mem.py'
pytest -v -s basic_correctness/cpu_offload &&
pytest -v -s basic_correctness/memory'
+1 -1
View File
@@ -63,7 +63,7 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py &&
VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py'
pytest -v -s test_config.py'
- label: V1 e2e (2 GPUs)
key: v1-e2e-2-gpus
+17 -13
View File
@@ -18,7 +18,7 @@ steps:
key: xpu-example-test
depends_on:
- image-build-xpu
timeout_in_minutes: 50
timeout_in_minutes: 70
device: intel_gpu
agent_tags:
label: production
@@ -38,7 +38,7 @@ steps:
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
@@ -47,7 +47,7 @@ steps:
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC -tp 2 --max-model-len 8192
'
- label: "XPU W8A8 FP8 Linear Examples"
key: xpu-w8a8-fp8-linear-examples
@@ -74,8 +74,8 @@ steps:
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
'
- label: "XPU V1 test"
key: xpu-v1-test
@@ -105,8 +105,8 @@ steps:
pytest -v -s v1/structured_output &&
pytest -v -s v1/test_serial_utils.py &&
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --deselect "v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]"'
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head &&
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --deselect "v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]" --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py'
parallelism: 2
- label: "XPU server test"
key: xpu-server-test
@@ -196,16 +196,20 @@ steps:
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/compilation/
- vllm/config/compilation.py
- vllm/platforms/xpu.py
- vllm/v1/cudagraph_dispatcher.py
- vllm/v1/worker/encoder_cudagraph.py
- vllm/v1/worker/encoder_cudagraph_defs.py
- vllm/v1/worker/xpu_model_runner.py
- vllm/v1/worker/xpu_worker.py
- tests/v1/cudagraph
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_XPU_ENABLE_XPU_GRAPH=1 &&
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B &&
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8 &&
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096'
# Temporarily disabled (multi-GPU/MXFP8 not yet supported with XPU Graph):
# python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP8-CT-AutoRound -tp 2 --max-model-len 4096
'cd tests &&
pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py &&
pytest -v -s v1/cudagraph/test_cudagraph_mode.py &&
pytest -v -s v1/cudagraph/test_breakable_cudagraph.py &&
pytest -v -s v1/cudagraph/test_encoder_cudagraph.py'
+1 -1
View File
@@ -23,7 +23,7 @@ DEFAULT_CI_BASE_METADATA_VERSION="3"
# local-source stages rather than unreachable remote-fetch alternatives.
DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py"
DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies rocm-triton-kernels csrc-build"
DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore .git_archival.txt pyproject.toml requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py build_rust.sh"
DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore .git_archival.txt pyproject.toml requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/build_rust.sh"
DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm vllm-version rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build"
# Docker's 128-character tag limit minus the longest cache prefix
# ("csrc-rocm-branch-" and "rust-rocm-branch-", both 17 characters).
+67 -7
View File
@@ -80,13 +80,73 @@ fi
BUILD_JSON="$(mktemp)"
trap 'rm -f "${BUILD_JSON}"' EXIT
http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
-H "Authorization: Bearer ${BK_TOKEN}" \
"https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")"
if [[ "${http_code}" != "200" ]]; then
echo "buildkite API returned ${http_code} -- skipping report"
exit 0
fi
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
fetch_build() {
curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
}
# Count jobs that have not reached a terminal state, ignoring this job: the
# report is itself part of the build and cannot wait for itself to finish.
outstanding_jobs() {
python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY'
import json, sys
TERMINAL = {
"passed", "failed", "blocked", "canceled", "skipped", "not_run",
"broken", "timed_out", "waiting_failed", "finished", "expired",
}
build = json.load(open(sys.argv[1]))
self_id = sys.argv[2]
print(sum(
1 for j in build.get("jobs", [])
if j.get("type") == "script"
and j.get("id") != self_id
and j.get("state") not in TERMINAL
))
PY
}
# Wait for the rest of the build. The report is a snapshot of the Buildkite
# API's view of this build, so taking it early does not merely omit jobs -- it
# actively reports them as clean. Build 90830 reported with 14 of 347 jobs
# finished and none of the eventual 152 hard failures visible, and HUD recorded
# that nightly as green.
#
# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
# key, so a barrier means naming all ~250 step keys, which silently rots when a
# step is added and fails the pipeline upload when one is removed or excluded
# from this lane (AMD, retired A100). The pipeline does gate this step on the
# long pole so it is scheduled near the end of the build -- that keeps the agent
# from idling for hours, but it is only a hint, and this loop is what actually
# guarantees the snapshot is complete.
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
# Under the step's own timeout, so a stuck build still gets a partial report
# rather than the job being killed with nothing sent.
WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
while :; do
http_code="$(fetch_build)"
if [[ "${http_code}" != "200" ]]; then
echo "buildkite API returned ${http_code} -- skipping report"
exit 0
fi
remaining="$(outstanding_jobs)" || remaining=""
if [[ -z "${remaining}" ]]; then
echo "could not parse build JSON -- reporting what we have"
break
fi
if (( remaining == 0 )); then
echo "all other jobs have finished -- reporting"
break
fi
if (( $(date +%s) >= WAIT_DEADLINE )); then
echo "still ${remaining} job(s) running at the wait deadline --" \
"reporting a partial view"
break
fi
echo "waiting for ${remaining} job(s) to finish..."
sleep "${POLL_INTERVAL_S}"
done
# One callback per job, matching HUD's per-job crcr_workflow_job schema.
# delivery_id is synthetic: the nightly path has no upstream dispatch to borrow
+36 -5
View File
@@ -605,6 +605,33 @@ initialize_native_environment() {
fi
}
check_dpx_gpu_exclusivity() {
local devices=(/dev/dri/renderD*)
local device_id lock_status
if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then
echo "DPX guard requires exactly one render device; refusing to start tests." >&2
return 1
fi
# This queue mounts the same host-local HF cache into every pod on the node.
if ! mountpoint -q "${HF_HOME}"; then
echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2
return 1
fi
device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1
mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1
# Hold the descriptor through workload and teardown; never delete the file.
exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1
flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0
lock_status=$?
if [[ "${lock_status}" -eq 75 ]]; then
echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2
else
echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2
fi
return 1
}
run_native_preflight() {
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
@@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() {
# Bus data identifies a card within the public node without publishing its
# persistent UUID, serial number, or process list.
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi static -b -g all
amd-smi static --bus --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -e -k -P -x -g all
amd-smi metric --ecc --ecc-blocks --pcie --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi bad-pages -p -r -u -g all
amd-smi static --ras --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -p -t -u -m -v -g all
amd-smi metric --power --temperature --usage --mem-usage --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi xgmi -l -g all
amd-smi xgmi --link-status --gpu all
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
&& command -v rocm-smi >/dev/null 2>&1; then
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
@@ -1485,6 +1512,10 @@ if is_native_runtime; then
echo "Failed to initialize the native test environment"
exit 1
fi
if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \
&& "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then
check_dpx_gpu_exclusivity || exit 1
fi
if [[ "${commands}" == *python_only_compile.sh* ]]; then
# This no-GPU job validates the ROCm precompiled/editable install path,
# rather than CPU runtime platform selection.
@@ -5,7 +5,7 @@
set -ex
# Skip the new torch installation during build since we are using the specified version for arm64 in the Dockerfile
python3 use_existing_torch.py
python3 tools/use_existing_torch.py
# Try building the docker image
DOCKER_BUILDKIT=1 docker build . \
@@ -17,7 +17,7 @@ case "${test_suite}" in
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
@@ -30,8 +30,8 @@ case "${test_suite}" in
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
;;
v1)
cd tests
@@ -43,7 +43,7 @@ case "${test_suite}" in
pytest -v -s v1/structured_output
pytest -v -s v1/test_serial_utils.py
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
;;
server)
@@ -65,8 +65,6 @@ case "${test_suite}" in
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8
;;
graph)
export VLLM_XPU_ENABLE_XPU_GRAPH=1
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B
python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096
+1453 -937
View File
File diff suppressed because it is too large Load Diff
+76 -11
View File
@@ -1,27 +1,92 @@
group: Basic Correctness
depends_on:
depends_on:
- image-build
steps:
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness"
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Models %N"
key: basic-correctness
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies:
parallelism: 2
source_file_dependencies: &basic-correctness-deps
- vllm/
- "!vllm/distributed/kv_transfer/"
- tests/basic_correctness/test_basic_correctness
- tests/basic_correctness/test_cpu_offload
- tests/basic_correctness/test_mem.py
- tests/basic_correctness/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
- pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI300) Basic Correctness"
label: ":amd: (MI355 DPX) Basic Correctness Models %N"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CuMem"
key: basic-correctness-cumem
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/memory/cumem
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness CuMem"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Sleep Mode"
key: basic-correctness-sleep-mode
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/memory/sleep_mode
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CPU Offload"
key: basic-correctness-cpu-offload
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/cpu_offload
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness CPU Offload"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Prefetch Offload"
key: basic-correctness-prefetch-offload
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/prefetch_offload
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
+17 -2
View File
@@ -15,9 +15,9 @@ steps:
- pytest -v -s benchmarks/
mirror:
amd:
label: ":amd: (MI300) Benchmarks CLI"
label: ":amd: (MI355 DPX) Benchmarks CLI"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 40
depends_on:
- image-build-amd
@@ -34,3 +34,18 @@ steps:
- vllm/v1/attention/
commands:
- python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
mirror:
amd:
label: ":amd: (MI355) Attention Benchmark Smoke"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- benchmarks/attention_benchmarks/
- vllm/v1/attention/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k"
+97 -4
View File
@@ -67,6 +67,31 @@ steps:
- pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py
# this runner has 2 GPUs available even though num_devices=2 is not set
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
mirror:
amd:
label: ":amd: (MI355) Fusion and Compile FP8"
dind: false
device: mi355_2
num_devices: 2
working_dir: "/vllm-workspace/"
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers/
- vllm/v1/attention/
- vllm/distributed/
- vllm/_aiter_ops.py
- tests/compile/passes/
- tests/compile/correctness_e2e/test_attn_quant.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/models/llama.py
commands:
- amd-smi
# MLA static FP8 and distributed AR/RMS are owned by the existing ROCm jobs.
- pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
- pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
- label: ":nvidia: (H100) Fusion E2E Quick"
key: fusion-e2e-quick-h100
@@ -88,9 +113,9 @@ steps:
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E Quick"
label: ":amd: (MI355 DPX) Fusion E2E Quick"
dind: false
device: mi355_1
device: mi355_dpx
working_dir: "/vllm-workspace/"
timeout_in_minutes: 30
depends_on:
@@ -129,9 +154,9 @@ steps:
- pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E Config Sweep"
label: ":amd: (MI355 DPX) Fusion E2E Config Sweep"
dind: false
device: mi355_1
device: mi355_dpx
working_dir: "/vllm-workspace/"
timeout_in_minutes: 40
depends_on:
@@ -181,6 +206,47 @@ steps:
# Run all models and attn backends but only Inductor partition and native custom ops
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
- pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))"
mirror:
amd:
label: ":amd: (MI355) Fusion E2E TP2 Quick"
dind: false
device: mi355_2
timeout_in_minutes: 60
depends_on:
- image-build-amd
working_dir: /vllm-workspace/
source_file_dependencies:
- csrc/quantization/
- vllm/compilation/
- vllm/distributed/
- vllm/model_executor/
- vllm/v1/attention/
- tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and Llama-3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
+quant_fp8 and +rms_norm and qwen3 and
ROCM_AITER_UNIFIED_ATTN'
- >-
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_USE_AITER_CUSTOM_AR=1
pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py
-k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and
not +quant_fp8 and not +rms_norm and DeepSeek-Coder and
ROCM_AITER_MLA'
- label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep"
key: fusion-e2e-tp2-ar-rms-config-sweep-h100
@@ -221,6 +287,33 @@ steps:
- nvidia-smi
# Run just llama3 (fp8 & bf16) for all config combinations
- pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "llama-3"
mirror:
amd:
label: ':amd: (MI355) Fusion E2E TP2 AsyncTP BF16 Config Sweep'
dind: false
device: mi355_2
num_devices: 2
working_dir: /vllm-workspace/
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers/linear.py
- vllm/model_executor/layers/utils.py
- vllm/distributed/
- vllm/v1/attention/backends/triton_attn.py
- tests/compile/fusions_e2e/
- tests/compile/passes/distributed/test_async_tp.py
- tests/compile/rocm/test_async_tp.py
- tests/compile/correctness_e2e/test_async_tp.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/models/qwen3.py
commands:
- rocm-smi
- pytest -v -s tests/compile/rocm/test_async_tp.py
- pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py
- pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness
- label: ":nvidia: (B200) Fusion E2E TP2"
key: fusion-e2e-tp2-b200
+19 -1
View File
@@ -1,13 +1,31 @@
group: CRCR Report
# The report waits for the rest of the build (see crcr-report.sh), so gate it on
# the long pole rather than letting it wait on an agent. A torch-nightly build
# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
#
# Depending on it means Buildkite does not schedule this step until the build is
# nearly over, so the agent polls briefly instead of idling ~13h. It is a
# scheduling hint, not the correctness mechanism: the script still waits for
# every job, so if this stops being the slowest step the report is late to start
# but still correct.
depends_on:
- image-build-cpu
- lm-eval-large-models-4xh100
steps:
- label: ":satellite: Report nightly results to CRCR"
key: crcr-report
if: build.env("TORCH_NIGHTLY") == "1"
soft_fail: true
allow_dependency_failure: true
timeout_in_minutes: 15
# depends_on defers the start to near the end of the build, so in the normal
# case the script polls for minutes. This only has to cover the case where
# the long pole finishes early and something else is still running, so it is
# sized against the whole build rather than the expected wait. The script's
# own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
# report; this is the backstop that stops the job outliving the build.
timeout_in_minutes: 840
device: cpu-small
commands:
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh
+2 -2
View File
@@ -19,9 +19,9 @@ steps:
- pytest -v -s cuda/test_cuda_compatibility_path.py
mirror:
amd:
label: ":amd: (MI355) CUDA Platform"
label: ":amd: (MI355 DPX) CUDA Platform"
dind: false
device: mi355_1
device: mi355_dpx
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 25
depends_on:
+81 -1
View File
@@ -42,6 +42,29 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Preserve all six model/TP pairs with native AITER attention.
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy"
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
@@ -130,7 +153,7 @@ steps:
label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 60
timeout_in_minutes: 80
depends_on:
- image-build-amd
source_file_dependencies:
@@ -205,6 +228,25 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- requirements/kv_connectors_rocm.txt
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
@@ -295,6 +337,44 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP"
dind: false
device: mi355_8
timeout_in_minutes: 85
depends_on:
- image-build-amd
source_file_dependencies:
- requirements/kv_connectors_rocm.txt
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/models/deepseek_v4/
- vllm/v1/attention/
- tests/v1/kv_connector/nixl_integration/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# A mirror does not inherit the parent env, so it is repeated here in
# the parent's order. The layouts must stay empty: unset, the script
# defaults to HND, which the V4 indexer rejects.
- >-
VLLM_ROCM_USE_AITER=1
ENABLE_HMA_FLAG=1
DP_EP=1
GPU_MEMORY_UTILIZATION=0.85
PREFILLER_TP_SIZE=4
DECODER_TP_SIZE=4
PREFILL_BLOCK_SIZE=256
DECODE_BLOCK_SIZE=256
MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash
VLLM_ENGINE_READY_TIMEOUT_S=1800
VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8
PREFILLER_KV_LAYOUT=
DECODER_KV_LAYOUT=
bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
- label: ":nvidia: (H200) Kimi-Linear-48B-A3B Disaggregated DP EP"
key: kimi-linear-disaggregated
@@ -19,6 +19,30 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Same full accuracy/TP sweep, using same-host HIP GPU transport.
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
key: mooncake-ec-tcp-e2e-2-gpus
+67 -12
View File
@@ -35,9 +35,9 @@ steps:
- pytest -v -s distributed/test_sharded_rdt_trainer.py
mirror:
amd:
label: ":amd: (MI355) Sharded RDT Weight Transfer"
label: ":amd: (MI355 DPX) Sharded RDT Weight Transfer"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
@@ -67,7 +67,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
@@ -98,7 +98,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
@@ -117,7 +117,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
@@ -141,7 +141,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
@@ -162,7 +162,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
@@ -219,6 +219,41 @@ steps:
# default does not fit an L4.
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
mirror:
amd:
label: ":amd: (MI355) Distributed Torchrun + Examples"
dind: false
device: mi355_4
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- tests/distributed/test_torchrun_example.py
- tests/distributed/test_torchrun_example_moe.py
- examples/rl/
- examples/features/data_parallel/data_parallel_offline.py
- .buildkite/scripts/install-kv-connectors.sh
- requirements/kv_connectors_rocm.txt
- requirements/test/rocm.in
- requirements/test/rocm.txt
- vllm/platforms/rocm.py
commands:
# Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
- uv pip install --system --no-deps 'ray==2.56.1'
- export NCCL_CUMEM_HOST_ENABLE=0
- torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
- label: ":nvidia: (L4) Distributed DP Extended"
key: distributed-dp-tests-4-gpus
@@ -291,6 +326,23 @@ steps:
- export NCCL_CUMEM_HOST_ENABLE=0
# test with torchrun tp=2 and dp=4 with ep
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
mirror:
amd:
label: ":amd: (MI355) Distributed DP + EP"
dind: false
device: mi355_8
timeout_in_minutes: 50
depends_on:
- image-build-amd
source_file_dependencies:
- examples/features/torchrun/torchrun_dp_example_offline.py
- vllm/config/parallel.py
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/v1/engine/llm_engine.py
- vllm/v1/executor/uniproc_executor.py
- vllm/v1/worker/gpu_worker.py
- vllm/platforms/rocm.py
- label: ":nvidia: (A100) Distributed"
key: distributed-tests-4xa100
@@ -321,6 +373,7 @@ steps:
- VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- pytest -v -s tests/v1/distributed/test_dbo.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer_m2n.py
- pytest -v -s tests/distributed/test_packed_tensor.py
mirror:
amd:
@@ -412,7 +465,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/model_executor/models/
- tests/distributed/
- examples/features/data_parallel/data_parallel_offline.py
@@ -429,8 +482,10 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/model_executor/models/
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- tests/distributed/
- tests/v1/distributed/test_pp_dp_v2.py
commands:
@@ -449,7 +504,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/model_executor/models/
- vllm/model_executor/layers/
- vllm/v1/attention/backends/
@@ -471,11 +526,11 @@ steps:
- tests/distributed/test_ray_v2_executor.py
- tests/distributed/test_ray_v2_executor_e2e.py
- tests/distributed/test_pipeline_parallel.py
- tests/basic_correctness/test_basic_correctness.py
- tests/basic_correctness/models/test_basic_correctness.py
commands:
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
- export NCCL_CUMEM_HOST_ENABLE=0
- pytest -v -s distributed/test_ray_v2_executor.py
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
+81 -2
View File
@@ -11,6 +11,28 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
mirror:
amd:
label: ":amd: (MI355) DeepSeek V2-Lite Sync EPLB Accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 40
env:
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/backends/mla/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
@@ -21,6 +43,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP2"
dind: false
device: mi355_4
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy"
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
@@ -31,6 +72,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP1"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/distributed/eplb
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
@@ -41,6 +101,25 @@ steps:
working_dir: "/vllm-workspace"
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
mirror:
amd:
label: ":amd: (MI355) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/distributed/eplb
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy"
key: deepseek-v2-lite-prefetch-offload-accuracy-h100
@@ -53,9 +132,9 @@ steps:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
mirror:
amd:
label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy"
label: ":amd: (MI355 DPX) DeepSeek V2-Lite Prefetch Offload Accuracy"
dind: false
device: mi355_1
device: mi355_dpx
working_dir: "/vllm-workspace"
timeout_in_minutes: 30
depends_on:
+4 -4
View File
@@ -29,9 +29,9 @@ steps:
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py
mirror:
amd:
label: ":amd: (MI300) Engine Core"
label: ":amd: (MI355 DPX) Engine Core"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 40
depends_on:
- image-build-amd
@@ -117,9 +117,9 @@ steps:
- pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async
mirror:
amd:
label: ":amd: (MI355) E2E Core Large Memory"
label: ":amd: (MI355 DPX) E2E Core Large Memory"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
+23 -22
View File
@@ -12,6 +12,7 @@ steps:
working_dir: "."
env:
SNAPSHOT_E2E_RUN_LIMIT_S: "1200"
NVIDIA_VISIBLE_DEVICES: "0"
source_file_dependencies:
- .buildkite/scripts/initialized-snapshot-e2e.sh
- .buildkite/test_areas/entrypoints.yaml
@@ -38,9 +39,9 @@ steps:
- pytest -v -s entrypoints/launchers
mirror:
amd:
label: ":amd: (MI355) Entrypoints Unit"
label: ":amd: (MI355 DPX) Entrypoints Unit"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 50
depends_on:
- image-build-amd
@@ -63,9 +64,9 @@ steps:
- pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
mirror:
amd:
label: ":amd: (MI300) Entrypoints Integration (LLM)"
label: ":amd: (MI355 DPX) Entrypoints Integration (LLM)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 90
depends_on:
- image-build-amd
@@ -88,9 +89,9 @@ steps:
- pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI300) Entrypoints Integration (API Server) %N"
label: ":amd: (MI355 DPX) Entrypoints Integration (API Server) %N"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -110,9 +111,9 @@ steps:
- pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
mirror:
amd:
label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)"
label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -131,9 +132,9 @@ steps:
- pytest -v -s entrypoints/openai/chat_completion
mirror:
amd:
label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)"
label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API chat_completion)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 70
depends_on:
- image-build-amd
@@ -159,9 +160,9 @@ steps:
- pytest -v -s entrypoints/cohere
mirror:
amd:
label: ":amd: (MI300) Entrypoints Integration (API Server Generate)"
label: ":amd: (MI355 DPX) Entrypoints Integration (API Server Generate)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -179,9 +180,9 @@ steps:
- pytest -v -s entrypoints/openai/responses
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Responses API)"
label: ":amd: (MI355 DPX) Entrypoints Integration (Responses API)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 50
depends_on:
- image-build-amd
@@ -200,9 +201,9 @@ steps:
- pytest -v -s entrypoints/speech_to_text
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Speech to Text)"
label: ":amd: (MI355 DPX) Entrypoints Integration (Speech to Text)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -222,9 +223,9 @@ steps:
- pytest -v -s entrypoints/multimodal
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Multimodal)"
label: ":amd: (MI355 DPX) Entrypoints Integration (Multimodal)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 55
depends_on:
- image-build-amd
@@ -243,9 +244,9 @@ steps:
- pytest -v -s entrypoints/pooling
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Pooling)"
label: ":amd: (MI355 DPX) Entrypoints Integration (Pooling)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 65
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -262,9 +263,9 @@ steps:
- pytest -s entrypoints/openai/correctness/
mirror:
amd:
label: ":amd: (MI300) OpenAI API Correctness"
label: ":amd: (MI355 DPX) OpenAI API Correctness"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 30
depends_on:
- image-build-amd
+16 -3
View File
@@ -16,9 +16,9 @@ steps:
- pytest -v -s distributed/test_eplb_utils.py
mirror:
amd:
label: ":amd: (MI300) EPLB Algorithm"
label: ":amd: (MI355 DPX) EPLB Algorithm"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 30
depends_on:
- image-build-amd
@@ -40,6 +40,19 @@ steps:
commands:
- pytest -v -s distributed/test_eplb_execute.py
- pytest -v -s distributed/test_eplb_spec_decode.py
mirror:
amd:
label: ":amd: (MI355) EPLB Execution"
dind: false
device: mi355_4
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/eplb
- tests/distributed/test_eplb_execute.py
- tests/distributed/test_eplb_spec_decode.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) Elastic EP Scaling"
key: elastic-ep-scaling-test
@@ -51,7 +64,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/compilation/
- tests/distributed/
commands:
@@ -24,3 +24,33 @@ steps:
# https://github.com/NVIDIA/nccl/issues/1838
- export NCCL_CUMEM_HOST_ENABLE=0
- pytest -v -s v1/fault_tolerance/test_fault_tolerance_e2e.py
mirror:
amd:
label: ":amd: (MI355) MoRI EP Healthy Baseline"
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/utils/import_utils.py
- vllm/utils/math_utils.py
- vllm/utils/torch_utils.py
- vllm/platforms/
- vllm/_aiter_ops.py
# TODO: Add fault injection, detection and recovery once MoRI supports them.
# This mirror currently checks healthy EP execution only.
commands:
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph
- VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]'
+21
View File
@@ -20,3 +20,24 @@ steps:
# instead of running until the build timeout.
- export PYTHONFAULTHANDLER=1
- pytest -v -s jit_monitor/test_no_runtime_jit.py --timeout=900 --timeout-method=thread
mirror:
amd:
# Focused MRV2 regression; the broader model/backend matrix is pending.
label: ':amd: (MI355) MRV2 Sampler JIT Warmup'
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 10
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- tests/jit_monitor/
- tests/models/utils.py
- tests/utils.py
- vllm/v1/worker/gpu/
- vllm/v1/sample/ops/topk_topp_sampler.py
- vllm/platforms/rocm.py
- vllm/envs.py
commands:
- pytest -v -s tests/jit_monitor/test_no_runtime_jit_rocm.py --timeout=300
+167 -20
View File
@@ -15,9 +15,9 @@ steps:
- pytest -v -s tests/kernels/ir
mirror:
amd:
label: ":amd: (MI355) vLLM IR"
label: ":amd: (MI355 DPX) vLLM IR"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
depends_on:
@@ -44,9 +44,9 @@ steps:
parallelism: 3
mirror:
amd:
label: ":amd: (MI355) Core Operation Kernels Shard %N"
label: ":amd: (MI300) Core Operation Kernels Shard %N"
dind: false
device: mi355_1
device: mi300_1
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -113,9 +113,9 @@ steps:
- pytest -v -s kernels/test_top_k_per_row.py
mirror:
amd:
label: ":amd: (MI355) DeepSeek V4 Fused Kernels"
label: ":amd: (MI355 DPX) DeepSeek V4 Fused Kernels"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 60
working_dir: /vllm-workspace/tests
@@ -144,6 +144,32 @@ steps:
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
- pytest -v -s models/test_deepseek_v4_mega_moe.py
# Own job so mHC never shares a process with FlashInfer's import-time
# ptxas override.
- label: ":nvidia: (B200) mHC Kernels"
key: kernels-mhc-test-b200
timeout_in_minutes: 25
device: b200-k8s
source_file_dependencies:
- csrc/
- vllm/
- "!vllm/distributed/kv_transfer/"
- tests/kernels/
commands:
- pytest -v -s kernels/mhc
- label: ":nvidia: (B200) FLA Ops Kernels"
key: kernels-fla-ops-test-b200
timeout_in_minutes: 25
device: b200-k8s
source_file_dependencies:
- csrc/
- vllm/
- "!vllm/distributed/kv_transfer/"
- tests/kernels/
commands:
- pytest -v -s kernels/fla
# Catch-all for test files at the tests/kernels root. This job collects
# the whole root so new files are wired by default.
# Files with dedicated jobs elsewhere in this file are excluded via --ignore
@@ -151,7 +177,7 @@ steps:
# their own jobs / Kernels (B200)).
- label: ":nvidia: (B200) Miscellaneous Kernels"
key: kernels-root-misc-test-b200
timeout_in_minutes: 45
timeout_in_minutes: 30
device: b200-k8s
source_file_dependencies:
- csrc/
@@ -159,14 +185,14 @@ steps:
- "!vllm/distributed/kv_transfer/"
- tests/kernels/
commands:
# Isolate mHC from FlashInfer's import-time ptxas override.
- pytest -v -s kernels/test_mhc_kernels.py
- pytest -v -s kernels/
--ignore=kernels/attention
--ignore=kernels/core
--ignore=kernels/fla
--ignore=kernels/helion
--ignore=kernels/ir
--ignore=kernels/mamba
--ignore=kernels/mhc
--ignore=kernels/moe
--ignore=kernels/quantization
--ignore=kernels/test_concat_mla_q.py
@@ -176,7 +202,6 @@ steps:
--ignore=kernels/test_kda.py
--ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
--ignore=kernels/test_ll_bf16_gemm.py
--ignore=kernels/test_mhc_kernels.py
--ignore=kernels/test_shuffle_rows.py
# BROKEN on main, pending kernel fixes (B200):
# test_shuffle_rows.py (1: test_shuffle_rows_edge_cases)
@@ -230,9 +255,9 @@ steps:
- pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
mirror:
amd:
label: ":amd: (MI355) Attention DiffKV Kernels"
label: ":amd: (MI355 DPX) Attention DiffKV Kernels"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 35
working_dir: /vllm-workspace/tests
@@ -270,9 +295,9 @@ steps:
- pytest -v -s kernels/attention/test_mla_cross_layer_kernel_equivalence.py
mirror:
amd:
label: ":amd: (MI355) MLA Kernels"
label: ":amd: (MI355 DPX) MLA Kernels"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/
@@ -330,6 +355,8 @@ steps:
- vllm/envs.py
depends_on:
- image-build-amd
commands:
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (L4) MoE Kernels Shard %N"
key: kernels-moe-test
@@ -392,6 +419,7 @@ steps:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- vllm/third_party/flash_linear_attention/
- label: ":nvidia: (H100) DeepGEMM Kernels"
key: kernels-deepgemm-test-h100
@@ -522,6 +550,32 @@ steps:
exit 1
;;
esac
mirror:
amd:
label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/kernels/quantization/test_nvfp4_emulation.py
- tests/kernels/quantization/nvfp4_utils.py
- tests/conftest.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
- vllm/config/kernel.py
- vllm/platforms/
- vllm/envs.py
- vllm/utils/torch_utils.py
- requirements/rocm-test.txt
# W4A4 format numerics and replay via emulation. TODO: native packed
# activation/KV-cache layouts and CUDA-only kernels need AMD backends.
commands:
- amd-smi
- pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly"
key: b12x-linear-kernels-dgx-spark-nightly
@@ -572,18 +626,20 @@ steps:
commands:
- pip install helion==1.4.0
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
parallelism: 5
mirror:
amd:
label: ":amd: (MI250) Helion Kernels Shard %N"
dind: false
device: mi250_1
timeout_in_minutes: 45
# Explicit so the base step's count does not multiply the mirror's replicas.
parallelism: 2
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- scripts/benchmark_helion_kernels.py
- tools/benchmark_helion_kernels.py
- label: ":nvidia: (H100) FP8 MoE Kernels"
@@ -684,6 +740,9 @@ steps:
- vllm/utils/torch_utils.py
- vllm/platforms/
- vllm/_aiter_ops.py
commands:
# MoRI graph cases run in the Fault Tolerance group's healthy EP baseline.
- pytest -v -s kernels/moe/test_moe_layer.py -k "not test_moe_layer_mori_graph"
- label: ":nvidia: (B200) FusedMoE Layer Kernels"
key: kernels-fusedmoe-layer-test-2-b200s
@@ -701,11 +760,99 @@ steps:
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
- pytest -v -s kernels/moe/test_deepep_v2_moe.py
mirror:
amd:
label: ":amd: (MI355) MoRI EP Numerics"
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/kernels/moe/test_mori_moe.py
- tests/kernels/moe/utils.py
- tests/kernels/moe/modular_kernel_tools/parallel_utils.py
- tests/kernels/utils.py
- tests/utils.py
- tests/conftest.py
- vllm/utils/torch_utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/platforms/
- vllm/_aiter_ops.py
- requirements/rocm-test.txt
# Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the
# DeepEP v2 FlashInfer backend matrix has no direct AMD implementation.
commands:
- amd-smi
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py
- label: ":amd: (MI355) FP8 MoE Kernels"
- label: ":amd: (MI355 DPX) Native Quantization Kernels"
key: amd-native-quantization-kernels-mi355
dind: false
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/rocm/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/kernels/linear/
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
- tests/kernels/quantization/test_rocm_mxfp4.py
- tests/kernels/quantization/test_rocm_mxfp8_linear.py
- tests/kernels/quantization/test_rocm_skinny_gemms.py
- tests/quantization/reference_mxfp4.py
- tests/kernels/quant_utils.py
- tests/kernels/utils.py
commands:
- pytest -v -s kernels/quantization/test_rocm_mxfp4.py
- pytest -v -s kernels/quantization/test_rocm_mxfp8_linear.py
- pytest -v -s kernels/quantization/test_rocm_skinny_gemms.py -k wvsplitkrc
- label: ":amd: (MI355 DPX) FP4 and INT4 MoE Kernels"
key: amd-fp4-moe-kernels-mi355
dind: false
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- csrc/rocm/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
- vllm/config/
- tests/kernels/moe/test_rocm_aiter_moe.py
- tests/kernels/moe/test_ocp_mx_moe.py
- tests/kernels/moe/test_flydsl_moe.py
- tests/kernels/moe/utils.py
- tests/kernels/utils.py
commands:
- pytest -v -s kernels/moe/test_rocm_aiter_moe.py -k 'mi350_mxfp4 or quant_scheme_support_matches_gfx950'
- pytest -v -s kernels/moe/test_ocp_mx_moe.py::test_rocm_mxfp4_moe_oracle -k AITER_MXFP4
- pytest -v -s kernels/moe/test_flydsl_moe.py
- label: ":amd: (MI355 DPX) FP8 MoE Kernels"
key: amd-fp8-moe-kernels-mi355
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
@@ -732,10 +879,10 @@ steps:
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
- label: ":amd: (MI355) Kernels"
- label: ":amd: (MI355 DPX) Kernels"
key: amd-kernels-mi355
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 25
working_dir: /vllm-workspace/
+229 -10
View File
@@ -68,9 +68,9 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval Watermarking"
label: ":amd: (MI355 DPX) LM Eval Watermarking"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
@@ -114,6 +114,27 @@ steps:
commands:
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
mirror:
amd:
label: ":amd: (MI355) LM Eval Large Models FP8"
dind: false
device: mi355_4
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- .buildkite/lm-eval-harness/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_USE_DEEP_GEMM=0
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
- label: ":nvidia: (B200) LM Eval Small Models"
key: lm-eval-small-models-1xb200
@@ -125,6 +146,28 @@ steps:
- vllm/model_executor/layers/quantization
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
mirror:
amd:
label: ":amd: (MI355 DPX) LM Eval Small Models"
dind: false
device: mi355_dpx
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- tests/evals/gsm8k/
- vllm/model_executor/layers/quantization
- vllm/model_executor/kernels/linear/
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-small.txt
- label: ":nvidia: (B200) LM Eval Small Models Distributed"
key: lm-eval-small-models-distributed-2xb200
@@ -286,6 +329,7 @@ steps:
- csrc/
- tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml
- tests/evals/gsm8k/configs/models-spec-decode-rocm.txt
- tests/evals/gsm8k/test_gsm8k_correctness.py
- tests/evals/gsm8k/gsm8k_eval.py
@@ -342,6 +386,33 @@ steps:
- vllm/third_party/flash_linear_attention/ops/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval Qwen3.5 Models"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/qwen3_5.py
- vllm/model_executor/models/qwen3_5_mtp.py
- vllm/transformers_utils/configs/qwen3_5.py
- vllm/transformers_utils/configs/qwen3_5_moe.py
- vllm/model_executor/models/qwen2.py
- vllm/model_executor/models/qwen3.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/qwen3_next_mtp.py
- vllm/third_party/flash_linear_attention/ops/
- tests/evals/gsm8k/configs/models-qwen35-mi355.txt
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
- tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
- tests/evals/gsm8k/test_gsm8k_correctness.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
- label: ":nvidia: (H200) LM Eval DeepSeek R1 TP"
key: lm-eval-deepseek-r1-tp-8xh200
@@ -523,7 +594,7 @@ steps:
- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY"
key: lm-eval-humming-h100-act
optional: true
timeout_in_minutes: 70
timeout_in_minutes: 90
device: h100
num_devices: 1
source_file_dependencies:
@@ -534,7 +605,7 @@ steps:
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
# - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY"
key: lm-eval-humming-h100-deepep
@@ -587,9 +658,9 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant KV Cache"
key: lm-eval-turboquant-kv-cache
timeout_in_minutes: 55
- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k8v4"
key: lm-eval-turboquant-k8v4
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/model_executor/layers/quantization/turboquant/
@@ -597,7 +668,155 @@ steps:
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval TurboQuant k8v4"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/ops/triton_unified_attention.py
- vllm/v1/attention/selector.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
- vllm/v1/attention/ops/flydsl_turboquant_decode.py
- vllm/v1/attention/ops/flydsl_kernels/
- vllm/v1/attention/ops/turboquant_soa/
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt
- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t4nc"
key: lm-eval-turboquant-t4nc
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval TurboQuant t4nc"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/ops/triton_unified_attention.py
- vllm/v1/attention/selector.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
- vllm/v1/attention/ops/flydsl_turboquant_decode.py
- vllm/v1/attention/ops/flydsl_kernels/
- vllm/v1/attention/ops/turboquant_soa/
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt
- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k3v4nc"
key: lm-eval-turboquant-k3v4nc
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval TurboQuant k3v4nc"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/ops/triton_unified_attention.py
- vllm/v1/attention/selector.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
- vllm/v1/attention/ops/flydsl_turboquant_decode.py
- vllm/v1/attention/ops/flydsl_kernels/
- vllm/v1/attention/ops/turboquant_soa/
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt
- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t3nc"
key: lm-eval-turboquant-t3nc
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval TurboQuant t3nc"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/model_executor/layers/quantization/turboquant/
- vllm/v1/attention/backends/turboquant_attn.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/ops/triton_unified_attention.py
- vllm/v1/attention/selector.py
- vllm/v1/attention/ops/triton_turboquant_decode.py
- vllm/v1/attention/ops/triton_turboquant_store.py
- vllm/v1/attention/ops/flydsl_turboquant_decode.py
- vllm/v1/attention/ops/flydsl_kernels/
- vllm/v1/attention/ops/turboquant_soa/
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt
- label: ":nvidia: (H100) GPQA Eval (GPT-OSS)"
key: gpqa-eval-gpt-oss-2xh100
@@ -775,9 +994,9 @@ steps:
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
mirror:
amd:
label: ":amd: (MI355) MRCR Eval Small Models"
label: ":amd: (MI355 DPX) MRCR Eval Small Models"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 40
depends_on:
- image-build-amd
+2 -2
View File
@@ -14,9 +14,9 @@ steps:
parallelism: 4
mirror:
amd:
label: ":amd: (MI300) LoRA Shard %N"
label: ":amd: (MI355 DPX) LoRA Shard %N"
dind: false
device: mi300_1
device: mi355_dpx
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 85
source_file_dependencies:
+31 -24
View File
@@ -29,9 +29,9 @@ steps:
- pytest -v -s v1/sample
mirror:
amd:
label: ":amd: (MI300) V1 Sample"
label: ":amd: (MI355 DPX) V1 Sample"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 70
depends_on:
- image-build-amd
@@ -51,9 +51,9 @@ steps:
- pytest -v -s v1/test_outputs.py
mirror:
amd:
label: ":amd: (MI300) V1 Logits + Oracle"
label: ":amd: (MI355 DPX) V1 Logits + Oracle"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 70
depends_on:
- image-build-amd
@@ -96,9 +96,9 @@ steps:
- pytest -v -s -m 'not cpu_test' v1/core
mirror:
amd:
label: ":amd: (MI300) V1 Core"
label: ":amd: (MI355 DPX) V1 Core"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -116,9 +116,9 @@ steps:
- pytest -v -s v1/streaming_input
mirror:
amd:
label: ":amd: (MI300) V1 Executor + Worker"
label: ":amd: (MI355 DPX) V1 Executor + Worker"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -135,9 +135,9 @@ steps:
- pytest -v -s v1/simple_kv_offload
mirror:
amd:
label: ":amd: (MI300) V1 KV Offload"
label: ":amd: (MI355 DPX) V1 KV Offload"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -155,9 +155,9 @@ steps:
- pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI300) V1 KV Connectors Shard %N"
label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -182,9 +182,9 @@ steps:
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
mirror:
amd:
label: ":amd: (MI300) V1 Metrics + LM Eval"
label: ":amd: (MI355 DPX) V1 Metrics + LM Eval"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -211,6 +211,7 @@ steps:
- vllm/utils/
- vllm/v1/
- tests/v1
- tests/watermarking
device: cpu-small
commands:
# split the test to avoid interference
@@ -222,6 +223,7 @@ steps:
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
- pytest -v -s -m 'cpu_test' v1/ec_connector/unit
- pytest -v -s -m 'cpu_test' v1/metrics
- pytest -v -s watermarking
- label: ":nvidia: (H200 MIG 18GB) Extract Hidden States Integration"
key: extract-hidden-states-integration
@@ -239,9 +241,9 @@ steps:
- pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration
mirror:
amd:
label: ":amd: (MI355) Extract Hidden States Integration Single GPU"
label: ":amd: (MI355 DPX) Extract Hidden States Integration Single GPU"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 45
working_dir: /vllm-workspace
@@ -328,9 +330,9 @@ steps:
working_dir: "/vllm-workspace/tests" # optional
mirror:
amd:
label: ":amd: (MI355) Regression"
label: ":amd: (MI355 DPX) Regression"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 25
depends_on:
- image-build-amd
@@ -370,9 +372,9 @@ steps:
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
mirror:
amd:
label: ":amd: (MI300) Examples"
label: ":amd: (MI355 DPX) Examples"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 75
source_file_dependencies:
- vllm/entrypoints
@@ -472,9 +474,9 @@ steps:
- pytest -v -s utils_
mirror:
amd:
label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker"
label: ":amd: (MI355 DPX) Async Engine, Inputs, Utils, Worker"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 50
depends_on:
- image-build-amd
@@ -493,6 +495,7 @@ steps:
source_file_dependencies: &cpu-async-engine-deps
- vllm/assets/
- vllm/config/
- vllm/device_allocator/
- vllm/engine/arg_utils.py
- vllm/entrypoints/chat_utils.py
- vllm/entrypoints/mcp/
@@ -518,6 +521,7 @@ steps:
- vllm/v1/
- tests/test_envs.py
- tests/test_outputs.py
- tests/test_pcp_dp.py
- tests/test_pooling_params.py
- tests/test_ray_env.py
- tests/test_sampling_params.py
@@ -533,11 +537,14 @@ steps:
- tests/parser
- tests/transformers_utils
- tests/config
- tests/device_allocator
device: cpu-small
commands:
- python3 standalone_tests/lazy_imports.py
- pytest -v -s device_allocator
- pytest -v -s test_envs.py
- pytest -v -s test_outputs.py
- pytest -v -s test_pcp_dp.py
- pytest -v -s test_pooling_params.py
- pytest -v -s test_ray_env.py
- pytest -v -s test_sampling_params.py
@@ -665,9 +672,9 @@ steps:
- pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
mirror:
amd:
label: ":amd: (MI355) Acceptance Length (Large Models)"
label: ":amd: (MI355 DPX) Acceptance Length (Large Models)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 45
depends_on:
- image-build-amd
+6 -1
View File
@@ -33,7 +33,7 @@ steps:
label: ":amd: (MI300) Model Executor"
dind: false
device: mi300_1
timeout_in_minutes: 60
timeout_in_minutes: 75
depends_on:
- image-build-amd
source_file_dependencies:
@@ -48,3 +48,8 @@ steps:
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export PYTHONFAULTHANDLER=1
- pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
+29 -10
View File
@@ -16,9 +16,9 @@ steps:
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
mirror:
amd:
label: ":amd: (MI355) Basic Models (Initialization)"
label: ":amd: (MI355 DPX) Basic Models (Initialization)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 30
depends_on:
- image-build-amd
@@ -39,9 +39,9 @@ steps:
parallelism: 14
mirror:
amd:
label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N"
label: ":amd: (MI355 DPX) Basic Models (Extra Initialization) Shard %N"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 60
parallelism: 10
working_dir: "/vllm-workspace/tests"
@@ -70,9 +70,9 @@ steps:
- pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py
mirror:
amd:
label: ":amd: (MI300) Basic Models (Other)"
label: ":amd: (MI355 DPX) Basic Models (Other)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -109,6 +109,23 @@ steps:
commands:
# The native NVIDIA Kimi K3 kernels require the SM100 family.
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
mirror:
amd:
label: ":amd: (MI355) Kimi K3"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
commands:
# The ROCm KDA/AttnRes kernels are gated on gfx950, so this has to be MI355.
# The two kernels/ files the NVIDIA job adds are CUDA-only (they import
# cuda.bindings.driver), so they are deliberately left out here.
- pytest -v -s models/kimi_k3
- label: ":nvidia: (H200) GLM5Next"
key: glm5next-unit-tests
@@ -121,9 +138,9 @@ steps:
- pytest -v -s models/glm5next
mirror:
amd:
label: ":amd: (MI355) GLM5Next"
label: ":amd: (MI355 DPX) GLM5Next"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 35
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -145,13 +162,15 @@ steps:
- pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py
mirror:
amd:
label: ":amd: (MI300) Qwen4 Exp"
label: ":amd: (MI355 DPX) Qwen4 Exp"
dind: false
device: mi300_1
device: mi355_dpx
depends_on:
- image-build-amd
commands:
- pytest -v -s models/qwen4_exp/test_qsa_amd.py
# PLE and QSA import conflicting HC registrations in a shared process.
- pytest -v -s models/qwen4_exp/test_ple.py
- label: ":computer: (CPU) Qwen4 Exp"
key: qwen4-exp-unit-tests-cpu
+10 -10
View File
@@ -16,9 +16,9 @@ steps:
- pytest -v -s models/language -m 'core_model and (not slow_test)'
mirror:
amd:
label: ":amd: (MI300) Language Models (Standard)"
label: ":amd: (MI355 DPX) Language Models (Standard)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -40,9 +40,9 @@ steps:
parallelism: 2
mirror:
amd:
label: ":amd: (MI300) Language Models (Extra Standard) Shard %N"
label: ":amd: (MI355 DPX) Language Models (Extra Standard) Shard %N"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 40
depends_on:
- image-build-amd
@@ -82,9 +82,9 @@ steps:
parallelism: 2
mirror:
amd:
label: ":amd: (MI355) Language Models (Hybrid) Shard %N"
label: ":amd: (MI355 DPX) Language Models (Hybrid) Shard %N"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -113,9 +113,9 @@ steps:
- pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'
mirror:
amd:
label: ":amd: (MI355) Granite Language Model Compatibility"
label: ":amd: (MI355 DPX) Granite Language Model Compatibility"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 80
working_dir: /vllm-workspace/tests
@@ -151,9 +151,9 @@ steps:
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
mirror:
amd:
label: ":amd: (MI355) Language Models (Extended Generation)"
label: ":amd: (MI355 DPX) Language Models (Extended Generation)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 95
depends_on:
- image-build-amd
+14 -14
View File
@@ -15,9 +15,9 @@ steps:
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
mirror:
amd:
label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2"
label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -36,9 +36,9 @@ steps:
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
mirror:
amd:
label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma"
label: ":amd: (MI355 DPX) Multimodal Models (Standard) 2: qwen3 + gemma"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 55
depends_on:
- image-build-amd
@@ -78,9 +78,9 @@ steps:
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
mirror:
amd:
label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper"
label: ":amd: (MI355 DPX) Multimodal Models (Standard) 4: other + whisper"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 70
depends_on:
- image-build-amd
@@ -125,9 +125,9 @@ steps:
- pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI355) Multimodal Processor Shard %N"
label: ":amd: (MI355 DPX) Multimodal Processor Shard %N"
dind: false
device: mi355_1
device: mi355_dpx
parallelism: 4
timeout_in_minutes: 115
depends_on:
@@ -146,9 +146,9 @@ steps:
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
mirror:
amd:
label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)"
label: ":amd: (MI355 DPX) Multimodal Accuracy Eval (Small Models)"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 35
depends_on:
- image-build-amd
@@ -212,9 +212,9 @@ steps:
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI355) Multimodal Models (Extended Generation 2) Shard %N"
label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 2) Shard %N"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 100
depends_on:
- image-build-amd
@@ -231,9 +231,9 @@ steps:
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
mirror:
amd:
label: ":amd: (MI355) Multimodal Models (Extended Generation 3)"
label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 3)"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 90
depends_on:
- image-build-amd
+2 -2
View File
@@ -78,9 +78,9 @@ steps:
- pytest -v -s plugins_tests/gguf
mirror:
amd:
label: ":amd: (MI355) GGUF Plugin"
label: ":amd: (MI355 DPX) GGUF Plugin"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
+8 -4
View File
@@ -89,9 +89,9 @@ steps:
- "find compile/h100/ -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
mirror:
amd:
label: ":amd: (MI355) PyTorch Compilation H100 Cases"
label: ":amd: (MI355 DPX) PyTorch Compilation H100 Cases"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -192,10 +192,14 @@ steps:
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
mirror:
amd:
label: ":amd: (MI355) PyTorch Fullgraph"
label: ":amd: (MI355 DPX) PyTorch Fullgraph"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 90
# Remove once the ROCm Triton pin includes triton-lang/triton#11567.
# Avoid overlapping LLVM codegen/LLD by disabling background precompilation.
env:
TORCHINDUCTOR_PRECOMPILATION_TIMEOUT_SECONDS: "0"
depends_on:
- image-build-amd
+32 -6
View File
@@ -19,12 +19,12 @@ steps:
- uv pip install --system conch-triton-kernels
# The SM90-only checkpoint currently contains a removed weight_chan_scale
# parameter. It was not exercised by the previous L4 job.
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
mirror:
amd:
label: ":amd: (MI355) Quantization Shard %N"
label: ":amd: (MI300) Quantization Shard %N"
dind: false
device: mi355_1
device: mi300_1
timeout_in_minutes: 115
working_dir: "/vllm-workspace/tests"
depends_on:
@@ -43,7 +43,7 @@ steps:
- unset VLLM_USE_V2_MODEL_RUNNER
- uv pip install --system torchao==0.17.0
- uv pip install --system conch-triton-kernels
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
- label: ":nvidia: (H200 MIG 35GB) Quantized Fusions"
@@ -76,6 +76,32 @@ steps:
- vllm/v1/attention/backends/flashinfer.py
commands:
- pytest -s -v tests/quantization/test_blackwell_moe.py
mirror:
amd:
label: ":amd: (MI355) Quantized MoE"
dind: false
device: mi355_1
timeout_in_minutes: 45
working_dir: "/vllm-workspace/"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/quantization/test_rocm_moe.py
- tests/utils.py
- vllm/model_executor/models/deepseek_v2.py
- vllm/model_executor/models/qwen3_moe.py
- vllm/model_executor/models/gpt_oss.py
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/kernels/linear/
- vllm/v1/attention/
- vllm/v1/worker/gpu/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
commands:
- pytest -v -s tests/quantization/test_rocm_moe.py
- label: ":nvidia: (H200 MIG 35GB) Quantized Models"
device: h200_35gb
@@ -90,9 +116,9 @@ steps:
- pytest -v -s models/quantization
mirror:
amd:
label: ":amd: (MI355) Quantized Models"
label: ":amd: (MI355 DPX) Quantized Models"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 80
working_dir: "/vllm-workspace/tests"
depends_on:
+2 -2
View File
@@ -18,9 +18,9 @@ steps:
- bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
mirror:
amd:
label: ":amd: (MI355) Ray Dependency Compatibility Check"
label: ":amd: (MI355 DPX) Ray Dependency Compatibility Check"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 25
working_dir: "/"
depends_on:
+33 -9
View File
@@ -43,9 +43,9 @@ steps:
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend OpenAI Coverage"
label: ":amd: (MI355 DPX) Rust Frontend OpenAI Coverage"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 35
depends_on:
- image-build-amd
@@ -98,9 +98,9 @@ steps:
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage"
label: ":amd: (MI355 DPX) Rust Frontend Serve/Admin Coverage"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 30
depends_on:
- image-build-amd
@@ -133,9 +133,9 @@ steps:
- pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Core Correctness"
label: ":amd: (MI355 DPX) Rust Frontend Core Correctness"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 25
depends_on:
- image-build-amd
@@ -163,9 +163,9 @@ steps:
- pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Tool Use"
label: ":amd: (MI355 DPX) Rust Frontend Tool Use"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 30
depends_on:
- image-build-amd
@@ -188,7 +188,7 @@ steps:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/executor/
- vllm/v1/executor/
- vllm/v1/engine/
- vllm/v1/worker/
- tests/utils.py
@@ -204,3 +204,27 @@ steps:
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
mirror:
amd:
label: ":amd: (MI355) Rust Frontend Distributed"
dind: false
device: mi355_4
timeout_in_minutes: 45
env:
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
depends_on:
- image-build-amd
source_file_dependencies:
- rust/
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/v1/executor/
- vllm/v1/engine/
- vllm/v1/worker/
- tests/utils.py
- tests/v1/distributed/test_dense_dp_world_size.py
- tests/v1/distributed/test_external_lb_dp.py
- tests/v1/distributed/test_hybrid_lb_dp.py
- tests/v1/distributed/test_internal_lb_dp.py
- vllm/platforms/rocm.py
+2 -2
View File
@@ -53,9 +53,9 @@ steps:
- VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers/test_beam_search.py::test_beam_search_passes_multimodal_data
mirror:
amd:
label: ":amd: (MI355) Samplers Multimodal Beam Search"
label: ":amd: (MI355 DPX) Samplers Multimodal Beam Search"
dind: false
device: mi355_1
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
+31 -13
View File
@@ -44,9 +44,9 @@ steps:
- pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3"
mirror:
amd:
label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen"
label: ":amd: (MI355 DPX) Spec Decode Eagle 1: DeepSeek + Qwen"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -71,9 +71,9 @@ steps:
- pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3"
mirror:
amd:
label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
label: ":amd: (MI355 DPX) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -157,9 +157,9 @@ steps:
- python3 spec_decode/test_custom_proposer.py
mirror:
amd:
label: ":amd: (MI300) Spec Decode N-Gram + Suffix"
label: ":amd: (MI355 DPX) Spec Decode N-Gram + Suffix"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 35
depends_on:
- image-build-amd
@@ -172,22 +172,24 @@ steps:
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model"
- label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model Shard %N"
key: spec-decode-draft-model
timeout_in_minutes: 60
device: h200_18gb
parallelism: 4
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/draft_model/
- pytest -v -s v1/e2e/spec_decode/draft_model/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI300) Spec Decode Draft Model"
label: ":amd: (MI355 DPX) Spec Decode Draft Model Shard %N"
dind: false
device: mi300_1
device: mi355_dpx
timeout_in_minutes: 60
parallelism: 2
depends_on:
- image-build-amd
source_file_dependencies:
@@ -288,7 +290,8 @@ steps:
- vllm/model_executor/models/laguna_dflash.py
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/
# DFlash2 NVFP4 (27B) does not fit a 35GB MIG slice; runs on B200 below.
- pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2"
mirror:
amd:
label: ":amd: (MI300) Spec Decode AL DFlash Nightly"
@@ -314,6 +317,21 @@ steps:
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: ":nvidia: (B200) Spec Decode AL DFlash2 Nightly"
key: spec-decode-dflash2-nightly-b200
timeout_in_minutes: 60
device: b200-k8s
optional: true
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/models/qwen3_dflash.py
- vllm/model_executor/models/qwen3_dflash2.py
- vllm/model_executor/models/qwen3_5.py
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "dflash2"
- label: ":nvidia: (H200 MIG 35GB) Spec Decode AL DSpark Nightly"
key: spec-decode-dspark-nightly
timeout_in_minutes: 60
@@ -374,9 +392,9 @@ steps:
- if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
mirror:
amd:
label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly"
label: ":amd: (MI355 DPX) Spec Decode AL MTP + Other Acceptance Nightly"
dind: false
device: mi355_1
device: mi355_dpx
timeout_in_minutes: 55
depends_on:
- image-build-amd
-47
View File
@@ -1,47 +0,0 @@
[run]
# Track the installed vllm package (this is what actually gets imported during tests)
# Use wildcard pattern to match the installed location
source =
vllm
*/dist-packages/vllm
*/site-packages/vllm
omit =
*/tests/*
*/test_*
*/__pycache__/*
*/build/*
*/dist/*
*/vllm.egg-info/*
*/third_party/*
*/examples/*
*/benchmarks/*
*/docs/*
[paths]
# Map all possible vllm locations to a canonical "vllm" path
# This ensures coverage.combine properly merges data from different test runs
source =
vllm
/vllm-workspace/src/vllm
/vllm-workspace/vllm
*/site-packages/vllm
*/dist-packages/vllm
[report]
exclude_lines =
pragma: no cover
def __repr__
if self.debug:
if settings.DEBUG
raise AssertionError
raise NotImplementedError
if 0:
if __name__ == .__main__.:
class .*\bProtocol\):
@(abc\.)?abstractmethod
[html]
directory = htmlcov
[xml]
output = coverage.xml
-6
View File
@@ -1,6 +0,0 @@
# https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github
have_fun: false # Just review the code
code_review:
comment_severity_threshold: HIGH # Reduce quantity of comments
pull_request_opened:
summary: false # Don't summarize the PR in a separate comment
+2 -2
View File
@@ -50,7 +50,7 @@
# Rust Frontend
/rust/ @BugenZhao @njhill
/rust/src/bench @esmeetu
/build_rust.sh @BugenZhao @njhill
/tools/build_rust.sh @BugenZhao @njhill
/rust-toolchain.toml @BugenZhao @njhill
/.buildkite/test_areas/rust* @BugenZhao @njhill
@@ -134,7 +134,7 @@
mkdocs.yaml @hmellor
# Linting
.markdownlint.yaml @hmellor
/tools/pre_commit/.markdownlint.yaml @hmellor
.pre-commit-config.yaml @hmellor
/tools/pre_commit @hmellor
View File
+2 -2
View File
@@ -29,7 +29,7 @@ repos:
hooks:
- id: markdownlint-cli2
language_version: lts
args: [--fix]
args: [--config, tools/pre_commit/.markdownlint.yaml, --fix]
exclude: (^|/)CLAUDE\.md$
- repo: https://github.com/rhysd/actionlint
rev: v1.7.7
@@ -39,7 +39,7 @@ repos:
rev: 745eface02aef23e168a8afb6b5737818efbea95 # v0.11.0.1
hooks:
- id: shellcheck
args: ["-s", "bash", "--format=gcc"]
args: ["-s", "bash", "--format=gcc", "--rcfile=tools/pre_commit/.shellcheckrc"]
- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.11.1
hooks:
-1
View File
@@ -1 +0,0 @@
@AGENTS.md
-18
View File
@@ -460,8 +460,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
DEFINED CMAKE_CUDA_COMPILER_VERSION AND
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0)
list(APPEND VLLM_STABLE_EXT_SRC "csrc/libtorch_stable/all_reduce_mhc.cu")
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS
"9.0a;10.0f;10.1f;10.3f;10.7f;11.0f;12.0f;12.1f" "${CUDA_ARCHS}")
@@ -777,22 +775,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
"(requires SM90+ and CUDA >= 12.0).")
endif()
# Only build AllSpark kernels if we are building for at least some compatible archs.
cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}")
if (ALLSPARK_ARCHS)
set(ALLSPARK_SRCS
"csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu"
"csrc/libtorch_stable/quantization/gptq_allspark/allspark_qgemm_w8a16.cu")
set_gencode_flags_for_srcs(
SRCS "${ALLSPARK_SRCS}"
CUDA_ARCHS "${ALLSPARK_ARCHS}")
list(APPEND VLLM_STABLE_EXT_SRC "${ALLSPARK_SRCS}")
message(STATUS "Building AllSpark kernels for archs: ${ALLSPARK_ARCHS}")
else()
message(STATUS "Not building AllSpark kernels as no compatible archs found"
" in CUDA target architectures")
endif()
#
# CUTLASS scaled_mm kernels (moved from _C to _C_stable_libtorch)
#
-4
View File
@@ -630,10 +630,6 @@ def get_tokenizer(
pretrained_model_name_or_path
):
pretrained_model_name_or_path = get_model(pretrained_model_name_or_path)
if tokenizer_mode == "slow":
if kwargs.get("use_fast", False):
raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
kwargs["use_fast"] = False
if tokenizer_mode == "mistral":
try:
from vllm.tokenizers.mistral import MistralTokenizer
-59
View File
@@ -6,10 +6,6 @@ import torch.utils.benchmark as benchmark
from benchmark_shapes import WEIGHT_SHAPES
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.allspark_utils import (
ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD,
ALLSPARK_SUPPORTED_QUANT_TYPES,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
GPTQ_MARLIN_MAX_PARALLEL,
GPTQ_MARLIN_MIN_THREAD_N,
@@ -31,7 +27,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
from vllm.model_executor.layers.quantization.utils.quant_utils import (
gptq_pack,
gptq_quantize_weights,
quantize_weights,
)
from vllm.scalar_type import ScalarType, scalar_types
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -62,9 +57,6 @@ def bench_run(
return
repack_supported = group_size in MARLIN_SUPPORTED_GROUP_SIZES
allspark_supported = (
quant_type in ALLSPARK_SUPPORTED_QUANT_TYPES and group_size == -1
)
def gen_marlin_params():
# Marlin quant
@@ -104,35 +96,6 @@ def bench_run(
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
return q_w_gptq
def gen_allspark_params():
qw_reorder = s_reorder = zp_reorder = sm_count = sm_version = (
CUBLAS_M_THRESHOLD
) = None
nonlocal allspark_supported
if allspark_supported:
properties = torch.cuda.get_device_properties(b.device.index)
sm_count = properties.multi_processor_count
sm_version = properties.major * 10 + properties.minor
supported_arch = sm_version >= 80 and sm_version < 90
allspark_supported = allspark_supported and supported_arch
if supported_arch:
w_ref, qw, s, zp = quantize_weights(b, quant_type, group_size, has_zp)
qw = qw.to(torch.uint8)
qw_reorder, s_reorder, zp_reorder = ops.allspark_repack_weight(
qw, s, zp, has_zp
)
CUBLAS_M_THRESHOLD = ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD
return (
qw_reorder,
s_reorder,
zp_reorder,
sm_count,
sm_version,
CUBLAS_M_THRESHOLD,
)
(
marlin_w_ref,
marlin_q_w,
@@ -141,9 +104,6 @@ def bench_run(
marlin_zp,
) = gen_marlin_params()
q_w_gptq = gen_repack_params()
qw_reorder, s_reorder, zp_reorder, sm_count, sm_version, CUBLAS_M_THRESHOLD = (
gen_allspark_params()
)
# Prepare
marlin_workspace = MarlinWorkspace(
@@ -167,17 +127,9 @@ def bench_run(
"marlin_workspace": marlin_workspace,
# GPTQ params
"q_w_gptq": q_w_gptq,
# AllSpark W8A16 params
"qw_reorder": qw_reorder,
"s_reorder": s_reorder,
"zp_reorder": zp_reorder,
"sm_count": sm_count,
"sm_version": sm_version,
"CUBLAS_M_THRESHOLD": CUBLAS_M_THRESHOLD,
# Kernels
"marlin_gemm": ops.marlin_gemm,
"gptq_marlin_repack": ops.gptq_marlin_repack,
"allspark_w8a16_gemm": ops.allspark_w8a16_gemm,
}
min_run_time = 1
@@ -227,17 +179,6 @@ def bench_run(
).blocked_autorange(min_run_time=min_run_time)
)
if allspark_supported:
results.append(
benchmark.Timer(
stmt="output = allspark_w8a16_gemm(a, qw_reorder, s_reorder, zp_reorder, size_n, group_size, sm_count, sm_version, CUBLAS_M_THRESHOLD, False, True)", # noqa: E501
globals=globals,
label=label,
sub_label=sub_label,
description="allspark_w8a16_gemm_fp32",
).blocked_autorange(min_run_time=min_run_time)
)
def main(args):
print("Benchmarking models:")
@@ -0,0 +1,480 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark production MNNVL reduce-scatter backends on SM100/SM103.
The multimem candidate includes the device-to-device copy into its persistent
symmetric input buffer. Timings use pointer-distinct CUDA graphs and report the
median of the per-sample maximum across ranks.
Run the default BF16 message-size sweep with:
.venv/bin/python -m torch.distributed.run --standalone --nproc-per-node=8 \
benchmarks/kernels/benchmark_multimem_reduce_scatter.py
"""
import argparse
import json
import os
import statistics
from collections.abc import Callable
from dataclasses import dataclass
import torch
import torch.distributed as dist
import vllm._custom_ops as ops
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
_DTYPES = {
"bf16": torch.bfloat16,
"f16": torch.float16,
"f32": torch.float32,
}
@dataclass
class Candidate:
name: str
runs: list[Callable[[], None]]
outputs: list[torch.Tensor]
def parse_size(value: str) -> int:
suffixes = {"k": 1 << 10, "m": 1 << 20, "g": 1 << 30}
value = value.strip().lower()
multiplier = suffixes.get(value[-1], 1)
number = value[:-1] if multiplier != 1 else value
return int(float(number) * multiplier)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--message-bytes",
type=parse_size,
nargs="+",
default=[
16 << 20,
32 << 20,
64 << 20,
68 << 20,
72 << 20,
76 << 20,
96 << 20,
],
help="Full input bytes on each rank. K/M/G suffixes are accepted.",
)
parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16")
parser.add_argument(
"--blocks",
type=int,
nargs="+",
default=[8],
help="Maximum CTAs used by each multimem candidate.",
)
parser.add_argument("--num-workspaces", type=int, default=4)
parser.add_argument("--warmup-replays", type=int, default=8)
parser.add_argument("--samples", type=int, default=31)
parser.add_argument(
"--cold-l2",
action="store_true",
help="Flush approximately twice L2 capacity before each replay.",
)
return parser.parse_args()
def make_lamport_run(
comm: CustomAllreduce,
input_tensor: torch.Tensor,
output: torch.Tensor,
) -> Callable[[], None]:
def run() -> None:
ops.mnnvl_lamport_reduce_scatter(
comm._ptr,
input_tensor,
output,
comm.mnnvl_lamport_rs_local_ptr,
comm.mnnvl_lamport_rs_epoch_ptr,
comm.mnnvl_buffer_size,
)
return run
def make_multimem_run(
comm: CustomAllreduce,
input_tensor: torch.Tensor,
output: torch.Tensor,
blocks: int,
) -> Callable[[], None]:
def run() -> None:
ops.mnnvl_multimem_reduce_scatter(
comm._ptr,
input_tensor,
output,
comm.mnnvl_multimem_rs_local_ptr,
comm.mnnvl_multimem_rs_multicast_ptr,
comm.mnnvl_multimem_rs_buffer_size,
blocks,
)
return run
def make_nccl_run(
input_tensor: torch.Tensor,
output: torch.Tensor,
device_group: dist.ProcessGroup,
) -> Callable[[], None]:
def run() -> None:
dist.reduce_scatter_single(output, input_tensor, group=device_group)
return run
def build_candidates(
comm: CustomAllreduce,
inputs: list[torch.Tensor],
local_elements: int,
blocks_values: list[int],
device_group: dist.ProcessGroup,
) -> list[Candidate]:
candidates: list[Candidate] = []
lamport_outputs = [
input_tensor.new_empty(local_elements) for input_tensor in inputs
]
candidates.append(
Candidate(
"lamport",
[
make_lamport_run(comm, input_tensor, output)
for input_tensor, output in zip(inputs, lamport_outputs)
],
lamport_outputs,
)
)
for blocks in blocks_values:
outputs = [input_tensor.new_empty(local_elements) for input_tensor in inputs]
candidates.append(
Candidate(
f"multimem_b{blocks}",
[
make_multimem_run(comm, input_tensor, output, blocks)
for input_tensor, output in zip(inputs, outputs)
],
outputs,
)
)
nccl_outputs = [input_tensor.new_empty(local_elements) for input_tensor in inputs]
candidates.append(
Candidate(
"nccl",
[
make_nccl_run(input_tensor, output, device_group)
for input_tensor, output in zip(inputs, nccl_outputs)
],
nccl_outputs,
)
)
return candidates
def check_correctness(
candidates: list[Candidate],
inputs: list[torch.Tensor],
rank: int,
world_size: int,
cpu_group: dist.ProcessGroup,
device_group: dist.ProcessGroup,
graphs: dict[str, list[torch.cuda.CUDAGraph]] | None = None,
) -> None:
expected_outputs = []
for input_tensor in inputs:
expected = input_tensor.float().clone()
dist.all_reduce(expected, group=device_group)
expected_outputs.append(expected.to(input_tensor.dtype).chunk(world_size)[rank])
if inputs[0].dtype == torch.float32:
rtol, atol = 1e-5, 1e-5
elif inputs[0].dtype == torch.float16:
rtol, atol = 1e-2, 1e-2
else:
rtol, atol = 5e-2, 1.25e-1
for candidate in candidates:
for workspace_index, expected in enumerate(expected_outputs):
dist.barrier(group=cpu_group)
if graphs is None:
candidate.runs[workspace_index]()
else:
graphs[candidate.name][workspace_index].replay()
torch.accelerator.synchronize()
error = None
try:
torch.testing.assert_close(
candidate.outputs[workspace_index],
expected,
rtol=rtol,
atol=atol,
)
except AssertionError as assertion_error:
error = assertion_error
correct = torch.tensor(
error is None,
dtype=torch.int32,
device=candidate.outputs[workspace_index].device,
)
dist.all_reduce(correct, op=dist.ReduceOp.MIN, group=device_group)
if not correct.item():
location = f"{candidate.name} workspace {workspace_index}"
detail = str(error) if error is not None else "failed on another rank"
raise AssertionError(
f"{location} produced an incorrect shard: {detail}"
)
dist.barrier(group=cpu_group)
def capture_graph(
run: Callable[[], None],
cpu_group: dist.ProcessGroup,
) -> tuple[torch.cuda.CUDAGraph, torch.cuda.Stream]:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
dist.barrier(group=cpu_group)
with torch.cuda.stream(stream):
for _ in range(3):
run()
stream.synchronize()
dist.barrier(group=cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
run()
torch.cuda.current_stream().wait_stream(stream)
dist.barrier(group=cpu_group)
return graph, stream
def capture_candidates(
candidates: list[Candidate],
cpu_group: dist.ProcessGroup,
) -> tuple[dict[str, list[torch.cuda.CUDAGraph]], list[object]]:
graphs = {}
keepalive: list[object] = []
for candidate in candidates:
bundles = [capture_graph(run, cpu_group) for run in candidate.runs]
graphs[candidate.name] = [graph for graph, _ in bundles]
keepalive.extend(bundles)
return graphs, keepalive
def benchmark_graphs(
graphs: dict[str, list[torch.cuda.CUDAGraph]],
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
flush: torch.Tensor | None,
) -> dict[str, float]:
names = list(graphs)
for round_index in range(warmup_replays):
for candidate_index in range(len(names)):
name = names[(round_index + candidate_index) % len(names)]
device_barrier()
graphs[name][round_index % len(graphs[name])].replay()
torch.accelerator.synchronize()
timings = {name: [] for name in names}
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
for sample_index in range(samples):
for candidate_index in range(len(names)):
name = names[(sample_index + candidate_index) % len(names)]
if flush is not None:
flush.zero_()
device_barrier()
start.record()
graphs[name][sample_index % len(graphs[name])].replay()
end.record()
end.synchronize()
elapsed = torch.tensor(
start.elapsed_time(end) * 1000,
dtype=torch.float64,
device=torch.accelerator.current_device_index(),
)
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
timings[name].append(elapsed.item())
return {name: statistics.median(values) for name, values in timings.items()}
def benchmark_size(
args: argparse.Namespace,
comm: CustomAllreduce,
message_bytes: int,
dtype: torch.dtype,
device: torch.device,
rank: int,
world_size: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
) -> dict[str, object]:
element_size = dtype.itemsize
if message_bytes % (element_size * world_size) != 0:
raise ValueError("Message bytes must divide evenly into TP shards")
num_elements = message_bytes // element_size
local_elements = num_elements // world_size
if local_elements * element_size % 16 != 0:
raise ValueError("Each output shard must be a multiple of 16 bytes")
inputs = []
for workspace_index in range(args.num_workspaces):
generator = torch.Generator(device=device)
generator.manual_seed(1000 + rank * 101 + workspace_index)
input_tensor = torch.empty(num_elements, dtype=dtype, device=device)
input_tensor.uniform_(-1.0, 1.0, generator=generator)
inputs.append(input_tensor)
candidates = build_candidates(
comm,
inputs,
local_elements,
args.blocks,
device_group,
)
torch.accelerator.synchronize()
dist.barrier(group=cpu_group)
check_correctness(
candidates,
inputs,
rank,
world_size,
cpu_group,
device_group,
)
graphs, graph_keepalive = capture_candidates(candidates, cpu_group)
check_correctness(
candidates,
inputs,
rank,
world_size,
cpu_group,
device_group,
graphs,
)
flush = None
if args.cold_l2:
l2_bytes = torch.cuda.get_device_properties(device).L2_cache_size
flush = torch.empty(2 * l2_bytes, dtype=torch.uint8, device=device)
times = benchmark_graphs(
graphs,
args.warmup_replays,
args.samples,
device_group,
device_barrier,
flush,
)
graph_keepalive.clear()
bus_bytes = message_bytes * (world_size - 1) / world_size
fastest = min(times, key=times.__getitem__)
return {
"message_bytes": message_bytes,
"shard_bytes": message_bytes // world_size,
"latency_us": times,
"logical_bus_gbps": {
name: bus_bytes / (latency_us * 1e3) for name, latency_us in times.items()
},
"fastest": fastest,
}
def main() -> None:
args = parse_args()
if not args.message_bytes or min(args.message_bytes) <= 0:
raise ValueError("--message-bytes values must be positive")
if not args.blocks or min(args.blocks) <= 0 or max(args.blocks) > 8:
raise ValueError("--blocks values must be in [1, 8]")
if min(args.num_workspaces, args.samples) <= 0:
raise ValueError("Workspace and sample counts must be positive")
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
dist.init_process_group("nccl")
device_group = dist.group.WORLD
cpu_group = dist.new_group(backend="gloo")
rank = dist.get_rank(device_group)
world_size = dist.get_world_size(device_group)
device = torch.device("cuda", local_rank)
capability = torch.cuda.get_device_capability(device)
if world_size not in (2, 4, 8) or capability not in ((10, 0), (10, 3)):
raise RuntimeError(
"This benchmark targets TP2/TP4/TP8 SM100/SM103, "
f"got TP{world_size} SM{capability}"
)
max_message_bytes = max(args.message_bytes)
comm = CustomAllreduce(
group=cpu_group,
device=device,
max_mnnvl_reduce_scatter_size=max_message_bytes,
max_mnnvl_multimem_reduce_scatter_size=max_message_bytes,
)
if comm.disabled:
raise RuntimeError("The production custom all-reduce backend is unavailable")
comm._init_mnnvl_multimem_reduce_scatter_buffer()
if not comm.mnnvl_multimem_rs_multicast_ptr:
raise RuntimeError("The production MNNVL multimem backend is unavailable")
dtype = _DTYPES[args.dtype]
warmup = torch.zeros(1, dtype=torch.float32, device=device)
dist.all_reduce(warmup, group=device_group)
sync = torch.zeros_like(warmup)
def device_barrier() -> None:
dist.all_reduce(sync, group=device_group)
results = [
benchmark_size(
args,
comm,
message_bytes,
dtype,
device,
rank,
world_size,
device_group,
cpu_group,
device_barrier,
)
for message_bytes in args.message_bytes
]
if rank == 0:
metadata = {
"world_size": world_size,
"gpu": torch.cuda.get_device_name(device),
"sm": f"{capability[0]}{capability[1]}",
"num_sms": torch.cuda.get_device_properties(device).multi_processor_count,
"dtype": args.dtype,
"torch": torch.__version__,
"cuda": torch.version.cuda,
"blocks": args.blocks,
"num_workspaces": args.num_workspaces,
"cold_l2": args.cold_l2,
}
print(json.dumps({"metadata": metadata, "results": results}, indent=2))
comm.close()
dist.barrier(group=cpu_group)
dist.destroy_process_group(cpu_group)
dist.destroy_process_group()
if __name__ == "__main__":
main()
@@ -117,7 +117,18 @@ if __name__ == "__main__":
args = parse_args()
warmup_iters, bench_iters = args.warmup_iters, args.bench_iters
shapes = [(32, 128), (64, 256), (16, 512)]
shapes = [
(32, 128),
(64, 256),
(16, 512),
# Serving-sized activations, where this op is memory-bound and its
# schedule dominates: a prefill chunk and a decode batch at two
# common hidden sizes.
(1024, 6144),
(8192, 6144),
(1024, 7168),
(8192, 7168),
]
group_sizes = [64, 128]
dtypes = ["fp8", "int8"] if args.dtype == "both" else [args.dtype]
+18
View File
@@ -173,6 +173,18 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA)
set(AMX_FP8_SUPPORTED FALSE)
message(STATUS "AMX-FP8 disabled by ENABLE_AMX_FP8=OFF")
endif()
# AVX10.2 arrived in GCC 15, so the target attribute, the fp8 convert
# intrinsics and __builtin_cpu_supports("avx10.2") are all hard errors on
# the 12.3+ the backend otherwise supports. Probe instead of assuming.
check_cxx_compiler_flag("-mavx10.2" COMPILER_SUPPORTS_AVX10_2_FLAG)
if (COMPILER_SUPPORTS_AVX10_2_FLAG)
set(AVX10_2_SUPPORTED TRUE)
else()
set(AVX10_2_SUPPORTED FALSE)
message(STATUS "AVX10.2 disabled: compiler does not support -mavx10.2 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})")
endif()
list(APPEND CXX_COMPILE_FLAGS_AVX2
"-mavx2")
elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND)
@@ -629,6 +641,12 @@ if (ENABLE_X86_ISA)
message(STATUS "AMX-FP8 (Diamond Rapids) enabled")
endif()
# For the sgl-kernels AVX10.2 fp8 quantize paths
if (AVX10_2_SUPPORTED)
target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX10_2")
message(STATUS "AVX10.2 enabled")
endif()
# AVX512F
define_extension_target(
_C_AVX512
+1 -1
View File
@@ -13,7 +13,7 @@ else()
FetchContent_Declare(
flashkda
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
GIT_PROGRESS TRUE
GIT_SUBMODULES cutlass
)
+1 -1
View File
@@ -142,9 +142,9 @@ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS)
target_compile_definitions(_qutlass_C PRIVATE
QUTLASS_MINIMAL_BUILD=1
QUTLASS_DISABLE_PYBIND=1
TARGET_CUDA_ARCH=${QUTLASS_TARGET_CC}
CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1
TORCH_TARGET_VERSION=0x020B000000000000ULL
USE_CUDA)
set_property(SOURCE ${QUTLASS_SOURCES} APPEND PROPERTY COMPILE_OPTIONS
+2 -2
View File
@@ -12,8 +12,8 @@ if (DEFINED ENV{TRITON_KERNELS_SRC_DIR})
else()
if (VLLM_TARGET_DEVICE STREQUAL "rocm")
set(TRITON_GIT "https://github.com/ROCm/triton.git")
# Pinned from release/internal/3.6.x
set(TRITON_KERNELS_TAG "0f380657dbf3ee86eb57558ff71df24f03b5d4e7")
# Pinned from release/internal/3.8.x
set(TRITON_KERNELS_TAG "669b31acc1dd1b3fd93286afd5db67f65d9f7557")
else()
set(TRITON_GIT "https://github.com/triton-lang/triton.git")
set(TRITON_KERNELS_TAG "v3.5.1")
+13 -1
View File
@@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache(
input.block_table = block_table.data_ptr<int32_t>();
input.alibi_slopes =
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
// Attention sinks may be bf16 (native path) or fp32. Anything that is not
// bf16 must be provided as fp32 and is executed in full float precision.
if (s_aux.has_value()) {
TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 ||
s_aux->scalar_type() == at::ScalarType::Float,
"cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ",
s_aux->scalar_type(), " must be bfloat16 or float32");
input.s_aux = s_aux->data_ptr();
input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16;
} else {
input.s_aux = nullptr;
input.s_aux_is_bf16 = false;
}
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.scale = scale;
+24 -9
View File
@@ -879,7 +879,11 @@ struct AttentionInput {
int32_t* seq_lens;
int32_t* block_table;
float* alibi_slopes;
c10::BFloat16* s_aux;
// Attention sinks pointer. May reference bf16 or fp32 data depending on
// `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is
// provided as fp32 and executed in full float precision.
const void* s_aux;
bool s_aux_is_bf16;
bool* dynamic_causal;
float scale;
bool causal;
@@ -1565,7 +1569,8 @@ class AttentionMainLoop {
const float scale = input->scale;
const float softcap_scale = input->softcap;
const float* alibi_slopes = input->alibi_slopes;
const c10::BFloat16* s_aux = input->s_aux;
const void* s_aux = input->s_aux;
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
const bool* dynamic_causal = input->dynamic_causal;
const bool is_dynamic_causal = dynamic_causal != nullptr;
@@ -1773,9 +1778,6 @@ class AttentionMainLoop {
const float* curr_alibi_slopes =
(alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx
: nullptr);
const c10::BFloat16* curr_s_aux =
(s_aux != nullptr ? s_aux + q_head_start_idx : nullptr);
// copy the Q tile to q_buffer, the logical layout of q_buffer is
// [actual_q_token_num, curr_q_heads_per_kv, head_dim]
{
@@ -1787,10 +1789,23 @@ class AttentionMainLoop {
if (use_sink) {
alignas(64) float s_aux_fp32[16];
// All other platforms have BF16Vec16 available
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
// Sinks may be provided as bf16 or fp32. Either way they are
// loaded into fp32: bf16 sinks are up-converted (native
// BF16Vec16 path, available on all platforms), while fp32
// sinks are loaded directly and executed in full precision.
if (s_aux_is_bf16) {
const c10::BFloat16* curr_s_aux =
static_cast<const c10::BFloat16*>(s_aux) +
q_head_start_idx;
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
} else {
const float* curr_s_aux =
static_cast<const float*>(s_aux) + q_head_start_idx;
vec_op::FP32Vec16 vec_fp32(curr_s_aux);
vec_fp32.save(s_aux_fp32);
}
float* __restrict__ curr_sum_buffer = sum_buffer;
float* __restrict__ curr_max_buffer = max_buffer;
+7
View File
@@ -532,10 +532,17 @@ void zero_buffer(T* data, int64_t size) {
#endif
#if defined(__x86_64__)
// CPU_CAPABILITY_AVX10_2 is defined by cmake only when the compiler accepts
// -mavx10.2; without it the builtin below is not merely absent but a hard
// error, so the AVX512 paths have to stand in.
bool avx10_2_available() {
#ifdef CPU_CAPABILITY_AVX10_2
// __builtin_cpu_supports returns the masked feature bit rather than 0/1.
static const bool available = __builtin_cpu_supports("avx10.2");
return available;
#else
return false;
#endif
}
#endif
+6
View File
@@ -772,6 +772,7 @@ inline __m128i cvtfp32_fp8e4m3(__m512& src) {
return _mm512_cvtepi32_epi8(packed);
}
#ifdef CPU_CAPABILITY_AVX10_2
__attribute__((target("avx10.2"))) inline __m128i cvtfp32_fp8e4m3_avx10_2(__m512& src) {
__m256i f16_vec = _mm512_cvt_roundps_ph(src, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
return _mm256_cvtph_hf8(_mm256_castsi256_ph(f16_vec));
@@ -805,6 +806,7 @@ __attribute__((target("avx10.2"))) void quantize_bf16_block_avx10_2(
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), fp8_vec2);
}
}
#endif // CPU_CAPABILITY_AVX10_2
std::tuple<at::Tensor, at::Tensor> _quantize_fp8e4m3_bf16_per_tensor_no_scale(const at::Tensor& t) {
constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max
@@ -861,10 +863,12 @@ std::tuple<at::Tensor, at::Tensor> _quantize_fp8e4m3_bf16_per_tensor_no_scale(co
scale_data[c] = scale_val;
// Step 3: Scale and clamp using AVX512 (reuse the same loop structure)
#ifdef CPU_CAPABILITY_AVX10_2
if (avx10_2_available()) {
quantize_bf16_block_avx10_2(channel_src, quant_dst, elements_per_channel, scale_reciprocal);
continue;
}
#endif
i = 0;
const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal);
const __m512 quant_max_vec = _mm512_set1_ps(quant_max);
@@ -939,10 +943,12 @@ _quantize_fp8e4m3_bf16_per_tensor_with_scale(const at::Tensor& t, at::Tensor& sc
const at::BFloat16* src_data = t_bf16.data_ptr<at::BFloat16>() + c * elements_per_channel;
at::Float8_e4m3fn* quant_t_data = quant_t.data_ptr<at::Float8_e4m3fn>() + c * elements_per_channel;
#ifdef CPU_CAPABILITY_AVX10_2
if (avx10_2_available()) {
quantize_bf16_block_avx10_2(src_data, quant_t_data, elements_per_channel, scale_reciprocal);
continue;
}
#endif
int64_t i = 0;
// Process 32 elements at a time using AVX512
+125
View File
@@ -7,6 +7,10 @@ namespace vllm {
constexpr int kMnnvlLamportAgThreads = 128;
constexpr int kMnnvlLamportRsThreads = 256;
constexpr int kMnnvlLamportConcurrentPollMaxPacks = 8192;
constexpr int kMnnvlMultimemRsThreads = 1024;
constexpr int kMnnvlMultimemRsBlockLimit = 8;
constexpr int kMnnvlMultimemRsVectorBytes = 16;
constexpr int kMnnvlMultimemRsUnroll = 8;
using CopyPack = array_t<uint64_t, 2>;
@@ -48,6 +52,127 @@ __global__ void __launch_bounds__(512, 1)
barrier_at_end<ngpus, true>(sg, self_sg, rank);
}
// Multimem reduce-scatter reduces through an NVLS multicast mapping, which
// has no AMD equivalent. The whole path is compiled out on ROCm and the host
// entry point rejects the call instead of launching anything.
#if !defined(USE_ROCM)
template <typename T>
DINLINE void multimem_load_reduce_16(uint32_t (&result)[4], const T* address) {
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
if constexpr (std::is_same<T, nv_bfloat16>::value) {
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.bf16x2 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
} else if constexpr (std::is_same<T, half>::value) {
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.f16x2 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
} else {
static_assert(std::is_same<T, float>::value);
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.v4.f32 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
}
#else
asm volatile("trap;");
#endif
}
DINLINE void store_global_16(void* address, const uint32_t (&value)[4]) {
asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};"
:
: "l"(address), "r"(value[0]), "r"(value[1]), "r"(value[2]),
"r"(value[3])
: "memory");
}
DINLINE void mnnvl_multimem_publish_flag(FlagType* flag_addr, FlagType flag) {
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
asm volatile("multimem.st.release.sys.global.u32 [%0], %1;"
:
: "l"(flag_addr), "r"(flag)
: "memory");
// The flag is stored through the multicast alias and polled through the
// local unicast alias in the same grid.
asm volatile("fence.proxy.alias;" ::: "memory");
#else
asm volatile("trap;");
#endif
}
template <bool ready, int ngpus>
__global__ void mnnvl_multimem_barrier_kernel(Signal* local_signal,
Signal* multicast_signal,
int rank) {
FlagType flag = local_signal->_flag[0] + 1;
FlagType* multicast_counters =
ready ? multicast_signal->start[0] : multicast_signal->end[0];
FlagType* local_counters =
ready ? local_signal->start[0] : local_signal->end[0];
// Replicate this rank's counter into every backing allocation.
mnnvl_multimem_publish_flag(&multicast_counters[rank], flag);
#pragma unroll
for (int peer = 0; peer < ngpus; ++peer) {
while (ld_flag_acquire(&local_counters[peer]) != flag);
}
local_signal->_flag[0] = flag;
}
template <typename T, int ngpus>
__global__ void __launch_bounds__(kMnnvlMultimemRsThreads, 1)
mnnvl_multimem_reduce_scatter_kernel(const T* __restrict__ multicast_input,
T* __restrict__ result, int rank,
int packs_per_rank) {
constexpr int kThreadsPerWarp = 32;
constexpr int kWarpsPerCta = kMnnvlMultimemRsThreads / kThreadsPerWarp;
constexpr int kPacksPerWarpIteration =
kThreadsPerWarp * kMnnvlMultimemRsUnroll;
int lane;
asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane));
int warp = blockIdx.x * kWarpsPerCta + threadIdx.x / kThreadsPerWarp;
int num_warps = gridDim.x * kWarpsPerCta;
int pack_offset = warp * kPacksPerWarpIteration + lane;
int pack_stride = num_warps * kPacksPerWarpIteration;
auto* rank_input = reinterpret_cast<const char*>(multicast_input) +
rank * packs_per_rank * kMnnvlMultimemRsVectorBytes;
auto* rank_output = reinterpret_cast<char*>(result);
while (pack_offset < packs_per_rank) {
uint32_t reduced[kMnnvlMultimemRsUnroll][4];
#pragma unroll
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
int pack = pack_offset + u * kThreadsPerWarp;
if (pack < packs_per_rank) {
multimem_load_reduce_16<T>(
reduced[u], reinterpret_cast<const T*>(
rank_input + pack * kMnnvlMultimemRsVectorBytes));
}
}
#pragma unroll
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
int pack = pack_offset + u * kThreadsPerWarp;
if (pack < packs_per_rank) {
store_global_16(rank_output + pack * kMnnvlMultimemRsVectorBytes,
reduced[u]);
}
}
pack_offset += pack_stride;
}
}
#endif // !defined(USE_ROCM)
template <typename P>
union LamportPack {
P packed;
+6
View File
@@ -339,6 +339,12 @@ class CustomAllreduce {
void mnnvl_lamport_reduce_scatter(cudaStream_t stream, T* input, T* output,
void* local_buffer, uint32_t* epochs,
int size, int stage_size_bytes);
template <typename T>
void mnnvl_multimem_reduce_scatter(cudaStream_t stream,
const T* multicast_input, T* output,
Signal* local_signal,
Signal* multicast_signal, int size,
int block_limit);
~CustomAllreduce() {
for (auto [_, ptr] : ipc_handles_) {
-212
View File
@@ -1,212 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#include "torch_utils.h"
#include "custom_all_gather_reduce_scatter.cuh"
#include <cooperative_groups.h>
#include <torch/csrc/stable/library.h>
namespace {
using Tensor = torch::stable::Tensor;
using Pack = vllm::packed_t<nv_bfloat16>::P;
constexpr int kHidden = 5120;
constexpr int kRanks = 4;
constexpr int kThreads = 128;
constexpr int kCluster = 5;
constexpr int kPacks = kHidden / Pack::size;
__global__ __launch_bounds__(kThreads)
__cluster_dims__(1, kCluster, 1) void all_reduce_mhc_kernel(
const Pack* input, const Pack* residual, const float* post,
const float* comb, const float* pre, const Pack* weight, Pack* output,
Pack* normalized, Pack* local, Pack* multicast, uint32_t* epochs,
int rank, int stage_size, float eps) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
cudaGridDependencySynchronize();
const int token = blockIdx.x;
const int column = blockIdx.y * kThreads + threadIdx.x;
const int idx = token * kPacks + column;
const int size = gridDim.x * kPacks;
const int stage = epochs[0] % 3;
const int dirty = vllm::mnnvl_lamport_dirty_stage(stage);
const int dirty_size = epochs[2 + dirty];
Pack value = input[idx];
vllm::store_multimem_lamport_payload(
multicast + stage * stage_size + rank * size + idx,
vllm::sanitize_lamport_payload(value));
// Multi-token forwards benefit from launching dependents before mHC finishes.
if (gridDim.x > 1) cudaTriggerProgrammaticLaunchCompletion();
vllm::lamport_cta_arrive(epochs + 1);
for (int i = idx; i < dirty_size; i += size) {
local[dirty * stage_size + i] = vllm::lamport_sentinel<Pack>();
}
Pack peers[kRanks];
bool pending;
do {
vllm::LamportPack<Pack> packets[kRanks];
#pragma unroll
for (int peer = 0; peer < kRanks; ++peer) {
auto ptr = local + stage * stage_size + idx + peer * size;
asm volatile("ld.relaxed.sys.global.v4.u32 {%0,%1,%2,%3}, [%4];"
: "=r"(packets[peer].words[0]), "=r"(packets[peer].words[1]),
"=r"(packets[peer].words[2]), "=r"(packets[peer].words[3])
: "l"(ptr)
: "memory");
}
pending = false;
#pragma unroll
for (int peer = 0; peer < kRanks; ++peer) {
pending |= vllm::is_lamport_dirty(packets[peer]);
peers[peer] = packets[peer].packed;
}
} while (pending);
auto sum = vllm::upcast(peers[0]);
#pragma unroll
for (int r = 1; r < kRanks; ++r) {
vllm::packed_assign_add(sum, vllm::upcast(peers[r]));
}
// Keep the BF16 boundary between the collective and mHC post.
auto reduced = vllm::upcast(vllm::downcast<Pack>(sum));
decltype(sum) streams[4];
#pragma unroll
for (int r = 0; r < 4; ++r) {
streams[r] = vllm::upcast(residual[(token * 4 + r) * kPacks + column]);
}
decltype(sum) collapsed{};
#pragma unroll
for (int c = 0; c < 4; ++c) {
decltype(sum) mixed;
#pragma unroll
for (int e = 0; e < Pack::size; ++e) {
// Match post*x followed by the four residual FMAs.
float acc = __fmul_rn(post[token * 4 + c], reduced.data[e]);
#pragma unroll
for (int r = 0; r < 4; ++r) {
acc = fmaf(comb[token * 16 + r * 4 + c], streams[r].data[e], acc);
}
mixed.data[e] = acc;
}
Pack rounded = vllm::downcast<Pack>(mixed);
output[(token * 4 + c) * kPacks + column] = rounded;
auto as_float = vllm::upcast(rounded);
#pragma unroll
for (int e = 0; e < Pack::size; ++e) {
collapsed.data[e] =
fmaf(pre[token * 4 + c], as_float.data[e], collapsed.data[e]);
}
}
// RMSNorm consumes the rounded BF16 collapse in the unfused path.
collapsed = vllm::upcast(vllm::downcast<Pack>(collapsed));
float sqr = 0;
#pragma unroll
for (int e = 0; e < Pack::size; ++e) {
sqr = fmaf(collapsed.data[e], collapsed.data[e], sqr);
}
#pragma unroll
for (int delta = 16; delta; delta /= 2) {
sqr += __shfl_xor_sync(0xffffffff, sqr, delta);
}
__shared__ float warp_sums[kCluster][kThreads / 32];
auto cluster = cooperative_groups::this_cluster();
const int lane = threadIdx.x % 32;
if (lane < kCluster) {
*cluster.map_shared_rank(&warp_sums[blockIdx.y][threadIdx.x / 32], lane) =
sqr;
}
cluster.sync();
float total = 0;
#pragma unroll
for (int c = 0; c < kCluster; ++c) {
#pragma unroll
for (int w = 0; w < kThreads / 32; ++w) total += warp_sums[c][w];
}
const float scale = rsqrtf(__fadd_rn(total / kHidden, eps));
auto weights = vllm::upcast(weight[column]);
#pragma unroll
for (int e = 0; e < Pack::size; ++e) {
collapsed.data[e] = collapsed.data[e] * scale * weights.data[e];
}
normalized[idx] = vllm::downcast<Pack>(collapsed);
if (idx == 0) {
while (*reinterpret_cast<volatile uint32_t*>(epochs + 1) <
gridDim.x * gridDim.y) {
}
epochs[2 + stage] = kRanks * size;
epochs[0] = vllm::mnnvl_lamport_next_stage(stage);
epochs[1] = 0;
}
if (gridDim.x == 1) cudaTriggerProgrammaticLaunchCompletion();
#else
asm volatile("trap;");
#endif
}
void all_reduce_mhc(Tensor input, Tensor residual, Tensor post, Tensor comb,
Tensor pre, Tensor weight, Tensor output, Tensor normalized,
int64_t local, int64_t multicast, Tensor epochs,
int64_t rank, int64_t stage_bytes, double eps) {
using torch::headeronly::ScalarType;
const torch::stable::accelerator::DeviceGuard guard(input.get_device_index());
const int64_t n = input.size(0);
STD_TORCH_CHECK(input.dim() == 2 && input.size(1) == kHidden && n > 0 &&
n <= 16);
for (const auto& t : {input, residual, weight, output, normalized}) {
STD_TORCH_CHECK(t.is_contiguous() &&
t.scalar_type() == ScalarType::BFloat16);
STD_TORCH_CHECK(t.get_device_index() == input.get_device_index());
STD_TORCH_CHECK(reinterpret_cast<uintptr_t>(t.const_data_ptr()) % 16 == 0);
}
for (const auto& t : {post, comb, pre}) {
STD_TORCH_CHECK(t.is_contiguous() && t.scalar_type() == ScalarType::Float);
STD_TORCH_CHECK(t.get_device_index() == input.get_device_index());
}
STD_TORCH_CHECK(residual.numel() == n * 4 * kHidden &&
output.numel() == residual.numel() &&
normalized.numel() == input.numel() &&
weight.numel() == kHidden);
STD_TORCH_CHECK(post.numel() == n * 4 && pre.numel() == n * 4 &&
comb.numel() == n * 16);
STD_TORCH_CHECK(epochs.is_contiguous() &&
epochs.scalar_type() == ScalarType::Int &&
epochs.numel() >= 5 &&
epochs.get_device_index() == input.get_device_index());
STD_TORCH_CHECK(local && multicast && rank >= 0 && rank < kRanks &&
stage_bytes % 16 == 0 &&
stage_bytes >= n * kHidden * 2 * kRanks);
STD_TORCH_CHECK(get_device_prop()->major == 10);
cudaLaunchAttribute attr{};
attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;
attr.val.programmaticStreamSerializationAllowed = 1;
cudaLaunchConfig_t config{
.gridDim = dim3(n, kCluster),
.blockDim = dim3(kThreads),
.dynamicSmemBytes = 0,
.stream = get_current_cuda_stream(input.get_device_index()),
.attrs = &attr,
.numAttrs = 1};
STD_CUDA_CHECK(cudaLaunchKernelEx(
&config, all_reduce_mhc_kernel,
static_cast<const Pack*>(input.const_data_ptr()),
static_cast<const Pack*>(residual.const_data_ptr()),
static_cast<const float*>(post.const_data_ptr()),
static_cast<const float*>(comb.const_data_ptr()),
static_cast<const float*>(pre.const_data_ptr()),
static_cast<const Pack*>(weight.const_data_ptr()),
static_cast<Pack*>(output.mutable_data_ptr()),
static_cast<Pack*>(normalized.mutable_data_ptr()),
reinterpret_cast<Pack*>(local), reinterpret_cast<Pack*>(multicast),
static_cast<uint32_t*>(epochs.mutable_data_ptr()), static_cast<int>(rank),
static_cast<int>(stage_bytes / sizeof(Pack)), static_cast<float>(eps)));
}
} // namespace
STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, mhc) {
mhc.def(
"all_reduce_mhc(Tensor input, Tensor residual, Tensor post, Tensor comb, "
"Tensor pre, Tensor weight, Tensor! output, Tensor! normalized, "
"int local, int multicast, Tensor! epochs, int rank, int stage_bytes, "
"float eps) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, mhc) {
mhc.impl("all_reduce_mhc", TORCH_BOX(&all_reduce_mhc));
}
+11 -6
View File
@@ -549,16 +549,20 @@ __global__ void concat_and_cache_ds_mla_kernel(
scalar_t* kv_cache_16bit =
reinterpret_cast<scalar_t*>(&kv_cache[dst_idx_start]);
// The last warp handles the RoPE part
// Zero the reserved RoPE tail for NoPE rows.
if (threadIdx.x >= 64) {
// Each thread handles two elements of RoPE
const int8_t pe_idx_start = (threadIdx.x - 64) * 2;
const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start;
// Vectorized load of two 16-bit values, performed as one 32-bit load
const int32_t vals = *reinterpret_cast<const int32_t*>(&k_pe[src_idx]);
// RoPE values start after the packed 8-bit NoPE values and the
// 32-bit scales
const int64_t dst_idx = kv_lora_rank / 2 + 8 + pe_idx_start;
if (pe_dim == 0) {
*reinterpret_cast<int32_t*>(&kv_cache_16bit[dst_idx]) = 0;
return;
}
const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start;
// Vectorized load of two 16-bit values, performed as one 32-bit load
const int32_t vals = *reinterpret_cast<const int32_t*>(&k_pe[src_idx]);
// Vectorized store of two 16-bit values, performed as one 32-bit store
*reinterpret_cast<int32_t*>(&kv_cache_16bit[dst_idx]) = vals;
return;
@@ -600,7 +604,7 @@ __global__ void concat_and_cache_ds_mla_kernel(
}
// Now all threads in the block scale and write their elements
// NoPE data is packed in the first kv_lora_rank/2 bytes (first 256 bytes)
// NoPE data occupies the first kv_lora_rank bytes (512 bytes).
const int64_t dst_idx_base = dst_idx_start + (threadIdx.x * 8);
uint8_t result[8];
@@ -936,7 +940,8 @@ void concat_and_cache_mla(
if (kv_cache_dtype == "fp8_ds_mla") {
STD_TORCH_CHECK(kv_lora_rank == 512,
"kv_lora_rank must be 512 for fp8_ds_mla");
STD_TORCH_CHECK(pe_dim == 64, "pe_dim must be 64 for fp8_ds_mla");
STD_TORCH_CHECK(pe_dim == 64 || pe_dim == 0,
"pe_dim must be 64 or 0 for fp8_ds_mla");
STD_TORCH_CHECK(kv_cache.size(2) == 656 / kv_cache.element_size(),
"kv_cache.size(2) must be 656 bytes for fp8_ds_mla");
STD_TORCH_CHECK(kv_c.element_size() == 2,
@@ -191,6 +191,55 @@ void CustomAllreduce::mnnvl_lamport_reduce_scatter(cudaStream_t stream,
#undef MNNVL_LAMPORT_RS_LAUNCH
}
template <typename T>
void CustomAllreduce::mnnvl_multimem_reduce_scatter(
cudaStream_t stream, const T* multicast_input, T* output,
Signal* local_signal, Signal* multicast_signal, int size, int block_limit) {
#if defined(USE_ROCM)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires NVLS multicast and is not "
"supported on ROCm");
#else
if (size <= 0)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires a non-empty input");
int size_bytes = size * sizeof(T);
if (size_bytes % (kMnnvlMultimemRsVectorBytes * world_size_) != 0)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires each output shard byte size "
"to be a multiple of 16");
if (block_limit <= 0 || block_limit > kMnnvlMultimemRsBlockLimit)
throw std::runtime_error(
"MNNVL multimem reduce-scatter block limit is out of range");
int packs_per_rank = size_bytes / kMnnvlMultimemRsVectorBytes / world_size_;
int packs_per_block = kMnnvlMultimemRsThreads * kMnnvlMultimemRsUnroll;
int blocks = std::min(
block_limit, (packs_per_rank + packs_per_block - 1) / packs_per_block);
#define MNNVL_MULTIMEM_RS_CASE(ngpus) \
case ngpus: \
mnnvl_multimem_barrier_kernel<true, ngpus> \
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
mnnvl_multimem_reduce_scatter_kernel<T, ngpus> \
<<<blocks, kMnnvlMultimemRsThreads, 0, stream>>>( \
multicast_input, output, rank_, packs_per_rank); \
mnnvl_multimem_barrier_kernel<false, ngpus> \
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
break;
switch (world_size_) {
MNNVL_MULTIMEM_RS_CASE(2)
MNNVL_MULTIMEM_RS_CASE(4)
MNNVL_MULTIMEM_RS_CASE(8)
default:
throw std::runtime_error(
"MNNVL multimem reduce-scatter only supports num gpus in (2,4,8)");
}
#undef MNNVL_MULTIMEM_RS_CASE
#endif
}
} // namespace vllm
using fptr_t = int64_t;
@@ -360,3 +409,62 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
"bfloat16");
}
}
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t _local_buffer,
fptr_t _multicast_buffer,
int64_t stage_sz_bytes,
int64_t block_limit) {
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
const torch::stable::accelerator::DeviceGuard device_guard(
inp.get_device_index());
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel()));
STD_TORCH_CHECK(inp.numel() > 0);
STD_TORCH_CHECK(_is_weak_contiguous(out));
STD_TORCH_CHECK(_is_weak_contiguous(inp));
auto input_size = inp.numel() * inp.element_size();
STD_TORCH_CHECK(input_size <= stage_sz_bytes);
auto local_buffer = reinterpret_cast<void*>(_local_buffer);
auto multicast_buffer = reinterpret_cast<void*>(_multicast_buffer);
STD_TORCH_CHECK(local_buffer != nullptr);
STD_TORCH_CHECK(multicast_buffer != nullptr);
auto* local_signal = reinterpret_cast<vllm::Signal*>(
static_cast<char*>(local_buffer) - sizeof(vllm::Signal));
auto* multicast_signal = reinterpret_cast<vllm::Signal*>(
static_cast<char*>(multicast_buffer) - sizeof(vllm::Signal));
STD_CUDA_CHECK(cudaMemcpyAsync(local_buffer, inp.const_data_ptr(), input_size,
cudaMemcpyDeviceToDevice, stream));
switch (out.scalar_type()) {
case torch::headeronly::ScalarType::Float: {
fa->mnnvl_multimem_reduce_scatter<float>(
stream, reinterpret_cast<float*>(multicast_buffer),
reinterpret_cast<float*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
case torch::headeronly::ScalarType::Half: {
fa->mnnvl_multimem_reduce_scatter<half>(
stream, reinterpret_cast<half*>(multicast_buffer),
reinterpret_cast<half*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
case torch::headeronly::ScalarType::BFloat16: {
fa->mnnvl_multimem_reduce_scatter<nv_bfloat16>(
stream, reinterpret_cast<nv_bfloat16*>(multicast_buffer),
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
#endif
default:
throw std::runtime_error(
"MNNVL multimem reduce-scatter only supports float32, float16 and "
"bfloat16");
}
}
@@ -17,6 +17,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ag_rs) {
custom_ag_rs.def(
"mnnvl_lamport_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
"local_buffer, int epoch_buffer, int stage_sz_bytes) -> ()");
custom_ag_rs.def(
"mnnvl_multimem_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
"local_buffer, int multicast_buffer, int stage_sz_bytes, int "
"block_limit) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
@@ -26,4 +30,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
custom_ag_rs.impl("custom_reduce_scatter", TORCH_BOX(&custom_reduce_scatter));
custom_ag_rs.impl("mnnvl_lamport_reduce_scatter",
TORCH_BOX(&mnnvl_lamport_reduce_scatter));
custom_ag_rs.impl("mnnvl_multimem_reduce_scatter",
TORCH_BOX(&mnnvl_multimem_reduce_scatter));
}
+4 -3
View File
@@ -175,9 +175,6 @@ torch::stable::Tensor awq_dequantize(torch::stable::Tensor _kernel,
// DSV3 fused A GEMM: conditionally compiled so declaration and impl
// registration are in the source file (dsv3_fused_a_gemm.cu)
// AllSpark ops: declarations are in the source files
// (allspark_repack.cu and allspark_qgemm_w8a16.cu)
#endif
// CPU tensor -> CUDA UVA view (shared CUDA/ROCm)
@@ -530,6 +527,10 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t local_buffer, fptr_t epoch_buffer,
int64_t stage_sz_bytes);
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t local_buffer, fptr_t multicast_buffer,
int64_t stage_sz_bytes, int64_t block_limit);
void dispose(fptr_t _fa);
int64_t meta_size();
void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
File diff suppressed because it is too large Load Diff
@@ -1,176 +0,0 @@
#include "allspark_utils.cuh"
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "core/registration.h"
#include "libtorch_stable/torch_utils.h"
namespace allspark {
// Rearrange B to facilitate Ampere Tensor Core load data
// reorder B from (K, N) to (N_32align / 4, K * 4)
// K % 16 == 0, N % 16 == 0, N_32align % 32 == 0
template <typename FType>
__global__ void __launch_bounds__(128)
rearrange_kn_weight_as_n32k16_order_ldg16_kernel(
const uint8_t* B, const FType* B_scale, const FType* B_zero,
uint8_t* B_result, FType* B_scale_result, FType* B_zero_result,
const int K, const int N, const int N_32align) {
const auto lane_id = threadIdx.x % 32;
const auto warp_id = threadIdx.x / 32;
if (blockIdx.x != gridDim.x - 1) {
// Load B
// per block process 64(k) * 128(n) B elements
// per warp process 16(k) * 128 B elements
const int src_row_base_idx =
blockIdx.x * 64 + warp_id * 16 + ((lane_id % 8) / 2) * 2;
const int src_col_idx =
blockIdx.y * 128 + (lane_id / 8) * 32 + (lane_id % 2) * 16;
uint8_t B_frag[4][16];
#pragma unroll
for (int i = 0; i < 4; ++i) {
int src_row_idx = src_row_base_idx + (i / 2) * 8 + (i % 2);
int src_offset = src_row_idx * N + src_col_idx;
bool guard = src_row_idx < K && src_col_idx < N;
ldg128_cg_0(*reinterpret_cast<uint32_t*>(B_frag[i]),
*(reinterpret_cast<uint32_t*>(B_frag[i]) + 1),
*(reinterpret_cast<uint32_t*>(B_frag[i]) + 2),
*(reinterpret_cast<uint32_t*>(B_frag[i]) + 3), B + src_offset,
guard);
}
// reorder B
uint8_t B_reorder_frag[8][8];
#pragma unroll
for (int i = 0; i < 4; ++i) {
#pragma unroll
for (int j = 0; j < 16; ++j) {
int dst_i = j % 8;
int dst_j = i + (j / 8) * 4;
B_reorder_frag[dst_i][dst_j] = B_frag[i][j];
}
}
// Store B
const auto dst_row_base_idx = blockIdx.y * (128 / 4) + (lane_id / 8) * 8;
const int dst_col_idx =
blockIdx.x * (64 * 4) + warp_id * 64 + (lane_id % 8) * 8;
for (int i = 0; i < 8; ++i) {
int dst_row_idx = dst_row_base_idx + i;
int dst_offset = dst_row_idx * K * 4 + dst_col_idx;
bool guard = (dst_row_base_idx < N_32align / 4) && (dst_col_idx < K * 4);
if (guard) {
*reinterpret_cast<int2*>(B_result + dst_offset) =
*reinterpret_cast<int2*>(B_reorder_frag[i]);
}
}
} else {
// Load B_scale and B_zero
FType b_scale_reg, b_zero_reg;
auto src_offset = blockIdx.y * 128 + threadIdx.x;
ldg16_cg_0(b_scale_reg, B_scale + src_offset, src_offset < N);
if (B_zero != nullptr)
ldg16_cg_0(b_zero_reg, B_zero + src_offset, src_offset < N);
int dst_offset =
blockIdx.y * 128 + warp_id * 32 + (lane_id % 8) * 4 + lane_id / 8;
if (dst_offset < N_32align) {
B_scale_result[dst_offset] = b_scale_reg;
if (B_zero != nullptr) B_zero_result[dst_offset] = b_zero_reg;
}
}
}
template <typename FType>
void rearrange_kn_weight_as_n32k16_order_ldg16(
const uint8_t* B, const FType* B_scale, const FType* B_zero,
uint8_t* B_result, FType* B_scale_result, FType* B_zero_result,
const int64_t K, const int64_t N, const int64_t N_32align,
cudaStream_t stream) {
if (N % 16 != 0 || K % 16 != 0) {
std::cerr << "Now only support N and K is multiples of 16" << std::endl;
}
const int BLOCK = 128;
int grid_x = (K + 64 - 1) / 64 + 1;
int grid_y = (N + 128 - 1) / 128;
dim3 grid(grid_x, grid_y);
rearrange_kn_weight_as_n32k16_order_ldg16_kernel<FType>
<<<grid, BLOCK, 0, stream>>>(B, B_scale, B_zero, B_result, B_scale_result,
B_zero_result, K, N, N_32align);
}
} // namespace allspark
void rearrange_kn_weight_as_n32k16_order(
torch::stable::Tensor const& b_qweight,
torch::stable::Tensor const& b_scales,
std::optional<torch::stable::Tensor> const& b_zeros, bool has_zp,
torch::stable::Tensor& b_qweight_reorder,
torch::stable::Tensor& b_scales_reorder,
std::optional<torch::stable::Tensor> const& b_zeros_reorder,
const int64_t K, const int64_t N, const int64_t N_32align) {
// Verify device and strides
STD_TORCH_CHECK(b_qweight.device().is_cuda(), "b_qweight is not on GPU");
STD_TORCH_CHECK(b_qweight.is_contiguous(), "b_qweight is not contiguous");
STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
STD_TORCH_CHECK(b_qweight_reorder.device().is_cuda(),
"b_qweight_reorder is not on GPU");
STD_TORCH_CHECK(b_qweight_reorder.is_contiguous(),
"b_qweight_reorder is not contiguous");
STD_TORCH_CHECK(b_scales_reorder.device().is_cuda(),
"b_scales_reorder is not on GPU");
STD_TORCH_CHECK(b_scales_reorder.is_contiguous(),
"b_scales_reorder is not contiguous");
if (has_zp) {
STD_TORCH_CHECK(b_zeros.value().device().is_cuda(),
"b_zeros is not on GPU");
STD_TORCH_CHECK(b_zeros.value().is_contiguous(),
"b_zeros is not contiguous");
STD_TORCH_CHECK(b_zeros_reorder.value().device().is_cuda(),
"b_zeros_reorder is not on GPU");
STD_TORCH_CHECK(b_zeros_reorder.value().is_contiguous(),
"b_zeros_reorder is not contiguous");
}
const uint8_t* matB = reinterpret_cast<const uint8_t*>(b_qweight.data_ptr());
const void* b_scale = b_scales.data_ptr();
const void* b_zero = has_zp ? b_zeros.value().data_ptr() : nullptr;
uint8_t* matB_reorder =
reinterpret_cast<uint8_t*>(b_qweight_reorder.mutable_data_ptr());
void* b_scale_reorder = b_scales_reorder.mutable_data_ptr();
void* b_zero_reorder =
has_zp ? b_zeros_reorder.value().mutable_data_ptr() : nullptr;
const torch::stable::accelerator::DeviceGuard device_guard(
b_qweight.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) {
allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__half>(
matB, reinterpret_cast<const __half*>(b_scale),
reinterpret_cast<const __half*>(b_zero), matB_reorder,
reinterpret_cast<__half*>(b_scale_reorder),
reinterpret_cast<__half*>(b_zero_reorder), K, N, N_32align, stream);
} else if (b_scales.scalar_type() ==
torch::headeronly::ScalarType::BFloat16) {
allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__nv_bfloat16>(
matB, reinterpret_cast<const __nv_bfloat16*>(b_scale),
reinterpret_cast<const __nv_bfloat16*>(b_zero), matB_reorder,
reinterpret_cast<__nv_bfloat16*>(b_scale_reorder),
reinterpret_cast<__nv_bfloat16*>(b_zero_reorder), K, N, N_32align,
stream);
}
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
m.impl("rearrange_kn_weight_as_n32k16_order",
TORCH_BOX(&rearrange_kn_weight_as_n32k16_order));
}
@@ -1,387 +0,0 @@
#pragma once
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <iostream>
#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh"
using marlin::MarlinScalarType2;
namespace allspark {
#define CHECK_CUDA(cmd) \
do { \
cudaError_t cuda_status = cmd; \
if (cuda_status != cudaSuccess) { \
std::string err_str = cudaGetErrorString(cuda_status); \
std::cerr << "Failed: " << __FILE__ << ":" << __LINE__ << " " \
<< err_str; \
exit(-1); \
} \
} while (0)
#define CHECK_CUBLAS(cmd) \
do { \
cublasStatus_t cublas_status = cmd; \
if (cublas_status != CUBLAS_STATUS_SUCCESS) { \
std::cerr << "Failed: " << __FILE__ << ":" << __LINE__ << " " \
<< cublas_status << std::endl; \
exit(-1); \
} \
} while (0)
template <typename FType, typename QType>
struct SM8x_GEMM_W8A16_Splitk_Params {
const FType* A_ptr;
const QType* B_ptr;
const FType* B_scale_ptr;
const FType* B_zero_ptr;
FType* C_ptr;
int M;
int N;
int K;
int SplitK;
int GroupCnt;
int GroupSize;
FType* C_split_ptr; // for non-fused splitk reduce
float* C_tmp_ptr; // for fused splitk reduce
uint32_t* red_count_ptr; // for fused splitk reduce
};
struct alignas(16) BlockTileSplitkParams {
int Mtile;
int Ntile;
int SplitK;
bool EnableFuse;
};
template <typename FType, int BLOCK, int N_MATRIX>
__global__ void f16_gemm_splitk_reduce_kernel(const FType* C_split, FType* C,
uint32_t n, uint32_t n_matrix,
uint32_t matrix_size) {
auto idx = blockIdx.x * BLOCK + threadIdx.x;
if (idx >= matrix_size) {
return;
}
float sum = 0.f;
int n_mat = N_MATRIX > 0 ? N_MATRIX : (int)n_matrix;
for (int i = 0; i < n_mat; ++i) {
sum += MarlinScalarType2<FType>::num2float(C_split[idx + i * matrix_size]);
}
C[idx] = MarlinScalarType2<FType>::float2num(sum);
}
template <typename FType>
void f16_gemm_splitk_reduce(const FType* C_split, FType* C, const uint32_t m,
const uint32_t n, const uint32_t n_matrix,
cudaStream_t stream) {
const int BLOCK = 128;
uint32_t matrix_size = m * n;
int grid = (matrix_size + BLOCK - 1) / BLOCK;
void (*kernel)(const FType*, FType*, uint32_t, uint32_t, uint32_t) = nullptr;
switch (n_matrix) {
case 4:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 4>;
break;
case 5:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 5>;
break;
case 6:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 6>;
break;
case 7:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 7>;
break;
case 8:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 8>;
break;
case 9:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 9>;
break;
case 10:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 10>;
break;
case 11:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 11>;
break;
case 12:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, 12>;
break;
default:
kernel = f16_gemm_splitk_reduce_kernel<FType, BLOCK, -1>;
break;
}
kernel<<<grid, BLOCK, 0, stream>>>(C_split, C, n, n_matrix, matrix_size);
}
template <typename T>
struct HalfType;
template <>
struct HalfType<half> {
using T1 = __half;
using T2 = __half2;
};
template <>
struct HalfType<__nv_bfloat16> {
using T1 = __nv_bfloat16;
using T2 = __nv_bfloat162;
};
// convert 64-bit pointer to 32-bit smem addr
__device__ __forceinline__ uint32_t smem_u32addr(const void* smem_ptr) {
uint32_t addr;
asm("{.reg .u64 u64addr;\n"
" cvta.to.shared.u64 u64addr, %1;\n"
" cvt.u32.u64 %0, u64addr;}\n"
: "=r"(addr)
: "l"(smem_ptr));
return addr;
}
template <typename T>
__device__ __forceinline__ void ldg16_cg_0(T& r0, const void* ptr, bool guard) {
static_assert(sizeof(T) == 2, "ldg16_cg_0: invalid T");
asm volatile(
"{.reg .pred p;\n"
" setp.ne.b32 p, %2, 0;\n"
" @!p mov.b16 %0, 0;\n"
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \
__CUDA_ARCH__ >= 750
" @p ld.global.cg.L2::128B.b16 {%0}, [%1];}\n"
#else
" @p ld.global.ca.b16 {%0}, [%1];}\n"
#endif
: "=h"(reinterpret_cast<uint16_t&>(r0))
: "l"(ptr), "r"((int)guard));
}
template <typename T>
__device__ __forceinline__ void ldg64_ca(T& r0, T& r1, const void* ptr,
bool guard) {
static_assert(sizeof(T) == 4, "ldg64_ca: invalid T");
asm volatile(
"{.reg .pred p;\n"
" setp.ne.b32 p, %3, 0;\n"
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \
__CUDA_ARCH__ >= 750
" @p ld.global.ca.L2::128B.v2.b32 {%0, %1}, [%2];}\n"
#else
" @p ld.global.ca.v2.b32 {%0, %1}, [%2];}\n"
#endif
: "=r"(reinterpret_cast<uint32_t&>(r0)),
"=r"(reinterpret_cast<uint32_t&>(r1))
: "l"(ptr), "r"((int)guard));
}
template <typename T>
__device__ __forceinline__ void ldg128_cg_0(T& r0, T& r1, T& r2, T& r3,
const void* ptr, bool guard) {
static_assert(sizeof(T) == 4, "ldg128_cg_0: invalid T");
asm volatile(
"{.reg .pred p;\n"
" setp.ne.b32 p, %5, 0;\n"
" @!p mov.b32 %0, 0;\n"
" @!p mov.b32 %1, 0;\n"
" @!p mov.b32 %2, 0;\n"
" @!p mov.b32 %3, 0;\n"
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \
__CUDA_ARCH__ >= 750
" @p ld.global.cg.L2::128B.v4.b32 {%0, %1, %2, %3}, [%4];}\n"
#else
" @p ld.global.cg.v4.b32 {%0, %1, %2, %3}, [%4];}\n"
#endif
: "=r"(reinterpret_cast<uint32_t&>(r0)),
"=r"(reinterpret_cast<uint32_t&>(r1)),
"=r"(reinterpret_cast<uint32_t&>(r2)),
"=r"(reinterpret_cast<uint32_t&>(r3))
: "l"(ptr), "r"((int)guard));
}
template <typename T>
__device__ __forceinline__ void lds128(T& reg0, T& reg1, T& reg2, T& reg3,
const uint32_t addr) {
static_assert(sizeof(T) == 4, "lds128: invalid T");
asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(reinterpret_cast<uint32_t&>(reg0)),
"=r"(reinterpret_cast<uint32_t&>(reg1)),
"=r"(reinterpret_cast<uint32_t&>(reg2)),
"=r"(reinterpret_cast<uint32_t&>(reg3))
: "r"(addr));
}
template <typename T>
__device__ __forceinline__ void stg128(const T& r0, const T& r1, const T& r2,
const T& r3, const void* ptr,
bool guard) {
static_assert(sizeof(T) == 4, "stg128: invalid T");
asm volatile(
"{.reg .pred p;\n"
" setp.ne.b32 p, %1, 0;\n"
" @p st.global.v4.b32 [%0], {%2, %3, %4, %5};}\n"
:
: "l"(ptr), "r"((int)guard), "r"(reinterpret_cast<const uint32_t&>(r0)),
"r"(reinterpret_cast<const uint32_t&>(r1)),
"r"(reinterpret_cast<const uint32_t&>(r2)),
"r"(reinterpret_cast<const uint32_t&>(r3)));
}
template <typename T>
__device__ __forceinline__ void ldsm_4(T& r0, T& r1, T& r2, T& r3,
const uint32_t& addr) {
static_assert(sizeof(T) == 4, "ldsm_4: invalid T");
#if (__CUDA_ARCH__ >= 750) && (__CUDACC_VER_MAJOR__ >= 11)
asm volatile(
"ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(reinterpret_cast<uint32_t&>(r0)),
"=r"(reinterpret_cast<uint32_t&>(r1)),
"=r"(reinterpret_cast<uint32_t&>(r2)),
"=r"(reinterpret_cast<uint32_t&>(r3))
: "r"(addr));
#endif
}
template <typename FType>
__device__ __forceinline__ void hmma16816_f32(float (&d)[4],
const uint32_t (&a)[4],
const uint32_t (&b)[2]);
template <>
__device__ __forceinline__ void hmma16816_f32<__half>(float (&d)[4],
const uint32_t (&a)[4],
const uint32_t (&b)[2]) {
#if (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11)
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, "
"{%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
: "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
#endif
}
template <>
__device__ __forceinline__ void hmma16816_f32<__nv_bfloat16>(
float (&d)[4], const uint32_t (&a)[4], const uint32_t (&b)[2]) {
#if (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11)
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, "
"{%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
: "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
#endif
}
template <int SIZE_IN_BYTES>
__device__ __forceinline__ void cp_async(const uint32_t smem_addr,
const void* gmem_ptr,
const int src_in_bytes, bool guard) {
static_assert(
(SIZE_IN_BYTES == 4 || SIZE_IN_BYTES == 8 || SIZE_IN_BYTES == 16),
"Size is not supported");
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800
asm volatile(
"{.reg.pred p;\n"
" setp.ne.b32 p, %4, 0;\n"
#if __CUDACC_VER_MINOR__ >= 4
" @p cp.async.cg.shared.global.L2::256B [%0], [%1], %2, %3;}\n"
#else
" @p cp.async.cg.shared.global [%0], [%1], %2, %3;}\n"
#endif
::"r"(smem_addr),
"l"(gmem_ptr), "n"(SIZE_IN_BYTES), "r"(src_in_bytes), "r"((int)guard));
#endif
}
__device__ __forceinline__ void cp_async_commit_group() {
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800
asm volatile("cp.async.commit_group;\n");
#endif
}
template <int N>
__device__ __forceinline__ void cp_asyc_wait_group() {
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800
asm volatile("cp.async.wait_group %0;\n" : : "n"(N));
#endif
}
template <typename T>
__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128(const uint32_t& idata,
T* fdata);
template <>
// fast conversion: 4xuint8 to 4xhalf, subtracting bias = 128
__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128<__half2>(
const uint32_t& idata, __half2* fdata) {
uint32_t i10, i32;
asm volatile(
"prmt.b32 %0, %2, 0x64, 0x4140;"
"prmt.b32 %1, %2, 0x64, 0x4342;"
: "=r"(i10), "=r"(i32)
: "r"(idata));
static constexpr uint32_t MAGIC_NUM = 0x64806480;
fdata[0] = __hsub2(reinterpret_cast<const __half2&>(i10),
reinterpret_cast<const __half2&>(MAGIC_NUM));
fdata[1] = __hsub2(reinterpret_cast<const __half2&>(i32),
reinterpret_cast<const __half2&>(MAGIC_NUM));
}
template <>
// fast conversion: 4xuint8 to 4xbfloat16, subtracting bias = 128
// reference from marlin fast implementation
__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128<__nv_bfloat162>(
const uint32_t& idata, __nv_bfloat162* fdata) {
float fp32_imd[4];
uint32_t* fp32_imd_casted = reinterpret_cast<uint32_t*>(fp32_imd);
asm volatile(
"prmt.b32 %0, %4, 0x4B000000, 0x7650;"
"prmt.b32 %1, %4, 0x4B000000, 0x7651;"
"prmt.b32 %2, %4, 0x4B000000, 0x7652;"
"prmt.b32 %3, %4, 0x4B000000, 0x7653;"
: "=r"(fp32_imd_casted[0]), "=r"(fp32_imd_casted[1]),
"=r"(fp32_imd_casted[2]), "=r"(fp32_imd_casted[3])
: "r"(idata));
fp32_imd[0] -= 8388736.f;
fp32_imd[1] -= 8388736.f;
fp32_imd[2] -= 8388736.f;
fp32_imd[3] -= 8388736.f;
uint32_t* bf16_res = reinterpret_cast<uint32_t*>(fdata);
asm volatile(
"prmt.b32 %0, %2, %3, 0x7632;"
"prmt.b32 %1, %4, %5, 0x7632;"
: "=r"(bf16_res[0]), "=r"(bf16_res[1])
: "r"(fp32_imd_casted[0]), "r"(fp32_imd_casted[1]),
"r"(fp32_imd_casted[2]), "r"(fp32_imd_casted[3]));
}
static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
assert(false);
#else
return __bfloat162bfloat162(x);
#endif
__builtin_unreachable(); // Suppress missing return statement warning
}
static __device__ half2 inline num2num2(const half x) {
return __half2half2(x);
}
} // namespace allspark
@@ -796,7 +796,7 @@ torch::stable::Tensor hadacore_transform(torch::stable::Tensor& x, bool inplace)
VLLM_STABLE_DISPATCH_HALF_TYPES(x.scalar_type(), "hadacore_transform_runfht", [&] {
auto constexpr SCALAR_TYPE = torch::headeronly::CppTypeToScalarType<scalar_t>::value;
hadacore::run_fht<SCALAR_TYPE>(x.data_ptr(), x.data_ptr(), x.numel(), had_size, stream);
hadacore::run_fht<SCALAR_TYPE>(x.data_ptr(), out.data_ptr(), x.numel(), had_size, stream);
});
if (numel % 256 != 0) {
@@ -1,11 +1,60 @@
#include "../../../../quantization/w8a8/fp8/common.cuh"
#include "../../../../cuda_compat.h"
#include "../../../dispatch_utils.h"
#include "../../../cub_helpers.h"
#include "../../vectorization_utils.cuh"
#include "../../../torch_utils.h"
#include <torch/csrc/stable/macros.h>
#include <algorithm>
namespace vllm {
namespace {
// 16 fp8 outputs form one 16-byte store; the matching 16-element input load
// is 32 B for 16-bit types and 64 B for fp32.
constexpr int kFp8QuantVecSize = 16;
constexpr int kFp8QuantBlockSize = 256;
template <typename T>
using fp8_quant_vec_t = vec_n_t<T, kFp8QuantVecSize>;
template <typename T>
bool ptr_vec_aligned(const T* ptr) {
return reinterpret_cast<uintptr_t>(ptr) % alignof(fp8_quant_vec_t<T>) == 0;
}
__device__ __forceinline__ float warp_reduce_max(float v) {
#pragma unroll
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset));
}
return v;
}
// Block-wide max, returned to every thread. Warp partials go through one
// shared-memory exchange, so the whole reduction costs a single
// __syncthreads. blockDim.x must be a multiple of WARP_SIZE and at most
// kFp8QuantBlockSize.
__device__ __forceinline__ float block_reduce_max(float v, float* warp_max) {
const int lane = threadIdx.x % WARP_SIZE;
const int warp = threadIdx.x / WARP_SIZE;
const int num_warps = blockDim.x / WARP_SIZE;
v = warp_reduce_max(v);
if (lane == 0) {
warp_max[warp] = v;
}
__syncthreads();
v = warp_max[0];
for (int w = 1; w < num_warps; ++w) {
v = fmaxf(v, warp_max[w]);
}
return v;
}
} // namespace
// STRIDE_I_ZERO: true if scale_stride_i == 0 (per-tensor or per-channel)
// STRIDE_J_ZERO: true if scale_stride_j == 0 (per-tensor or per-token)
template <typename scalar_t, typename fp8_type, bool STRIDE_I_ZERO,
@@ -75,42 +124,59 @@ __global__ void scaled_fp8_quant_kernel_strided_group_shape(
}
}
// Per-tensor abs-max of a contiguous tensor viewed as a flat array of
// 16-element vectors: grid-stride loop, one atomic per block.
template <typename scalar_t, typename fp8_type>
__global__ void segmented_max_reduction_strided(
float* __restrict__ scale, const scalar_t* __restrict__ input,
int hidden_size, int64_t in_row_stride, int64_t num_tokens) {
__shared__ float cache[256];
const int tid = threadIdx.x;
int64_t token_idx = blockIdx.x;
__global__ void __launch_bounds__(kFp8QuantBlockSize)
absmax_reduction_flat(float* __restrict__ scale,
const scalar_t* __restrict__ input,
int64_t num_vecs) {
using in_vec_t = fp8_quant_vec_t<scalar_t>;
const auto* v_in = reinterpret_cast<const in_vec_t*>(input);
const int64_t stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
// one block per token. Guard in case gridDim.x > num_tokens.
float thread_max = 0.0f;
#pragma unroll 4
for (int64_t v = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
v < num_vecs; v += stride) {
const in_vec_t vec = v_in[v];
#pragma unroll
for (int i = 0; i < kFp8QuantVecSize; ++i) {
thread_max = fmaxf(thread_max, fabsf(static_cast<float>(vec.val[i])));
}
}
__shared__ float warp_max[kFp8QuantBlockSize / 32];
const float block_max = block_reduce_max(thread_max, warp_max);
if (threadIdx.x == 0) {
atomicMaxFloat(scale, block_max / quant_type_max_v<fp8_type>);
}
}
// Per-tensor abs-max for strided or unaligned rows: one block per token.
template <typename scalar_t, typename fp8_type>
__global__ void __launch_bounds__(kFp8QuantBlockSize)
segmented_max_reduction_strided(float* __restrict__ scale,
const scalar_t* __restrict__ input,
int hidden_size, int64_t in_row_stride,
int64_t num_tokens) {
const int64_t token_idx = blockIdx.x;
if (token_idx >= num_tokens) {
return;
}
const scalar_t* row_ptr = input + token_idx * in_row_stride;
// each thread scans elements of the row in a strided fashion.
float thread_max = 0.0f;
for (int e = tid; e < hidden_size; e += blockDim.x) {
float v = fabsf(static_cast<float>(row_ptr[e]));
thread_max = fmaxf(thread_max, v);
}
vectorize_read_with_alignment<kFp8QuantVecSize>(
row_ptr, hidden_size, threadIdx.x, blockDim.x,
[&] __device__(const scalar_t& v) {
thread_max = fmaxf(thread_max, fabsf(static_cast<float>(v)));
});
cache[tid] = thread_max;
__syncthreads();
// parallel reduction to find row max.
for (int offset = blockDim.x / 2; offset > 0; offset >>= 1) {
if (tid < offset) {
cache[tid] = fmaxf(cache[tid], cache[tid + offset]);
}
__syncthreads();
}
// thread 0 updates global scale (per-tensor) atomically.
if (tid == 0) {
atomicMaxFloat(scale, cache[0] / quant_type_max_v<fp8_type>);
__shared__ float warp_max[kFp8QuantBlockSize / 32];
const float block_max = block_reduce_max(thread_max, warp_max);
if (threadIdx.x == 0) {
atomicMaxFloat(scale, block_max / quant_type_max_v<fp8_type>);
}
}
@@ -345,7 +411,7 @@ void dynamic_scaled_fp8_quant(torch::stable::Tensor& out, // [..., d]
const int hidden_size = input.size(-1);
const int num_tokens = input.numel() / hidden_size;
const int block_size = 256;
const int block_size = vllm::kFp8QuantBlockSize;
dim3 grid(num_tokens);
dim3 block(block_size);
@@ -364,11 +430,29 @@ void dynamic_scaled_fp8_quant(torch::stable::Tensor& out, // [..., d]
input.scalar_type(), "scaled_fp8_quant_kernel_scalar_type", [&] {
VLLM_STABLE_DISPATCH_FP8_TYPES(
out.scalar_type(), "scaled_fp8_quant_kernel_fp8_type", [&] {
vllm::segmented_max_reduction_strided<scalar_t, fp8_t>
<<<grid, block, 0, stream>>>(
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), hidden_size,
in_row_stride, static_cast<int64_t>(num_tokens));
const scalar_t* in_ptr = input.const_data_ptr<scalar_t>();
const bool flat = in_row_stride == hidden_size &&
hidden_size % vllm::kFp8QuantVecSize == 0 &&
vllm::ptr_vec_aligned(in_ptr);
if (flat) {
const int64_t num_vecs = input.numel() / vllm::kFp8QuantVecSize;
const int64_t max_blocks =
static_cast<int64_t>(
get_device_prop()->multiProcessorCount) *
8;
const int64_t num_blocks = std::max<int64_t>(
1,
std::min<int64_t>(
max_blocks, (num_vecs + block_size - 1) / block_size));
vllm::absmax_reduction_flat<scalar_t, fp8_t>
<<<num_blocks, block, 0, stream>>>(
scale.mutable_data_ptr<float>(), in_ptr, num_vecs);
} else {
vllm::segmented_max_reduction_strided<scalar_t, fp8_t>
<<<grid, block, 0, stream>>>(
scale.mutable_data_ptr<float>(), in_ptr, hidden_size,
in_row_stride, static_cast<int64_t>(num_tokens));
}
vllm::scaled_fp8_quant_kernel_strided_dynamic<scalar_t, fp8_t>
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<fp8_t>(),
@@ -169,6 +169,156 @@ __global__ void per_token_group_quant_8bit_kernel(
#endif
}
// Register-resident variant for the unpacked float32 scale layout, mirroring
// per_token_group_quant_8bit_packed_register_kernel below, which float32-scale
// callers cannot reach because it is hardwired to UE8M0 exponents packed
// 4-per-int32 with TMA-aligned strides.
//
// A group_size-128 group of 16-bit input is 256 B, so it fits in registers and
// the smem staging plus the __syncthreads() that the kernel above needs between
// its absmax and quantize passes are both avoidable.
//
// Requires GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE, i.e. group_size 128 with
// 16-bit input; CanUseRegisterPath enforces that.
template <typename T, typename DST_DTYPE, bool IS_COLUMN_MAJOR = false,
bool SCALE_UE8M0 = false, int GROUP_SIZE = 128>
__global__ void per_token_group_quant_8bit_register_kernel(
const T* __restrict__ input, void* __restrict__ output_q,
float* __restrict__ output_s, const int64_t num_groups, const float eps,
const float min_8bit, const float max_8bit, const int scale_num_rows,
const int scale_stride) {
constexpr int THREADS_PER_GROUP = 8;
constexpr int VEC_SIZE = 32 / sizeof(T);
static_assert(32 % THREADS_PER_GROUP == 0,
"THREADS_PER_GROUP must divide the warp size so the shuffle "
"mask stays inside one group");
static_assert(sizeof(DST_DTYPE) == 1,
"the uint4 store packs one byte per element");
// CanUseRegisterPath only routes 16-bit inputs here, but
// VLLM_STABLE_DISPATCH_FLOATING_TYPES instantiates every floating type, so
// gate the body on the shape this schedule can actually serve instead of a
// static_assert.
if constexpr (GROUP_SIZE != THREADS_PER_GROUP * VEC_SIZE) {
return;
} else {
const int64_t group_id =
(static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x) /
THREADS_PER_GROUP;
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
#endif
if (group_id >= num_groups) {
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaTriggerProgrammaticLaunchCompletion();
#endif
return;
}
const int lane_id = threadIdx.x % THREADS_PER_GROUP;
// alignas(16) is required for the uint4 reinterpret_cast: the default
// alignof for 16-bit T is 2.
alignas(16) T regs[VEC_SIZE];
{
const T* group_input = input + group_id * GROUP_SIZE + lane_id * VEC_SIZE;
uint4* dst = reinterpret_cast<uint4*>(&regs[0]);
const uint4* src = reinterpret_cast<const uint4*>(group_input);
dst[0] = src[0];
dst[1] = src[1];
}
float local_absmax = eps;
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
local_absmax = fmaxf(local_absmax, fabsf(static_cast<float>(regs[i])));
}
// Reduce within the 8-lane subgroup that shares this group.
#ifdef USE_ROCM
const int lane_in_wave = threadIdx.x % warpSize;
const unsigned long long mask = 0xFFull << (lane_in_wave & ~7);
local_absmax =
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4, 8));
local_absmax =
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2, 8));
local_absmax =
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1, 8));
#else
const unsigned mask = 0xffu << (threadIdx.x & 24u);
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1));
#endif
float y_s = local_absmax / max_8bit;
if constexpr (SCALE_UE8M0) {
// Bit-exact with exp2f(ceilf(log2f(fmaxf(fabsf(y_s), 1e-10f)))): bump the
// exponent whenever the mantissa is non-zero, then clear the mantissa.
y_s = fmaxf(fabsf(y_s), 1e-10f);
const uint32_t bits = __float_as_uint(y_s);
const uint32_t exp_bits = (bits >> 23) & 0xffu;
const uint32_t mant_bits = bits & 0x7fffffu;
y_s = __uint_as_float((exp_bits + (mant_bits != 0u ? 1u : 0u)) << 23);
}
if (lane_id == 0) {
if constexpr (IS_COLUMN_MAJOR) {
const int64_t row_idx = group_id / scale_num_rows;
const int64_t col_idx = group_id % scale_num_rows;
output_s[col_idx * scale_stride + row_idx] = y_s;
} else {
output_s[group_id] = y_s;
}
}
// A UE8M0 scale is an exact power of two, so multiplying by the reciprocal
// matches dividing. Otherwise the reciprocal costs an ulp and can move
// values that land exactly on an 8-bit boundary, so divide instead to stay
// bit-exact with the smem kernel.
const float inv_y = 1.0f / y_s;
uint32_t packed[VEC_SIZE / 4] = {};
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
const float v = static_cast<float>(regs[i]);
const float scaled = SCALE_UE8M0 ? (v * inv_y) : (v / y_s);
const float q = fminf(fmaxf(scaled, min_8bit), max_8bit);
DST_DTYPE qb = DST_DTYPE(q);
const uint8_t byte = *reinterpret_cast<uint8_t*>(&qb);
packed[i >> 2] |= static_cast<uint32_t>(byte) << ((i & 3) * 8);
}
DST_DTYPE* group_output = static_cast<DST_DTYPE*>(output_q) +
group_id * GROUP_SIZE + lane_id * VEC_SIZE;
*reinterpret_cast<uint4*>(group_output) =
make_uint4(packed[0], packed[1], packed[2], packed[3]);
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaTriggerProgrammaticLaunchCompletion();
#endif
}
}
// The register path needs group_size 128 with 16-bit input so that one group is
// exactly THREADS_PER_GROUP(8) x VEC_SIZE(16) elements, and it loads/stores
// through uint4, so both buffers must be 16 B aligned.
inline bool CanUseRegisterPath(const torch::stable::Tensor& input,
const torch::stable::Tensor& output_q,
int64_t group_size) {
const auto dtype = input.scalar_type();
if (group_size != 128) return false;
if (dtype != torch::headeronly::ScalarType::Half &&
dtype != torch::headeronly::ScalarType::BFloat16) {
return false;
}
if (!input.is_contiguous() || !output_q.is_contiguous()) return false;
const auto in_addr = reinterpret_cast<uintptr_t>(input.data_ptr());
const auto out_addr = reinterpret_cast<uintptr_t>(output_q.data_ptr());
return (in_addr % 16 == 0) && (out_addr % 16 == 0);
}
inline int GetGroupsPerBlock(int64_t num_groups) {
if (num_groups % 16 == 0) {
return 16;
@@ -235,6 +385,15 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
const int scale_num_rows = output_s.size(1);
const int scale_stride = output_s.stride(1);
const bool use_register_path =
CanUseRegisterPath(input, output_q, group_size);
constexpr int REG_THREADS_PER_GROUP = 8;
constexpr int REG_BLOCK_THREADS = 256;
const int64_t reg_groups_per_block =
REG_BLOCK_THREADS / REG_THREADS_PER_GROUP;
const int reg_num_blocks = static_cast<int>(
(num_groups + reg_groups_per_block - 1) / reg_groups_per_block);
#ifndef USE_ROCM
#define LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, CHECK_BOUNDS, \
SMEM_BYTES) \
@@ -281,21 +440,72 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
} \
} while (0)
#ifndef USE_ROCM
#define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \
do { \
cudaLaunchConfig_t config = {}; \
config.gridDim = dim3(reg_num_blocks); \
config.blockDim = dim3(REG_BLOCK_THREADS); \
config.dynamicSmemBytes = 0; \
config.stream = stream; \
cudaLaunchAttribute attrs[1]; \
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
config.numAttrs = 1; \
config.attrs = attrs; \
cudaLaunchKernelEx( \
&config, \
per_token_group_quant_8bit_register_kernel<T, DST_DTYPE, COL_MAJOR, \
UE8M0>, \
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
static_cast<float*>(output_s.data_ptr()), \
static_cast<int64_t>(num_groups), (float)eps, (float)min_8bit, \
(float)max_8bit, scale_num_rows, scale_stride); \
} while (0)
#else
#define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \
do { \
per_token_group_quant_8bit_register_kernel<T, DST_DTYPE, COL_MAJOR, \
UE8M0> \
<<<dim3(reg_num_blocks), dim3(REG_BLOCK_THREADS), 0, stream>>>( \
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
static_cast<float*>(output_s.data_ptr()), \
static_cast<int64_t>(num_groups), (float)eps, (float)min_8bit, \
(float)max_8bit, scale_num_rows, scale_stride); \
} while (0)
#endif
#define LAUNCH_KERNEL(T, DST_DTYPE) \
do { \
size_t smem_bytes = \
static_cast<size_t>(groups_per_block) * group_size * sizeof(T); \
if (is_column_major) { \
if (scale_ue8m0) { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, true, smem_bytes); \
if (use_register_path) { \
LAUNCH_REGISTER_INST(T, DST_DTYPE, true, true); \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, true, smem_bytes); \
} \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, false, smem_bytes); \
if (use_register_path) { \
LAUNCH_REGISTER_INST(T, DST_DTYPE, true, false); \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, false, smem_bytes); \
} \
} \
} else { \
if (scale_ue8m0) { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, true, smem_bytes); \
if (use_register_path) { \
LAUNCH_REGISTER_INST(T, DST_DTYPE, false, true); \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, true, smem_bytes); \
} \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, false, smem_bytes); \
if (use_register_path) { \
LAUNCH_REGISTER_INST(T, DST_DTYPE, false, false); \
} else { \
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, false, smem_bytes); \
} \
} \
} \
} while (0)
@@ -314,6 +524,7 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
#undef LAUNCH_KERNEL
#undef LAUNCH_KERNEL_INST
#undef LAUNCH_KERNEL_IMPL
#undef LAUNCH_REGISTER_INST
}
// Register-resident fast path for group_size==128.
-18
View File
@@ -327,21 +327,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// BF16/FP32 x FP32 -> FP32 router GEMM for H=3072, E=256, M<=32 (SM90+).
// conditionally compiled so impl registration is in source file
ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
ops.def(
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "
"Tensor? b_zeros, "
"bool has_zp, Tensor! b_qweight_reorder, Tensor! b_scales_reorder, "
"Tensor!? b_zeros_reorder, "
"int K, int N, int N_32align) -> ()");
// AllSpark quantization ops
ops.def(
"allspark_w8a16_gemm(Tensor a, Tensor b_qweight, Tensor b_scales, "
"Tensor? b_qzeros, "
"SymInt n, SymInt group_size, SymInt sm_count, SymInt sm_version, SymInt "
"CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) -> Tensor");
#endif
// Merge attn states
@@ -794,9 +779,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
// DSV3 fused A GEMM: conditionally compiled so impl registration is in
// source file (dsv3_fused_a_gemm.cu)
// AllSpark ops: conditionally compiled so impl registrations are in source
// files (allspark_repack.cu and allspark_qgemm_w8a16.cu)
#endif
ops.impl("merge_attn_states", TORCH_BOX(&merge_attn_states));
+20 -20
View File
@@ -207,7 +207,7 @@ ARG PYTORCH_NIGHTLY
# Check whether to install torch nightly instead of release for this build
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY pyproject.toml pyproject.toml
RUN --mount=type=cache,target=/opt/uv/cache \
--mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \
@@ -229,7 +229,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
fi \
&& if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
echo "Installing Rubin prerelease requirements..." \
&& /opt/venv/bin/python3 use_existing_torch.py --prefix \
&& /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt \
@@ -243,7 +243,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
&& uv pip install --python /opt/venv/bin/python3 torch torchaudio torchvision --pre \
--index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
&& echo "Installing other requirements..." \
&& /opt/venv/bin/python3 use_existing_torch.py --prefix \
&& /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
else \
@@ -316,12 +316,12 @@ COPY requirements/build/rust.txt requirements/build/rust.txt
RUN --mount=type=cache,target=/opt/uv/cache \
uv pip install --python /opt/venv/bin/python3 -r requirements/build/rust.txt
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed
# by the wheel build stage.
COPY rust rust
COPY rust-toolchain.toml rust-toolchain.toml
COPY tools/build_rust.py tools/build_rust.py
COPY build_rust.sh build_rust.sh
COPY tools/build_rust.sh tools/build_rust.sh
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
@@ -341,7 +341,7 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
export RUSTC_WRAPPER=sccache; \
sccache --show-stats; \
fi \
&& SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh \
&& SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh \
&& if [ "$USE_SCCACHE" = "1" ]; then \
sccache --show-stats; \
fi
@@ -352,7 +352,7 @@ FROM rust-build-cache AS rust-build
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
--mount=type=bind,source=.git,target=.git \
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh
#################### RUST BUILD IMAGE ####################
#################### CSRC BUILD IMAGE ####################
@@ -368,7 +368,7 @@ ARG PYTORCH_NIGHTLY
# Install build dependencies
COPY requirements/build/cuda.txt requirements/build/cuda.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
@@ -381,11 +381,11 @@ ENV UV_LINK_MODE=copy
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
echo "Installing build requirements with Rubin prerelease PyTorch..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \
elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing build requirements without torch..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \
&& echo "Installing torch nightly..." \
&& uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \
@@ -427,7 +427,7 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build"
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${PYTORCH_NIGHTLY}" = "1" ] || \
[ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
python3 use_existing_torch.py --prefix; \
python3 tools/use_existing_torch.py --prefix; \
fi
# Provision one bare Python per `requires-python` entry; cmake reads
@@ -605,7 +605,7 @@ ARG PYTORCH_NIGHTLY
# Install build dependencies
COPY requirements/build/cuda.txt requirements/build/cuda.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
@@ -618,11 +618,11 @@ ENV UV_LINK_MODE=copy
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
echo "Installing build requirements with Rubin prerelease PyTorch..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \
elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing build requirements without torch..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \
&& echo "Installing torch nightly..." \
&& uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \
@@ -658,7 +658,7 @@ ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${PYTORCH_NIGHTLY}" = "1" ] || \
[ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
python3 use_existing_torch.py --prefix; \
python3 tools/use_existing_torch.py --prefix; \
fi \
&& if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
sed -Ei '/^[[:space:]]*(flashinfer-python|nvidia-cutlass-dsl\[cu13\]|quack-kernels)[[:space:]]*==/Id' \
@@ -729,12 +729,12 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing dev requirements plus torch nightly..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& cat torch_lib_versions.txt >> requirements/test/cuda.in \
&& uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
@@ -999,7 +999,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
# https://docs.flashinfer.ai/installation.html
# From versions.json: .flashinfer.version
ARG FLASHINFER_VERSION=0.6.18.post1
ARG FLASHINFER_VERSION=0.7.0
RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
@@ -1143,14 +1143,14 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
if [ "$CUDA_MAJOR" -ge 12 ]; then \
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing dev requirements plus torch nightly..." \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& cat torch_lib_versions.txt >> requirements/test/cuda.in \
&& uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
+16 -88
View File
@@ -107,61 +107,6 @@ ENV UV_LINK_MODE="copy"
ENV TARGETARCH=${TARGETARCH}
######################### TRITON-CPU BUILD IMAGE #########################
# Branches off base-common directly (before requirements/cpu.txt is installed
# below) so a torch/etc. version bump doesn't invalidate this stage's cache --
# it builds triton-cpu in an isolated `uv build` environment and never touches
# these packages.
FROM base-common AS vllm-triton-cpu-build
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
# Re-declared here because this stage is `FROM base-common` (not `vllm-build`),
# so it does not inherit the ARG/ENV defined there. Without it, the guard
# below would see an empty value and build triton-cpu on non-x86 targets
# (e.g. arm64).
ARG VLLM_CPU_X86=0
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
ARG SCCACHE_LOCAL_ONLY=0
WORKDIR /vllm-workspace
RUN mkdir dist
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=cache,target=/root/.cache/ccache \
--mount=type=cache,target=/root/.cache/sccache-local,sharing=locked \
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
--mount=type=cache,target=/root/.triton \
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
set -ex; \
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
git clone --recurse-submodules --shallow-submodules --filter=blob:none "https://github.com/triton-lang/triton-cpu.git"; \
cd triton-cpu; \
git checkout "270e696d"; \
if [ ! -f third_party/sleef/CMakeLists.txt ]; then \
echo "sleef submodule empty after shallow fetch; retrying unshallow"; \
git submodule foreach --recursive 'git fetch --unshallow origin || true'; \
git submodule update --init --recursive --force; \
fi; \
test -f third_party/sleef/CMakeLists.txt || { echo "FATAL: sleef submodule unavailable"; exit 1; }; \
if [ "$USE_SCCACHE" = "1" ]; then \
if [ "$SCCACHE_LOCAL_ONLY" = "1" ]; then \
unset SCCACHE_BUCKET SCCACHE_REGION; \
export SCCACHE_DIR=/root/.cache/sccache-local SCCACHE_CACHE_SIZE=20G; \
elif [ -n "${SCCACHE_ENDPOINT}" ]; then \
export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; \
fi; \
export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
sccache --show-stats; \
fi; \
uv build --wheel --out-dir=../dist; \
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
cd ..; \
rm -rf triton-cpu; \
fi
######################### COMMON BASE IMAGE + PYTHON DEPS #########################
FROM base-common AS base-common-pydeps
@@ -174,10 +119,10 @@ ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
# Copy requirements files for installation
COPY requirements/common.txt requirements/common.txt
COPY requirements/cpu.txt requirements/cpu.txt
COPY use_existing_torch.py use_existing_torch.py
COPY tools/use_existing_torch.py tools/use_existing_torch.py
# For nightly: install torch nightly (CPU) first, strip torch pins from the
# requirements via use_existing_torch.py so they can't pull it back to a release
# requirements via tools/use_existing_torch.py so they can't pull it back to a release
# build, then install the rest from the nightly CPU index (no --torch-backend,
# which would force the stable channel). Release path is unchanged.
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -186,7 +131,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
echo "Installing torch nightly (CPU)..." \
&& uv pip install torch torchaudio torchvision torchcodec --pre \
--index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu \
&& python3 use_existing_torch.py --prefix \
&& python3 tools/use_existing_torch.py --prefix \
&& uv pip install -r requirements/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
@@ -238,12 +183,12 @@ ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
# Avoid port collision with vllm-build's own sccache daemon.
ENV SCCACHE_SERVER_PORT=4227
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed
# by the wheel build stage.
COPY rust rust
COPY rust-toolchain.toml rust-toolchain.toml
COPY tools/build_rust.py tools/build_rust.py
COPY build_rust.sh build_rust.sh
COPY tools/build_rust.sh tools/build_rust.sh
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
@@ -265,10 +210,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
fi; \
sccache --show-stats; \
fi && \
SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh && \
SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh && \
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
# build_rust.sh installed rustup here via rustup.rs; put it on PATH so the
# tools/build_rust.sh installed rustup here via rustup.rs; put it on PATH so the
# child stage below finds it on disk instead of re-downloading it.
ENV PATH="/root/.cargo/bin:${PATH}"
@@ -278,7 +223,7 @@ FROM rust-build-cache AS rust-build
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
--mount=type=bind,source=.git,target=.git \
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh
######################### SOURCE PREP IMAGE #########################
# Shared prep (source, rust artifacts, build deps) for both vllm-build and
@@ -317,7 +262,7 @@ COPY requirements/build/cpu.txt requirements/build/cpu.txt
RUN --mount=type=cache,target=/root/.cache/uv \
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix \
python3 /workspace/tools/use_existing_torch.py --prefix \
&& uv pip install -r requirements/build/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
@@ -342,7 +287,7 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
# now-unpinned dependency. Done at the end of vllm-src so vllm-build and
# vllm-dev, which both derive from it, inherit the unpinned tree.
RUN if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix; \
python3 /workspace/tools/use_existing_torch.py --prefix; \
fi
######################### BUILD IMAGE #########################
@@ -385,9 +330,12 @@ WORKDIR /vllm-workspace
COPY requirements/test/cpu.txt requirements/test/cpu.txt
# cpu.txt is compiled for x86_64, so platform markers are resolved away. Drop
# packages unavailable on aarch64 (decord, terratorch) for arm builds.
# packages unavailable on aarch64 (decord, terratorch) for arm builds. Also
# drop the generic PyPI triton pulled in transitively (via xgrammar) on x86
# builds, so it doesn't clobber the triton-cpu wheel from requirements/cpu.txt.
RUN case "$(uname -m)" in \
aarch64|arm64) sed -i '/^decord==/d; /^terratorch==/d' requirements/test/cpu.txt ;; \
x86_64) sed -i '/^triton==/d' requirements/test/cpu.txt ;; \
esac
# pytrec-eval-terrier bundles pre-C23 trec_eval sources. GCC 15 defaults to C23;
@@ -395,7 +343,7 @@ RUN case "$(uname -m)" in \
RUN --mount=type=cache,target=/root/.cache/uv \
export CFLAGS="${CFLAGS:+${CFLAGS} }-std=gnu17" && \
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix \
python3 /workspace/tools/use_existing_torch.py --prefix \
&& uv pip install -r requirements/test/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
@@ -440,14 +388,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
fi && \
pre-commit install --hook-type pre-commit --hook-type commit-msg
# Installed after requirements/test/cpu.txt (which pins triton==3.6.0) so the
# triton-cpu wheel isn't overwritten by that pin.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \
if ls dist/*.whl >/dev/null 2>&1; then \
uv pip install dist/*.whl; \
fi
ENTRYPOINT ["bash"]
######################### TEST IMAGE #########################
@@ -459,12 +399,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \
uv pip install dist/*.whl
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \
if ls dist/*.whl >/dev/null 2>&1; then \
uv pip install dist/*.whl; \
fi
ADD ./tests/ ./tests/
ADD ./examples/ ./examples/
ADD ./benchmarks/ ./benchmarks/
@@ -494,13 +428,7 @@ WORKDIR /vllm-workspace
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=cache,target=/root/.cache/ccache \
--mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \
uv pip install "$(realpath dist/*.whl)[audio]"
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \
if ls dist/*.whl >/dev/null 2>&1; then \
uv pip install dist/*.whl; \
fi
uv pip install "$(realpath dist/*.whl)[audio,bench]"
# Add labels to document build configuration
LABEL org.opencontainers.image.title="vLLM CPU"
+2 -2
View File
@@ -36,10 +36,10 @@ COPY pyproject.toml ./
# The script is expected to install whatever python dependencies are missing
# as well as whatever system libraries need to be installed from source
COPY build_vllm_*.sh ./
COPY docker/build_vllm_*.sh docker/
RUN --mount=type=cache,target=/root/.cache/uv \
sh ./build_vllm_$(uname -m).sh
sh ./docker/build_vllm_$(uname -m).sh
# copy vllm source code to build cache
COPY . .
+16 -8
View File
@@ -135,7 +135,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
cd ${COMMON_WORKDIR}/vllm \
&& uv pip install --system -r requirements/build/rust.txt \
&& python3 -c \
"from setuptools_scm import get_version; print(get_version())" \
"from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \
> /vllm-version.txt
# -----------------------
@@ -171,7 +171,7 @@ COPY rust/proto /rust-input/vllm/rust/proto
COPY rust/src /rust-input/vllm/rust/src
COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
COPY build_rust.sh /rust-input/vllm/build_rust.sh
COPY tools/build_rust.sh /rust-input/vllm/tools/build_rust.sh
FROM base AS rust_input_1
ARG VLLM_REPO
@@ -188,7 +188,7 @@ RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
rust/src/** \
rust-toolchain.toml \
tools/build_rust.py \
build_rust.sh \
tools/build_rust.sh \
&& git checkout FETCH_HEAD
FROM rust_input_${REMOTE_VLLM} AS rust-input
@@ -244,7 +244,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
COPY --from=rust-input /rust-input/vllm/rust rust
COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
COPY --from=rust-input /rust-input/vllm/tools/build_rust.sh tools/build_rust.sh
COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt
# Build the release binary. Cargo's registry/git caches can be written by
@@ -258,7 +258,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
&& sccache --show-stats; \
fi \
&& export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \
&& bash build_rust.sh \
&& bash tools/build_rust.sh \
&& test -x vllm/vllm-rs \
&& if [ "$USE_SCCACHE" = "1" ]; then \
sccache --show-stats; \
@@ -368,7 +368,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
@@ -729,7 +729,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
@@ -776,13 +776,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
uv pip install --system /nixl_install/*.whl /deep_install/*.whl
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
&& apt-get install -q -y --no-install-recommends \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
librdmacm1 \
libibverbs1 \
ibverbs-providers \
ibverbs-utils \
util-linux \
unzip \
libsodium23 \
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
&& rm -rf /var/lib/apt/lists/*
+37 -7
View File
@@ -3,7 +3,9 @@ ARG PYTHON_VERSION=3.12
ARG TORCH_VERSION=2.12.0+rocm10.0.0
ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0
ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0
ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0
ARG TRITON_BRANCH="c0a142ffcbb42f9a67a56bf791cad472be1d6425" # release/internal/3.8.x as of 9/21/2026
ARG TRITON_VERSION=3.8.0+gitc0a142ff
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
ARG AITER_BRANCH="v0.1.22.post1"
@@ -90,19 +92,15 @@ RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
&& rocm-sdk init
# Install PyTorch wheels (nightly for latest torch, release as fallback)
# Triton is pinned here rather than left to torch's Requires-Dist, so a wheel bump that
# moves the triton torch wants fails resolution instead of quietly changing compilers.
ARG TORCH_VERSION
ARG TORCHAUDIO_VERSION
ARG TORCHVISION_VERSION
ARG TRITON_VERSION
RUN pip install \
--index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
--extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
"torch==${TORCH_VERSION}" \
"torchaudio==${TORCHAUDIO_VERSION}" \
"torchvision==${TORCHVISION_VERSION}" \
"triton==${TRITON_VERSION}"
"torchvision==${TORCHVISION_VERSION}"
# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH:
@@ -257,6 +255,20 @@ RUN pip install pyyaml && cd aiter \
RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
###
### Triton Build
###
FROM base AS build_triton
ARG TRITON_BRANCH
ARG TRITON_REPO
RUN git clone ${TRITON_REPO}
RUN cd triton \
&& git checkout ${TRITON_BRANCH} \
&& if [ ! -f setup.py ]; then cd python; fi \
&& python3 setup.py bdist_wheel --dist-dir=dist \
&& mkdir -p /app/install && cp dist/*.whl /app/install
###
### ROCr + CLR build
###
@@ -320,6 +332,8 @@ RUN mkdir -p /staging/lib \
# only includes dependencies used by wheel release pipeline
FROM base AS debs_wheel_release
RUN mkdir /app/debs
RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
cp /install/*.whl /app/debs
RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
cp /install/*.whl /app/debs
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
@@ -327,6 +341,8 @@ RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
FROM base AS debs
RUN mkdir /app/debs
RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
cp /install/*.whl /app/debs
RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
cp /install/*.whl /app/debs
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
@@ -335,6 +351,7 @@ RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
cp /install/*.whl /app/debs
FROM base AS final
ARG TRITON_VERSION
# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA
# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix.
@@ -359,7 +376,15 @@ RUN set -eux; \
RUN ldconfig
ARG TRITON_VERSION
# This is to prevent resolvers from checking METADATA and RECORD and clobbering
# the Triton source build we installed. Constraints files are unable to help
# with this. However, these entries are optional and can be blanked which
# solves the problem.
RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \
&& grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \
&& sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \
&& sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD"
RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
pip install /install/*.whl \
&& ! pip check 2>&1 | grep -iE '^(torch|triton) ' \
@@ -383,6 +408,8 @@ ARG TORCHAUDIO_VERSION
ARG TORCHVISION_VERSION
ARG PYTORCH_ROCM_ARCH
ARG AITER_ROCM_ARCH
ARG TRITON_BRANCH
ARG TRITON_REPO
ARG FA_BRANCH
ARG FA_REPO
ARG AITER_BRANCH
@@ -391,6 +418,7 @@ ARG MORI_BRANCH
ARG MORI_REPO
ARG ROCM_SYSTEMS_REPO
ARG ROCM_RUNTIME_COMMIT
ARG TRITON_VERSION
# Record hashes for Triton and pybind11 so we can verify later.
RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
&& echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \
@@ -400,6 +428,8 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
&& echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \
&& echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \
&& echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \
&& echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
&& echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
&& echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
&& echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
&& echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
+17 -11
View File
@@ -5,7 +5,7 @@ ARG VLLM_BRANCH="main"
ARG COMMON_WORKDIR=/app
ARG BASE_IMAGE=rocm/vllm-dev:base
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7
ARG ROCM_TRITON_KERNELS_COMMIT=669b31acc1dd1b3fd93286afd5db67f65d9f7557
ARG INSTALL_LMCACHE=false
# NIC backend for MoRI RDMA support.
# By default (all), drivers and userspace libraries for all supported NIC types
@@ -135,7 +135,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
cd ${COMMON_WORKDIR}/vllm \
&& uv pip install --system -r requirements/build/rust.txt \
&& python3 -c \
"from setuptools_scm import get_version; print(get_version())" \
"from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \
> /vllm-version.txt
# -----------------------
@@ -171,7 +171,7 @@ COPY rust/proto /rust-input/vllm/rust/proto
COPY rust/src /rust-input/vllm/rust/src
COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
COPY build_rust.sh /rust-input/vllm/build_rust.sh
COPY tools/build_rust.sh /rust-input/vllm/tools/build_rust.sh
FROM base AS rust_input_1
ARG VLLM_REPO
@@ -188,7 +188,7 @@ RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
rust/src/** \
rust-toolchain.toml \
tools/build_rust.py \
build_rust.sh \
tools/build_rust.sh \
&& git checkout FETCH_HEAD
FROM rust_input_${REMOTE_VLLM} AS rust-input
@@ -244,7 +244,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
COPY --from=rust-input /rust-input/vllm/rust rust
COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
COPY --from=rust-input /rust-input/vllm/tools/build_rust.sh tools/build_rust.sh
COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt
# Build the release binary. Cargo's registry/git caches can be written by
@@ -258,7 +258,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
&& sccache --show-stats; \
fi \
&& export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \
&& bash build_rust.sh \
&& bash tools/build_rust.sh \
&& test -x vllm/vllm-rs \
&& if [ "$USE_SCCACHE" = "1" ]; then \
sccache --show-stats; \
@@ -366,8 +366,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
@@ -749,8 +748,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
@@ -799,13 +797,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
&& apt-get install -q -y --no-install-recommends \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
librdmacm1 \
libibverbs1 \
ibverbs-providers \
ibverbs-utils \
util-linux \
unzip \
libsodium23 \
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
&& rm -rf /var/lib/apt/lists/*
+3 -3
View File
@@ -1,9 +1,9 @@
ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete
ARG TRITON_BRANCH="f0b55c0" # release/internal/3.7.x as of 08/18
ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
ARG PYTORCH_BRANCH="6bbd260" # release/2.12 as of 08/02
ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17
ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git"
ARG PYTORCH_VISION_BRANCH="v0.27.1"
ARG PYTORCH_VISION_BRANCH="v0.28.0"
ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
+11 -3
View File
@@ -136,7 +136,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
cd ${COMMON_WORKDIR}/vllm \
&& uv pip install --system -r requirements/build/rust.txt \
&& python3 -c \
"from setuptools_scm import get_version; print(get_version())" \
"from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \
> /vllm-version.txt
# -----------------------
@@ -166,7 +166,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
cd ${COMMON_WORKDIR}/vllm \
&& export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \
&& bash build_rust.sh \
&& bash tools/build_rust.sh \
&& test -x vllm/vllm-rs
# -----------------------
@@ -560,11 +560,19 @@ RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \
COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
&& apt-get install -q -y --no-install-recommends \
-o Acquire::Retries=3 \
-o Acquire::http::Timeout=30 \
-o Acquire::https::Timeout=30 \
librdmacm1 \
libibverbs1 \
ibverbs-providers \
ibverbs-utils \
libsodium23 \
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
&& rm -rf /var/lib/apt/lists/*
+22 -4
View File
@@ -14,12 +14,12 @@ WORKDIR /workspace
COPY requirements/build/rust.txt requirements/build/rust.txt
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed
# by the wheel build stage.
COPY rust rust
COPY rust-toolchain.toml rust-toolchain.toml
COPY tools/build_rust.py tools/build_rust.py
COPY build_rust.sh build_rust.sh
COPY tools/build_rust.sh tools/build_rust.sh
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
@@ -29,7 +29,7 @@ ENV CARGO_BUILD_JOBS=4
# inputs. The child stage below relinks the crates that embed the exact version.
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh
SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh
# Relink with the exact Git-derived package version.
FROM rust-build-cache AS rust-build
@@ -37,7 +37,7 @@ FROM rust-build-cache AS rust-build
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
--mount=type=bind,source=.git,target=.git \
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh
SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh
FROM ubuntu:24.04 AS vllm-base
@@ -87,11 +87,14 @@ RUN mkdir neo && \
# Install XPU-Manager payloads without registering conflicting libze package
# dependencies in dpkg.
# Note that ocl-icd-opencl-dev is required by DLE upgrade but xpu-smi, should
# can be removed in future.
RUN apt-get update -y && \
apt-get install -y --no-install-recommends software-properties-common && \
add-apt-repository -y ppa:kobuk-team/intel-graphics && \
apt-get update -y && \
apt-get install -y --no-install-recommends libhwloc15 libpciaccess0 libigsc1 intel-metrics-library && \
apt-get install -y ocl-icd-opencl-dev && \
mkdir xpu-m && \
cd xpu-m && \
wget https://github.com/intel/xpumanager/releases/download/v2.1.0/xpu-smi_2.1.0+26.33.6468cec-1.24.04_amd64.deb && \
@@ -264,4 +267,19 @@ RUN --mount=type=cache,target=/root/.cache/uv \
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install -e tests/vllm_test_utils
# Replace 2026.1.2 DLE packages to 2026.1.3
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=.git,target=.git \
uv pip install \
"intel-cmplr-lib-rt==2026.1.1" \
"intel-cmplr-lib-ur==2026.1.1" \
"intel-cmplr-lic-rt==2026.1.1" \
"intel-sycl-rt==2026.1.1" \
"oneccl-devel==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \
"oneccl==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \
"dpcpp-cpp-rt==2026.1.1" \
"intel-opencl-rt==2026.1.1" \
"intel-openmp==2026.1.1" \
"intel-pti==1.1.0"
ENTRYPOINT ["vllm", "serve"]
+1 -1
View File
@@ -71,7 +71,7 @@
"default": "true"
},
"FLASHINFER_VERSION": {
"default": "0.6.18.post1"
"default": "0.7.0"
},
"GDRCOPY_CUDA_VERSION": {
"default": "12.8"
+28 -1
View File
@@ -9,7 +9,7 @@ vllm --help
Available Commands:
```bash
vllm {chat,complete,serve,launch,bench,collect-env,run-batch}
vllm {chat,complete,serve,launch,bench,collect-env,run-batch,preload}
```
## serve
@@ -212,6 +212,33 @@ vllm run-batch \
See [vllm run-batch](./run-batch.md) for the full reference of all available arguments.
## preload
Launch weight cache daemons (one per GPU) that hold the post-quantized,
TP-sharded weights in GPU memory and serve CUDA IPC handles to vLLM engines
over a Unix domain socket. Restarting engines then map the weights via
zero-copy IPC instead of reloading from disk, enabling fast engine restarts.
```bash
# Launch one daemon per GPU
vllm preload --model meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4
# Engines then load from the daemons
vllm serve meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4 \
--load-format ipc_cache
```
The daemon accepts the standard engine arguments (model, dtype, quantization,
tensor-parallel-size, ...) plus `--weight-cache-socket-dir` to override the
directory holding the per-GPU Unix sockets, and `--weight-cache-master-port` /
`--weight-cache-draft-master-port` to pin the daemon rendezvous ports for
multi-node and speculative-decoding setups. Tensor, expert and data
parallelism are supported; pipeline parallelism is rejected at launch.
See [Preload](../features/preload.md) for how it works, cache modes, and
limitations, and [vllm preload](./preload.md) for the full reference of all
available arguments.
## More Help
For detailed options of any subcommand, use:
+7 -2
View File
@@ -46,8 +46,8 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e .
To rebuild only the Rust frontend binary:
```bash
./build_rust.sh # release build
./build_rust.sh --debug # faster build for development
./tools/build_rust.sh # release build
./tools/build_rust.sh --debug # faster build for development
```
If you are developing vLLM's Python and CUDA/C++ code, install PyTorch first:
@@ -89,6 +89,11 @@ pre-commit install
vLLM's `pre-commit` hooks will now run automatically every time you commit.
When running ShellCheck or markdownlint directly, or configuring an editor
integration, use the configuration files in `tools/pre_commit/`:
`--rcfile=tools/pre_commit/.shellcheckrc` for ShellCheck and
`--config tools/pre_commit/.markdownlint.yaml` for markdownlint-cli2.
!!! tip "Tips"
You can manually run the `pre-commit` hooks using:
+2
View File
@@ -172,6 +172,8 @@ class AttentionCGSupport(enum.Enum):
Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation mode. For the complete fallback policy, please see the code for [this][vllm.v1.worker.gpu_model_runner.GPUModelRunner._check_and_update_cudagraph_mode].
Variable-length decode batches, where each request carries a different number of query tokens read from the device `query_start_loc` (as in adaptive verification), are declared separately and not ordered against the enum: `AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len()` returns the largest per-request query length a FULL decode graph can replay. It returns `None` for builders reporting `ALWAYS`, which replay any batch, and for builders that cannot replay variable-length batches; FlashInfer returns `1 + num_speculative_tokens` when its TRTLLM-GEN varlen decode path is active. Batches with a prefill never replay these graphs.
The following table lists backends that support full CUDA Graphs at the time of writing.
| Attention Backend | cudagraph_support | Comments |
+23 -17
View File
@@ -42,34 +42,39 @@ When new data is passed in, we first check which items are in the cache, and whi
### Fused Normalisation on the Device
To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the heavy numerical preprocessing from the CPU to the GPU and optimise data movement.
To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the **normalisation and rescaling** steps from the CPU to the device and optimises data movement.
#### Fusing Normalisation and Rescaling on the GPU
Traditionally, the CPU would divide pixel values by 255, then subtract the mean and divide by the standard deviation. We fuse these steps into one operation and run it entirely on the GPU.
Traditionally, the CPU would divide pixel values by 255, then subtracts the mean and divides by the standard deviation. We fuse these steps into a single per-channel affine transform on the device.
**How it works**: We use a dedicated `FusedInputNorm` module that bakes the rescale factor (1/255) directly into the layer's `weight` and `bias` parameters. Instead of performing three separate steps (scale, subtract, divide), the module does everything in a single affine transformation: `y = x * weight + bias`.
`FusedMMInputNorm` dispatches to `fused_mm_input_norm_triton` on CUDA, which applies:
The parameters are set as follows:
y = x * weight[c] + bias[c]
= (x * rescale_factor - image_mean[c]) / image_std[c]
- `weight` controls both the standard deviation and the rescale factor
- `bias` centers the data using the mean and the same rescale factor
with:
This means the module takes raw pixel values (0–255) and outputs properly normalised values without ever explicitly dividing by 255 as a separate step.
weight[c] = rescale_factor / image_std[c]
bias[c] = -image_mean[c] / image_std[c]
#### Optimized Data Path for Fused Normalisation
The kernel accepts `uint8`, `float16`, `bfloat16` and `float32` inputs. The `uint8` path is the primary fast path: raw bytes travel to the device unprocessed, and the rescale factor is folded into `weight` — no separate divide-by-255 step. Compute is always done in fp32 inside the kernel.
Performing fused normalisation directly on the device allows us to keep the entire transfer path—from **Entrypoint** through **Engine Core** to **GPU memory**—in **`uint8`**. This halves PCIe bandwidth and reduces CPU memory footprint.
#### Optimized Data Path
Only after data reaches GPU memory do we cast to `fp32` for the `FusedInputNorm` layer (to ensure numerical accuracy), then cast to `bf16` for subsequent layers—all within the GPU, avoiding any host‑side conversions.
Before: CPU decode → CPU resize → CPU rescale (÷255) → CPU normalize
→ cast to bf16 → PCIe (2 B/elem) → GPU
Overall path: **`Entrypoint (uint8) → Engine Core (uint8) → GPU Memory (uint8)`** → GPU‑local `fp32` `FusedInputNorm` → `bf16` output.
After: CPU decode → CPU resize → uint8 pixel_values
→ PCIe (1 B/elem) → GPU fused affine (fp32 compute) → bf16
The transfer path **Entrypoint (API server / offline LLM) → Engine Core(scheduler + workers) → Device Memory** stays in `uint8`. On the device,`fused_mm_input_norm_triton` computes in fp32 internally and writes the requested `visual_dtype` (a per-call argument to `forward_*`, commonly `bf16`) directly, without materialising a global fp32 intermediate.
#### Toggle: `mm_device_do_normalize`
This GPU‑side fusion is controlled by a config flag called **`mm_device_do_normalize`**.
GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
- When `True`, normalisation and rescaling are done on the GPU using the `FusedInputNorm` layer; when `False`, we fall back to the old CPU‑side path.
- When `True`, normalisation and rescaling are done on the GPU using the `FusedMMInputNorm` layer; when `False`, we fall back to the old CPU‑side path.
- The flag is **enabled by default** for all models that support it.
- Currently, it’s on by default for these architectures:
@@ -78,8 +83,9 @@ This GPU‑side fusion is controlled by a config flag called **`mm_device_do_nor
| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. |
| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. |
#### What We Gain Overall
#### Key Properties and Gains
- **CPU offload**: The arithmetic for normalisation and rescaling is completely gone from the CPU.
- **PCIe savings**: Sending `uint8` (1 byte) instead of `bf16` (2 bytes) slashes data transfer volume by **50%** .
- **GPU overhead**: The fused kernel is very lightweight and can often be merged with subsequent CUDA operations, so it hardly adds any extra cost.
- **CPU offload & 50% PCIe savings** — normalisation/rescaling leaves the CPU entirely; sending `uint8` (1 byte) instead of `bf16` (2 bytes) halves transfer volume.
- **Single-pass fusion** — `y = x * weight[c] + bias[c]` in one kernel launch via `fused_mm_input_norm_triton`; input is read in its native dtype (`uint8`).
- **float32 compute for free** — arithmetic runs in fp32 inside the kernel regardless of I/O dtypes. Accuracy matches fp32, with no extra bandwidth: intermediates stay in registers, and no global fp32 tensor is materialised.
- **Preallocated output with batch-dim padding** — callers can pass a larger buffer; only the leading `N` rows are written, so buffers are reusable across calls.
+6
View File
@@ -27,6 +27,12 @@ With all these factors taken into consideration, usually we can guarantee that t
A unique aspect of vLLM's `torch.compile` integration, is that we guarantee all the compilation finishes before we serve any requests. No requests will trigger new compilations. Otherwise, the engine would be blocked on that request, and the response time will have unexpected spikes.
An explicit `TRITON_CACHE_DIR` is preserved, allowing processes and model
configurations to reuse Triton artifacts. The directory can start empty: the
first compilation populates it and later matching specializations reuse it.
When copying compilation caches, also copy this directory if set. Cold-cache
tests should set it to a fresh directory.
By default, the cache saves compiled artifacts as binary files. If you would like to interact with the generated code for debugging purposes, set the field `compile_cache_save_format=unpacked` in the compilation config, or omit this and set the env variable `VLLM_COMPILE_CACHE_SAVE_FORMAT=unpacked`.
## Dynamic shapes and vllm guard dropping
+4 -1
View File
@@ -144,7 +144,10 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
!!! note
Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility.
+13
View File
@@ -61,6 +61,19 @@ and compares the disaggregated outputs against it. Set `MODEL` to use another mo
or `MOONCAKE_EC_PROTOCOL=rdma` to use RDMA on supported hardware. See the script for
additional configuration options.
### Audio inputs
The Python EPD proxy also rewrites audio in `/v1/chat/completions` into
metadata-only references for Qwen2-Audio, AudioFlamingo3, Ultravox,
Qwen2.5-Omni, and Qwen3-Omni. The encoder publishes `audio_num_tokens`
(the placeholder token count of each audio) together with any feature
lengths the model needs; the consumer uses them to reconstruct the audio
placeholders and receives the embeddings through its EC connector without
repeating audio preprocessing.
This covers pure audio inputs, not video with an embedded audio track. It does
not add `/v1/audio/transcriptions` or realtime routes to the example proxy.
---
## 3 Test Script
+203
View File
@@ -0,0 +1,203 @@
# Preload
vLLM's preload feature keeps model artifacts resident in GPU memory across
engine restarts, so a restarting engine reuses them instead of rebuilding them
from scratch. The design is extensible to different kinds of artifacts; it
currently supports **model weights** via the weight cache daemon.
With weight preloading, a daemon process per GPU holds its rank's
post-quantized, TP-sharded weights and serves CUDA IPC handles to vLLM engines
over a Unix domain socket, so a restarting engine maps the weights via
zero-copy IPC instead of reloading them from disk.
Key benefits:
- **Fast engine restarts**: weight loading reduces to mapping existing GPU
memory, cutting restart time from minutes (disk load + quantization
processing) to seconds.
- **Zero-copy sharing**: in the default `zero_copy` mode the engine directly
shares the daemon's GPU memory, so a restart adds no extra GPU memory
footprint.
- **Safe fallback**: if the daemon is unavailable or its cached weights don't
match the engine's configuration, the engine falls back to loading from
disk.
!!! note
Weight preloading is only supported on CUDA and ROCm platforms. Tensor,
expert and data parallelism are supported, including across nodes;
pipeline parallelism is rejected when launching the daemon.
## Quick start
Launch one weight cache daemon per GPU with a single command:
```bash
vllm preload --model meta-llama/Llama-3.1-8B-Instruct --tensor-parallel-size 4
```
The daemon process loads and post-processes the weights once, then waits for
engines. Each rank binds its Unix socket only after its shard is fully cached,
so engines connecting before that simply fall back to disk loading.
Then start (or restart) engines with the `ipc_cache` load format:
```bash
vllm serve meta-llama/Llama-3.1-8B-Instruct \
--tensor-parallel-size 4 \
--load-format ipc_cache
```
The daemon itself must load from disk; passing `--load-format ipc_cache` to
`vllm preload` is an error.
## How it works
1. `vllm preload` spawns one daemon process per GPU. Each daemon loads its
shard from disk using the configured loader, runs quantization
post-processing, and exports every parameter/buffer as a CUDA IPC handle.
2. An engine started with `--load-format ipc_cache` builds its model on the
meta device (no storage) and requests the tensors from the daemon on its
GPU.
3. Before serving anything, the engine and daemon compare a fingerprint of the
cached weights: checkpoint content (hashed from safetensors metadata, so
identical weights in different directories still match), model
architecture, TP/DP size/rank, dtype, quantization method and config, model
revision, and vLLM version. On any mismatch the engine falls back to disk
loading (unless `fallback` is disabled).
4. The engine also verifies the daemon's GPU UUID matches its own device, so a
stale socket cannot serve weights for the wrong GPU.
Tied weights (e.g. `lm_head.weight` sharing storage with
`embed_tokens.weight`) are exported once and re-established as aliases in the
engine, preserving parameter identity.
## Multi-node and data parallelism
CUDA IPC handles are node-local, so each node serves only its local GPUs'
shards. For multi-node tensor parallelism, run one `vllm preload` launcher per
node with a shared rendezvous: reuse the `--nnodes` / `--node-rank` /
`--master-addr` flags you pass the engine, plus a `--weight-cache-master-port`
distinct from the engine's `--master-port`:
```bash
# node 0 (8 local GPUs)
vllm preload --model /path/to/model --tensor-parallel-size 16 \
--nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \
--weight-cache-master-port 29600
# node 1 (8 local GPUs)
vllm preload --model /path/to/model --tensor-parallel-size 16 \
--nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \
--weight-cache-master-port 29600
```
For data parallelism (e.g. a TP1 x DP16 x EP decode fleet), run one launcher
per node with the engine's DP placement flags. Local GPU `i` serves DP rank
`start_rank + i // tp_size` and TP rank `i % tp_size`, and all
`dp_size * tp_size` daemons form one world group on `--data-parallel-address`
/ `--weight-cache-master-port` so the expert shards are laid out exactly as in
the engine:
```bash
# node r (4 local GPUs)
vllm preload --model /path/to/model --tensor-parallel-size 1 \
--enable-expert-parallel \
--data-parallel-size 16 --data-parallel-size-local 4 \
--data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \
--weight-cache-master-port 29600
```
Data parallelism also combines with multi-node tensor parallelism: pass both
flag sets. Each node then serves a contiguous block of the
`dp_size * tp_size` global ranks.
## Speculative decoding
With MTP, EAGLE or EAGLE3 speculative decoding, `vllm preload` additionally
starts a draft daemon group that caches the draft model. It uses its own cache
key, Unix sockets (`*_draft.sock`) and rendezvous port
(`--weight-cache-draft-master-port`, default `--weight-cache-master-port + 1`),
so each daemon process serves exactly one model role. Other draft types are
not cached and keep loading from disk in the engine.
## Cache modes
The loader supports two modes, selected via `--model-loader-extra-config`:
- `zero_copy` (default): the engine maps the daemon's CUDA allocations
directly. The daemon must stay alive for the engine's lifetime, and each
GPU carries only one copy of the weights.
- `copy`: the engine clones every tensor into its own GPU memory and then
asks the daemon to release its cache. Use this when the daemon should free
GPU memory after handing off, at the cost of a full copy per restart.
!!! warning
In `zero_copy` mode the weights live in the daemon's CUDA IPC allocations,
so [sleep mode](sleep_mode.md) (CuMemAllocator weight offloading) must not
be used with this loader.
## Loader configuration
The `ipc_cache` loader accepts extra keys via `--model-loader-extra-config`:
```bash
vllm serve meta-llama/Llama-3.1-8B-Instruct \
--load-format ipc_cache \
--model-loader-extra-config '{"mode": "copy", "fallback": false}'
```
| Key | Default | Description |
| --- | ------- | ----------- |
| `socket_path` | per-GPU path derived from the GPU UUID | Explicit daemon socket path. |
| `socket_dir` | per-user private dir under the temp dir | Directory containing the daemon sockets. |
| `mode` | `zero_copy` | `zero_copy` or `copy` (see above). |
| `fallback` | `true` | Fall back to disk loading when the daemon is unavailable or the fingerprints mismatch. |
| `connect_timeout_s` | `5.0` | Socket connect timeout in seconds. |
| `state_timeout_s` | `300.0` | Timeout for the weight-transfer request in seconds. |
The daemon's `--weight-cache-socket-dir` and the loader's `socket_dir` /
`socket_path` must agree when the default per-user directory is not used.
Socket paths are derived from the GPU UUID, so they are stable regardless of
`CUDA_VISIBLE_DEVICES` index remapping.
## Limitations
- **Platform**: CUDA and ROCm only; other platforms raise
`UnsupportedPlatformForIPCError` even when `fallback` is enabled, since it
is a permanent misconfiguration rather than a transient daemon outage.
- **Parallelism**: tensor, expert and data parallelism are supported;
launching the daemon with pipeline parallelism is rejected.
- **Quantization**: every quantization method in the model must declare
support for pre-processed weights (the daemon transfers weights *after*
quantization post-processing). Unsupported methods raise
`UnsupportedQuantForIPCError`.
- **Consistency**: the daemon and engine must run the same vLLM version and
agree on model, dtype, quantization, and TP/DP layout, otherwise the
fingerprint mismatch triggers the disk fallback.
- **Localhost only**: the daemon serves over a Unix domain socket; both the
daemon and the engines must run on the same node as the same user.
## Security
The socket protocol uses pickle and is intended only for trusted local
processes owned by the same user:
- Daemon sockets live in a per-user private directory (mode `0700`) and the
socket files are restricted to the owner (`0600`).
- Both sides reject symlinked or non-owned socket paths; the auto-derived
directory is additionally rejected if it is group/world accessible.
- On Linux the daemon verifies the connecting peer's UID via `SO_PEERCRED`.
See the [security documentation](../usage/security.md) for vLLM's general
threat model.
## Daemon lifecycle
- Each GPU's socket path is guarded by an exclusive lock file, so a second
daemon for the same GPU fails fast instead of clobbering the live socket.
- `vllm preload` reports readiness once every rank is serving; if any rank
dies during startup, the remaining ranks are terminated and the command
exits with that rank's exit code.
- `SIGINT`/`SIGTERM` to the `vllm preload` process terminates all daemon
ranks, which unlinks their sockets and releases the GPU memory.
See [vllm preload](../cli/preload.md) for the full CLI reference.
@@ -143,14 +143,14 @@ If you encounter any issues or have feature requests, please open an issue on th
## Online Dynamic Quantization
Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor.
Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8_per_tensor"` in the command line or setting `quantization="fp8_per_tensor"` in the LLM constructor.
In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode.
```python
from vllm import LLM
llm = LLM("facebook/opt-125m", quantization="fp8")
llm = LLM("facebook/opt-125m", quantization="fp8_per_tensor")
# INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB
result = llm.generate("Hello, my name is")
print(result[0].outputs[0].text)
+2
View File
@@ -25,12 +25,14 @@ vLLM currently supports the following reasoning models:
| [Holo2 series](https://huggingface.co/collections/Hcompany/holo2) | `holo2` | `json`, `regex` | ✅ |
| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ |
| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
| [IBM Granite 4.2 language models](https://huggingface.co/ibm-granite/granite-4.2-30b) | `granite_thinking_parser` | `json`, `regex` | ✅ |
| [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ |
| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ |
| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ |
!!! note
IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
IBM Granite 4.2 reasoning is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.
The reasoning feature for the Qwen3 series is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.
Gemma 4 reasoning is disabled by default; to enable it, pass `enable_thinking=True` in your `chat_template_kwargs` or set `reasoning_effort` (which enables it automatically).
DeepSeek-V3.1 tool calling is supported in non-thinking mode.
@@ -9,6 +9,7 @@ To train your own draft models for optimized speculative decoding, see [vllm-pro
vLLM supports a variety of methods of speculative decoding. Model-based methods such as EAGLE, MTP, draft models, PARD and MLP provide the best latency reduction, while simpler methods such as n-gram and suffix decoding provide modest speedups without increasing workload during peak traffic.
- [EAGLE](eagle.md)
- [LiLiCorr](lilicorr.md)
- [Multi-Token Prediction (MTP)](mtp.md)
- [Draft Model](draft_model.md)
- [Parallel Draft Model (PARD)](parallel_draft_model.md)
@@ -0,0 +1,77 @@
# LiLiCorr
LiLiCorr uses a DFlash backbone to produce per-position candidates, then scores
candidate combinations with a learned correlator. It requires a compatible
trained checkpoint and Model Runner V2, which is selected automatically.
Both plain and grouped-convolution checkpoints are supported.
## Usage
For a checkpoint trained with block size 16:
```bash
vllm serve /path/to/target --dtype bfloat16 \
--speculative-config '{"method":"dflash","model":"/path/to/lilicorr","num_speculative_tokens":15,"draft_sample_method":"probabilistic"}'
```
We strongly recommend explicitly setting `num_speculative_tokens` to the
checkpoint's trained `block_size - 1` (for example, `15` for `block_size=16`).
The trained block includes one anchor position. Other supported lengths are
available, but may degrade acceptance length unpredictably; benchmark them with
your checkpoint and workload before using them.
Shorter drafts, down to one token, use the corresponding prefix of the learned
slot embeddings and attention bias without changing checkpoint parameter shapes.
Lengths above `block_size - 1` are unsupported because the checkpoint has no
learned slot embeddings for those positions.
Fixed-length drafting with shorter verification is a separate option provided by
adaptive verification (`enable_adaptive_verification`). LiLiCorr shares the
candidate-sampling and acceptance-estimation path, but this combination has not
been validated end-to-end. Adaptive verification also requires compatible target
attention backends and variable-length CUDA graph support; do not assume it works
with Mamba2 targets.
## Proposal sampling
- `"draft_sample_method":"greedy"` selects the highest-scoring candidate at each
step of the correlator's conditional walk.
- `"draft_sample_method":"probabilistic"` samples from those conditional scores
using the request's temperature and supplies the realized proposal
distributions to the rejection sampler.
These settings control proposals; they do not replace the target model's
sampling settings. With request temperature zero, probabilistic proposals also
use argmax. For nonzero temperatures, benchmark both modes with your checkpoint
and workload: compare acceptance length and end-to-end throughput or latency.
Neither mode is universally faster.
The default rejection method is `"rejection_sample_method":"standard"`.
The shared Model Runner V2 rejection sampler also exposes `"block"`, but
LiLiCorr-specific GPU correctness and performance validation for that combination
is still pending. Do not assume that changing the rejection method improves
throughput; evaluate it separately from the proposal sampling mode.
## Checkpoint requirements
The checkpoint must declare `LiLiCorrDraftModel` and include all `lilicorr_*`
geometry fields and trained head weights. Geometry is read from `dflash_config`.
Convolution tensors must match the configured `conv_kernel_size` and
`conv_group_size`. Target input embeddings must be available on the draft rank.
By default, candidates use the target LM head, which must also be available there.
A checkpoint with top-level `"has_own_lm_head": true` instead uses its own
`lm_head` with the draft quantization configuration and exclusions. Its weights
must be present; for ModelOpt NVFP4 this includes `lm_head.weight`,
`lm_head.weight_scale`, and `lm_head.weight_scale_2`. The owned head is preserved
and is not replaced by the target head. Candidate token embeddings still come
from the target.
Correlator linear layers and convolution kernel projections use the draft
quantization configuration and its module exclusions. The following parameters
must remain in floating-point model dtype:
- Lattice QKV `in_proj_weight` and `in_proj_bias`, which retain the exported
parameter layout.
- `lilicorr.factor_input_proj`, `lilicorr.out_head`, and `lilicorr.in_head`, which
are used by the split factor and fused edge projections. Exclude these modules
when exporting a quantized checkpoint.
@@ -111,8 +111,8 @@ This command will do the following:
If you need to recompile the `vllm-rs` Rust frontend binary, you can rebuild and install it without re-running the full pip install:
```bash
./build_rust.sh # release build
./build_rust.sh --debug # faster build for development
./tools/build_rust.sh # release build
./tools/build_rust.sh --debug # faster build for development
```
This will install the required Rust toolchain if needed, build the binary, and place it in `vllm/vllm-rs`.
@@ -183,7 +183,7 @@ To build vLLM using an existing PyTorch installation:
# install PyTorch first, either from PyPI or from source
git clone https://github.com/vllm-project/vllm.git
cd vllm
python use_existing_torch.py
python tools/use_existing_torch.py
uv pip install -r requirements/build/cuda.txt
uv pip install --no-build-isolation -e .
```

Some files were not shown because too many files have changed in this diff Show More