diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml index c2da292a83..baeb1af145 100644 --- a/.buildkite/ci_config_rocm.yaml +++ b/.buildkite/ci_config_rocm.yaml @@ -21,7 +21,7 @@ run_all_patterns: - "requirements/test/rocm.txt" - "pyproject.toml" - "setup.py" - - "build_rust.sh" + - "tools/build_rust.sh" - "rust/" - "rust-toolchain.toml" - "tools/build_rust.py" diff --git a/.buildkite/image_build/image_build_torch_nightly.sh b/.buildkite/image_build/image_build_torch_nightly.sh index 8aa313d0e6..56c7116e14 100755 --- a/.buildkite/image_build/image_build_torch_nightly.sh +++ b/.buildkite/image_build/image_build_torch_nightly.sh @@ -49,7 +49,16 @@ echo "Image not found, proceeding with build..." # Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9 NIGHTLY_CUDA_VERSION="13.0.2" NIGHTLY_BUILD_BASE_IMAGE="pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359" -NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04" +# Must track ARG UBUNTU_VERSION in docker/Dockerfile. The nightly lane exists to +# run the normal pipeline against nightly torch, so it has to differ from the +# regular image in torch only -- not in the OS underneath it. This was pinned to +# 22.04 while everything else moved to 24.04, and the runtime dependency list in +# tools/install_snapshot_runtime.sh is 24.04-only (libgnutls30t64 is the 64-bit +# time_t rename and does not exist on jammy), so the nightly build failed with +# "E: Unable to locate package libgnutls30t64" while regular CI stayed green on +# a cached layer. +NIGHTLY_UBUNTU_VERSION="24.04" +NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu${NIGHTLY_UBUNTU_VERSION}" echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})" docker buildx build --file docker/Dockerfile \ diff --git a/.buildkite/intel_jobs/basic_correctness_intel.yaml b/.buildkite/intel_jobs/basic_correctness_intel.yaml index 173baf40f6..548b6435a2 100644 --- a/.buildkite/intel_jobs/basic_correctness_intel.yaml +++ b/.buildkite/intel_jobs/basic_correctness_intel.yaml @@ -18,11 +18,12 @@ steps: VLLM_TEST_DEVICE: "xpu" source_file_dependencies: - vllm/ - - tests/basic_correctness/test_cumem.py + - tests/basic_correctness/memory/ + - tests/basic_correctness/cpu_offload/ commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s basic_correctness/test_cpu_offload.py && - pytest -v -s basic_correctness/test_mem.py' + pytest -v -s basic_correctness/cpu_offload && + pytest -v -s basic_correctness/memory' diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml index 2619e1c588..98218f23b7 100644 --- a/.buildkite/intel_jobs/engine_intel.yaml +++ b/.buildkite/intel_jobs/engine_intel.yaml @@ -63,7 +63,7 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py && - VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py' + pytest -v -s test_config.py' - label: V1 e2e (2 GPUs) key: v1-e2e-2-gpus diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index d5ae40ec6e..ad1a03e9cf 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -18,7 +18,7 @@ steps: key: xpu-example-test depends_on: - image-build-xpu - timeout_in_minutes: 50 + timeout_in_minutes: 70 device: intel_gpu agent_tags: label: production @@ -38,7 +38,7 @@ steps: python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 && + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && @@ -47,7 +47,7 @@ steps: python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 && VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && - python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 + python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC -tp 2 --max-model-len 8192 ' - label: "XPU W8A8 FP8 Linear Examples" key: xpu-w8a8-fp8-linear-examples @@ -74,8 +74,8 @@ steps: python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 ' - label: "XPU V1 test" key: xpu-v1-test @@ -105,8 +105,8 @@ steps: pytest -v -s v1/structured_output && pytest -v -s v1/test_serial_utils.py && pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --deselect "v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]"' + pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head && + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --deselect "v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]" --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py' parallelism: 2 - label: "XPU server test" key: xpu-server-test @@ -196,16 +196,20 @@ steps: VLLM_TEST_DEVICE: "xpu" source_file_dependencies: - vllm/compilation/ + - vllm/config/compilation.py - vllm/platforms/xpu.py + - vllm/v1/cudagraph_dispatcher.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py - vllm/v1/worker/xpu_model_runner.py - vllm/v1/worker/xpu_worker.py + - tests/v1/cudagraph - .buildkite/intel_jobs/test-intel.yaml commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_XPU_ENABLE_XPU_GRAPH=1 && - python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B && - python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8 && - python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096' - # Temporarily disabled (multi-GPU/MXFP8 not yet supported with XPU Graph): - # python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP8-CT-AutoRound -tp 2 --max-model-len 4096 + 'cd tests && + pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py && + pytest -v -s v1/cudagraph/test_cudagraph_mode.py && + pytest -v -s v1/cudagraph/test_breakable_cudagraph.py && + pytest -v -s v1/cudagraph/test_encoder_cudagraph.py' diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 0a9e6e7589..ae81f59e84 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -23,7 +23,7 @@ DEFAULT_CI_BASE_METADATA_VERSION="3" # local-source stages rather than unreachable remote-fetch alternatives. DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py" DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies rocm-triton-kernels csrc-build" -DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore .git_archival.txt pyproject.toml requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py build_rust.sh" +DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore .git_archival.txt pyproject.toml requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/build_rust.sh" DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm vllm-version rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build" # Docker's 128-character tag limit minus the longest cache prefix # ("csrc-rocm-branch-" and "rust-rocm-branch-", both 17 characters). diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 3b12d451cd..8812e876ae 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -80,13 +80,73 @@ fi BUILD_JSON="$(mktemp)" trap 'rm -f "${BUILD_JSON}"' EXIT -http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ - -H "Authorization: Bearer ${BK_TOKEN}" \ - "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")" -if [[ "${http_code}" != "200" ]]; then - echo "buildkite API returned ${http_code} -- skipping report" - exit 0 -fi +BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}" + +fetch_build() { + curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ + -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}" +} + +# Count jobs that have not reached a terminal state, ignoring this job: the +# report is itself part of the build and cannot wait for itself to finish. +outstanding_jobs() { + python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY' +import json, sys +TERMINAL = { + "passed", "failed", "blocked", "canceled", "skipped", "not_run", + "broken", "timed_out", "waiting_failed", "finished", "expired", +} +build = json.load(open(sys.argv[1])) +self_id = sys.argv[2] +print(sum( + 1 for j in build.get("jobs", []) + if j.get("type") == "script" + and j.get("id") != self_id + and j.get("state") not in TERMINAL +)) +PY +} + +# Wait for the rest of the build. The report is a snapshot of the Buildkite +# API's view of this build, so taking it early does not merely omit jobs -- it +# actively reports them as clean. Build 90830 reported with 14 of 347 jobs +# finished and none of the eventual 152 hard failures visible, and HUD recorded +# that nightly as green. +# +# Polling rather than a full `depends_on` barrier: Buildkite groups carry no +# key, so a barrier means naming all ~250 step keys, which silently rots when a +# step is added and fails the pipeline upload when one is removed or excluded +# from this lane (AMD, retired A100). The pipeline does gate this step on the +# long pole so it is scheduled near the end of the build -- that keeps the agent +# from idling for hours, but it is only a hint, and this loop is what actually +# guarantees the snapshot is complete. +POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}" +# Under the step's own timeout, so a stuck build still gets a partial report +# rather than the job being killed with nothing sent. +WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} )) +while :; do + http_code="$(fetch_build)" + if [[ "${http_code}" != "200" ]]; then + echo "buildkite API returned ${http_code} -- skipping report" + exit 0 + fi + remaining="$(outstanding_jobs)" || remaining="" + if [[ -z "${remaining}" ]]; then + echo "could not parse build JSON -- reporting what we have" + break + fi + if (( remaining == 0 )); then + echo "all other jobs have finished -- reporting" + break + fi + if (( $(date +%s) >= WAIT_DEADLINE )); then + echo "still ${remaining} job(s) running at the wait deadline --" \ + "reporting a partial view" + break + fi + echo "waiting for ${remaining} job(s) to finish..." + sleep "${POLL_INTERVAL_S}" +done # One callback per job, matching HUD's per-job crcr_workflow_job schema. # delivery_id is synthetic: the nightly path has no upstream dispatch to borrow diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 2705c3b87e..d07f3b3830 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -605,6 +605,33 @@ initialize_native_environment() { fi } +check_dpx_gpu_exclusivity() { + local devices=(/dev/dri/renderD*) + local device_id lock_status + + if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then + echo "DPX guard requires exactly one render device; refusing to start tests." >&2 + return 1 + fi + # This queue mounts the same host-local HF cache into every pod on the node. + if ! mountpoint -q "${HF_HOME}"; then + echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2 + return 1 + fi + device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1 + mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1 + # Hold the descriptor through workload and teardown; never delete the file. + exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1 + flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0 + lock_status=$? + if [[ "${lock_status}" -eq 75 ]]; then + echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2 + else + echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2 + fi + return 1 +} + run_native_preflight() { local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" @@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() { # Bus data identifies a card within the public node without publishing its # persistent UUID, serial number, or process list. run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi static -b -g all + amd-smi static --bus --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi metric -e -k -P -x -g all + amd-smi metric --ecc --ecc-blocks --pcie --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi bad-pages -p -r -u -g all + amd-smi static --ras --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi metric -p -t -u -m -v -g all + amd-smi metric --power --temperature --usage --mem-usage --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi xgmi -l -g all + amd-smi xgmi --link-status --gpu all elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ && command -v rocm-smi >/dev/null 2>&1; then run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ @@ -1485,6 +1512,10 @@ if is_native_runtime; then echo "Failed to initialize the native test environment" exit 1 fi + if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \ + && "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then + check_dpx_gpu_exclusivity || exit 1 + fi if [[ "${commands}" == *python_only_compile.sh* ]]; then # This no-GPU job validates the ROCm precompiled/editable install path, # rather than CPU runtime platform selection. diff --git a/.buildkite/scripts/hardware_ci/run-gh200-test.sh b/.buildkite/scripts/hardware_ci/run-gh200-test.sh index c95d74d3b4..20295fc25e 100644 --- a/.buildkite/scripts/hardware_ci/run-gh200-test.sh +++ b/.buildkite/scripts/hardware_ci/run-gh200-test.sh @@ -5,7 +5,7 @@ set -ex # Skip the new torch installation during build since we are using the specified version for arm64 in the Dockerfile -python3 use_existing_torch.py +python3 tools/use_existing_torch.py # Try building the docker image DOCKER_BUILDKIT=1 docker build . \ diff --git a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh index 58c0403b17..d71ee9cf62 100644 --- a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh @@ -17,7 +17,7 @@ case "${test_suite}" in python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 @@ -30,8 +30,8 @@ case "${test_suite}" in python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 ;; v1) cd tests @@ -43,7 +43,7 @@ case "${test_suite}" in pytest -v -s v1/structured_output pytest -v -s v1/test_serial_utils.py pytest -v -s v1/e2e/general/test_correctness_sliding_window.py - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py + pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py ;; server) @@ -65,8 +65,6 @@ case "${test_suite}" in pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8 ;; graph) - export VLLM_XPU_ENABLE_XPU_GRAPH=1 - python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8 python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-CT-AutoRound --max-model-len 4096 diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 9a2116a16d..1b2b854bc4 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -327,7 +327,7 @@ steps: working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/utils/import_utils.py - - scripts/benchmark_helion_kernels.py + - tools/benchmark_helion_kernels.py - tests/kernels/helion/ - vllm/platforms/rocm.py commands: @@ -538,25 +538,6 @@ steps: #----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------# -- label: ":amd: (MI300) Basic Correctness" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/basic_correctness/test_basic_correctness - - tests/basic_correctness/test_cpu_offload - - tests/basic_correctness/test_mem.py - optional: true - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s basic_correctness/test_mem.py - - VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py - - pytest -v -s basic_correctness/test_cpu_offload.py - - label: ":amd: (MI300) Distributed Models Shard %N" timeout_in_minutes: 80 parallelism: 3 @@ -589,22 +570,6 @@ steps: commands: - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac -#-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------# - -- label: ":amd: (MI300) Benchmarks CLI" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/benchmarks/ - - rust/src/bench/tests/python_serve_flags.txt - commands: - - pytest -v -s benchmarks/ - #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------# - label: ":amd: (MI300) PyTorch Compilation" # TBD @@ -922,22 +887,6 @@ steps: - pytest -v -s distributed/test_shm_buffer.py - pytest -v -s distributed/test_shm_storage.py -- label: ":amd: (MI300) EPLB Algorithm" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/eplb - - tests/distributed/test_eplb_algo.py - - tests/distributed/test_eplb_utils.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s distributed/test_eplb_algo.py - - pytest -v -s distributed/test_eplb_utils.py - - label: ":amd: (MI300) EPLB Execution" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1024,7 +973,7 @@ steps: - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=4)' - label: ":amd: (MI300) Distributed Torchrun + Examples" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1081,14 +1030,14 @@ steps: - tests/distributed/test_ray_v2_executor.py - tests/distributed/test_ray_v2_executor_e2e.py - tests/distributed/test_pipeline_parallel.py - - tests/basic_correctness/test_basic_correctness.py + - tests/basic_correctness/models/test_basic_correctness.py - vllm/platforms/rocm.py commands: - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 - pytest -v -s distributed/test_ray_v2_executor.py - pytest -v -s distributed/test_ray_v2_executor_e2e.py - pytest -v -s distributed/test_pipeline_parallel.py -k "ray" - - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray" + - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray" - label: ":amd: (MI300) Distributed DP + EP" # TBD timeout_in_minutes: 50 @@ -1155,38 +1104,6 @@ steps: - pytest -v -s distributed/test_symm_mem_allreduce.py - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node -#---------------------------------------------------------- mi300 · engine -----------------------------------------------------------# - -- label: ":amd: (MI300) Engine Core" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/engine - - tests/test_sequence - - tests/test_config - - tests/test_logger - - tests/test_vllm_port - - tests/jit_monitor/test_hooks.py - - tests/jit_monitor/test_hooks_gpu.py - - vllm/compilation/ - - vllm/config/ - - vllm/engine/ - - vllm/entrypoints/logger.py - - vllm/envs.py - - vllm/logger.py - - vllm/logging_utils/ - - vllm/platforms/ - - vllm/sequence.py - - vllm/triton_utils/ - - vllm/utils/ - commands: - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py - #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------# - label: ":amd: (MI300) Entrypoints Unit" # TBD @@ -1208,95 +1125,6 @@ steps: - pytest -v -s entrypoints/weight_transfer - pytest -v -s entrypoints/launchers -- label: ":amd: (MI300) Entrypoints Integration (LLM)" # TBD - timeout_in_minutes: 90 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/llm - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode - - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests - -- label: ":amd: (MI300) Entrypoints Integration (API Server) %N" - timeout_in_minutes: 65 - parallelism: 4 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/serve - - tests/entrypoints/scale_out - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi - - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - -- label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - fast_check: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/openai - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - -- label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - fast_check: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/openai - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai/chat_completion - -- label: ":amd: (MI300) Entrypoints Integration (API Server Generate)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/tool_use - - tests/entrypoints/tool_parsers - - tests/entrypoints/anthropic - - tests/entrypoints/cohere - - tests/entrypoints/generate - commands: - - pytest -v -s tool_use - - pytest -v -s entrypoints/tool_parsers - - pytest -v -s entrypoints/generate - - pytest -v -s entrypoints/anthropic - - pytest -v -s entrypoints/cohere - - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1341,26 +1169,6 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal -- label: ":amd: (MI300) OpenAI API Correctness" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/ - - vllm/entrypoints/openai/ - - vllm/model_executor/layers/ - - vllm/v1/attention/backends/ - - vllm/v1/attention/selector.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - - vllm/model_executor/model_loader/ - commands: - - bash ../tools/install_torchcodec_rocm.sh || exit 1 - - pytest -s entrypoints/openai/correctness/ - - label: ":amd: (MI300) Scale-out EC Connector E2E" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1380,6 +1188,25 @@ steps: #----------------------------------------------------------- mi300 · evals -----------------------------------------------------------# +- label: ":amd: (MI300) LM Eval Small Models" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + optional: true + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt + - label: ":amd: (MI300) DeepSeek V2-Lite Prefetch Offload Accuracy" # TBD timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1402,25 +1229,6 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030 -- label: ":amd: (MI300) LM Eval Small Models" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/ - - vllm/model_executor/layers/quantization - - vllm/model_executor/models/ - - vllm/model_executor/model_loader/ - - vllm/v1/attention/backends/ - - vllm/v1/attention/selector.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - optional: true - commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt - - label: ":amd: (MI300) MRCR Eval Small Models" # TBD timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1459,22 +1267,6 @@ steps: commands: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt -- label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" # TBD - timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" - source_file_dependencies: - - vllm/multimodal/ - - vllm/inputs/ - - vllm/v1/core/ - - vllm/platforms/rocm.py - - vllm/model_executor/model_loader/ - commands: - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 - - label: ":amd: (MI300) GPQA Eval (GPT-OSS)" # TBD timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1697,44 +1489,84 @@ steps: - export VLLM_USE_DEEP_GEMM=0 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4 -#--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# +#---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------# -- label: ":amd: (MI300) Examples" # TBD - timeout_in_minutes: 75 +- label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + parallelism: 3 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - tests/kernels/core + - tests/kernels/test_top_k_per_row.py + - tests/kernels/test_concat_mla_q.py + - tests/kernels/test_rocm_fp32_router_gemm.py + - vllm/model_executor/layers/fused_moe/router/gate_linear.py + - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py + - vllm/model_executor/layers/rotary_embedding/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/kernels/test_fused_qk_norm_rope_gate.py + optional: true + commands: + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + +- label: ":amd: (MI300) Quantization Kernels Shard %N" + timeout_in_minutes: 120 + parallelism: 6 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 optional: true - working_dir: "/vllm-workspace/examples" + working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/entrypoints - - vllm/multimodal - - examples/ + - csrc/quantization/ + - csrc/rocm/ + - vllm/model_executor/layers/quantization + - tests/kernels/quantization + - tests/kernels/quant_utils.py + - tests/kernels/utils.py + - vllm/_aiter_ops.py + - vllm/kernels/aiter_ops.py + - vllm/_custom_ops.py + - vllm/envs.py + - vllm/platforms/rocm.py + - vllm/model_executor/kernels/ + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/config/ + - tests/kernels/quantization/test_rocm_skinny_gemms.py + commands: + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + +- label: ":amd: (MI300) MoE Kernels Shard %N" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + parallelism: 5 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/quantization/cutlass_w8a8/moe/ + - csrc/moe/ + - csrc/rocm/ + - tests/kernels/moe + - tests/kernels/utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/mxfp4.py + - vllm/model_executor/layers/quantization/utils/quant_utils.py + - vllm/distributed/device_communicators/ + - vllm/envs.py + - vllm/config + - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - # Basic - # Multi-modal models - # Pooling models - # Features demo - - pip install --no-deps tensorizer - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - - python3 basic/offline_inference/classify.py - - python3 basic/offline_inference/embed.py - - python3 basic/offline_inference/score.py - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - - python3 pooling/embed/vision_embedding_offline.py --seed 0 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - -#---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------# + - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --ignore=kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) vLLM IR" # TBD timeout_in_minutes: 50 @@ -1818,6 +1650,48 @@ steps: commands: - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py +- label: ":amd: (MI355) Kimi K3" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/models/kimi_k3/ + - csrc/libtorch_stable/kimi_k3/ + - tests/models/kimi_k3/ + - vllm/platforms/rocm.py + commands: + # Mirrors the ":amd: (MI355) Kimi K3" step generated from + # .buildkite/test_areas/models_basic.yaml. The ROCm KDA/AttnRes kernels are + # gated on gfx950, so this has to be MI355. The two kernels/ files the NVIDIA + # step runs are CUDA-only (they import cuda.bindings.driver), so they are + # deliberately left out here. + - pytest -v -s models/kimi_k3 + +- label: ":amd: (MI355) Kimi K3 Multi-GPU" # TBD + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/models/kimi_k3/ + - tests/models/kimi_k3/ + - vllm/platforms/rocm.py + commands: + # The single-GPU step above skips every @multi_gpu_test case. `-m "distributed"` + # selects exactly those: multi_gpu_test() applies pytest.mark.distributed + # alongside its skipif (tests/utils.py:2080), so this picks up a new + # multi-GPU test anywhere in the directory without editing this list. + # Four GPUs run the two tp4 cases in test_amd_latent_moe_runner.py; the tp8 + # case and the 8/16-GPU cases in test_latent_moe_tail.py still skip. + - pytest -v -s -m "distributed" models/kimi_k3 + - label: ":amd: (MI300) Mamba Kernels" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1830,64 +1704,11 @@ steps: - tests/kernels/mamba - vllm/model_executor/layers/mamba/gdn - vllm/model_executor/layers/mamba/ops + - vllm/third_party/flash_linear_attention/ - vllm/platforms/rocm.py commands: - pytest -v -s kernels/mamba -- label: ":amd: (MI300) MoE Kernels Shard %N" # TBD - timeout_in_minutes: 55 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - parallelism: 5 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/quantization/cutlass_w8a8/moe/ - - csrc/moe/ - - csrc/rocm/ - - tests/kernels/moe - - tests/kernels/utils.py - - vllm/model_executor/layers/fused_moe/ - - vllm/model_executor/layers/quantization/mxfp4.py - - vllm/model_executor/layers/quantization/utils/quant_utils.py - - vllm/distributed/device_communicators/ - - vllm/envs.py - - vllm/config - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --ignore=kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - -- label: ":amd: (MI300) Quantization Kernels Shard %N" - timeout_in_minutes: 120 - parallelism: 6 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/quantization/ - - csrc/rocm/ - - vllm/model_executor/layers/quantization - - tests/kernels/quantization - - tests/kernels/quant_utils.py - - tests/kernels/utils.py - - vllm/_aiter_ops.py - - vllm/kernels/aiter_ops.py - - vllm/_custom_ops.py - - vllm/envs.py - - vllm/platforms/rocm.py - - vllm/model_executor/kernels/ - - vllm/v1/attention/backends/rocm_aiter_fa.py - - vllm/config/ - - tests/kernels/quantization/test_rocm_skinny_gemms.py - commands: - - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1909,21 +1730,6 @@ steps: #----------------------------------------------------------- mi300 · lora ------------------------------------------------------------# -- label: ":amd: (MI300) LoRA Shard %N" # TBD - timeout_in_minutes: 85 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/lora - - tests/lora - - vllm/platforms/rocm.py - commands: - - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py - - label: ":amd: (MI300) LoRA TP (Distributed) %N" timeout_in_minutes: 65 parallelism: 4 @@ -1952,7 +1758,7 @@ steps: #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------# - label: ":amd: (MI300) Model Executor" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1971,7 +1777,6 @@ steps: - tests/model_executor/test_jit_warmup_cutedsl_launcher.py - tests/model_executor/test_jit_warmup_triton_launcher.py commands: - - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export PYTHONFAULTHANDLER=1 - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread @@ -1993,26 +1798,6 @@ steps: commands: - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset -- label: ":amd: (MI300) Basic Models (Other)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/test_terratorch.py - - tests/models/transformers/test_backend.py - - tests/models/test_registry.py - - tests/models/test_hyv4_rocm.py - - tests/models/test_vision.py - - tests/models/test_deepseek_v4_vl_rocm.py - commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py - - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed' - - pytest -v -s models/test_vision.py::test_simple_mrope_vision_model_spatial_merge - #----------------------------------------------------- mi300 · models / language -----------------------------------------------------# - label: ":amd: (MI300) Language Models (MTEB)" # TBD @@ -2042,43 +1827,6 @@ steps: commands: - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: ":amd: (MI300) Language Models (Standard)" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language - commands: - - pip freeze | grep -E 'torch' - - pytest -v -s models/language -m 'core_model and (not slow_test)' - -- label: ":amd: (MI300) Language Models (Extra Standard) Shard %N" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - parallelism: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/model_executor/models/ - - vllm/model_executor/model_loader/ - - vllm/model_executor/layers/ - - vllm/v1/attention/backends/ - - vllm/v1/attention/selector.py - - tests/models/language/pooling/test_embedding.py - - tests/models/language/generation/test_common.py - - tests/models/language/pooling/test_classification.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pip freeze | grep -E 'torch' - - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - label: ":amd: (MI300) Language Models (Extended Generation)" # TBD timeout_in_minutes: 95 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2139,20 +1887,6 @@ steps: commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' -- label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - - label: ":amd: (MI300) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2197,21 +1931,6 @@ steps: commands: - pytest -v -s models/multimodal/pooling -m 'not core_model' -- label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD - timeout_in_minutes: 55 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - - pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model - - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model - #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------# - label: ":amd: (MI300) Quantized Models" # TBD @@ -2537,7 +2256,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.18.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------# @@ -2608,6 +2327,51 @@ steps: #------------------------------------------------------------ mi300 · v1 -------------------------------------------------------------# +- label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD + timeout_in_minutes: 75 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + - vllm/v1/attention/backends/ + commands: + - pytest -v -s v1/e2e/spec_decode/speculators/ + - pytest -v -s v1/e2e/spec_decode/mtp/ + +- label: ":amd: (MI300) V1 Spec Decode" # TBD + timeout_in_minutes: 50 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1/spec_decode + - vllm/config/ + - vllm/distributed/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/models/qwen4_exp/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not slow_test' v1/spec_decode + - label: ":amd: (MI300) Acceptance Length (Large Models)" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2655,102 +2419,6 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/test_tensor_ipc_queue.py -- label: ":amd: (MI300) Spec Decode Draft Model" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/gpu/spec_decode/ - - vllm/model_executor/model_loader/ - - vllm/v1/sample/ - - vllm/model_executor/layers/ - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - commands: - - pytest -v -s v1/e2e/spec_decode/draft_model/ - -- label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/gpu/spec_decode/ - - vllm/model_executor/model_loader/ - - vllm/v1/sample/ - - vllm/model_executor/layers/ - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - commands: - - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" - -- label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/gpu/spec_decode/ - - vllm/model_executor/model_loader/ - - vllm/v1/sample/ - - vllm/model_executor/layers/ - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - commands: - - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" - - -- label: ":amd: (MI300) Spec Decode N-Gram + Suffix" # TBD - timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/gpu/spec_decode/ - - vllm/model_executor/model_loader/ - - vllm/v1/sample/ - - vllm/model_executor/layers/ - - tests/v1/e2e/spec_decode/ - - tests/spec_decode/ - - vllm/platforms/rocm.py - commands: - - pytest -v -s v1/e2e/spec_decode/ngram_suffix/ - - python3 spec_decode/test_custom_proposer.py - -- label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD - timeout_in_minutes: 75 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/spec_decode/ - - vllm/v1/worker/gpu/spec_decode/ - - vllm/model_executor/model_loader/ - - vllm/v1/sample/ - - vllm/model_executor/layers/ - - vllm/transformers_utils/configs/speculators/ - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - - vllm/v1/attention/backends/ - commands: - - pytest -v -s v1/e2e/spec_decode/speculators/ - - pytest -v -s v1/e2e/spec_decode/mtp/ - - label: ":amd: (MI300) Speculators Correctness Nightly" timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2784,30 +2452,6 @@ steps: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test -- label: ":amd: (MI300) V1 Spec Decode" # TBD - timeout_in_minutes: 50 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/spec_decode - - vllm/config/ - - vllm/distributed/ - - vllm/inputs/ - - vllm/model_executor/ - - vllm/models/qwen4_exp/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - optional: true - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not slow_test' v1/spec_decode - - label: ":amd: (MI300) Extract Hidden States Integration" # TBD timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2862,283 +2506,6 @@ steps: commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) V1 Core" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/simple_kv_offload - - tests/v1/worker - - tests/v1/streaming_input - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - optional: true - commands: - # - export HSA_NO_SCRATCH_RECLAIM=1 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/core - -- label: ":amd: (MI300) V1 Executor + Worker" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/simple_kv_offload - - tests/v1/worker - - tests/v1/streaming_input - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/executor - - pytest -v -s v1/worker - - pytest -v -s v1/streaming_input - optional: true - -- label: ":amd: (MI300) V1 KV Offload" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/simple_kv_offload - - tests/v1/worker - - tests/v1/streaming_input - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh - - pytest -v -s v1/kv_offload - - pytest -v -s v1/simple_kv_offload - optional: true - -- label: ":amd: (MI300) V1 KV Connectors Shard %N" - timeout_in_minutes: 60 - parallelism: 4 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/simple_kv_offload - - tests/v1/worker - - tests/v1/streaming_input - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - optional: true - -- label: ":amd: (MI300) V1 Metrics + LM Eval" - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/simple_kv_offload - - tests/v1/worker - - tests/v1/streaming_input - - tests/v1/kv_connector/unit - - tests/v1/ec_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export GIT_CONFIG_COUNT=1 - - export GIT_CONFIG_KEY_0=http.version - - export GIT_CONFIG_VALUE_0=HTTP/1.1 - - export GIT_TERMINAL_PROMPT=0 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/metrics - - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - optional: true - - -- label: ":amd: (MI300) V1 Sample" - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/sample - - tests/v1/logits_processors - - tests/v1/test_oracle.py - - tests/v1/test_request.py - - tests/v1/test_outputs.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/sample - -- label: ":amd: (MI300) V1 Logits + Oracle" - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/sample - - tests/v1/logits_processors - - tests/v1/test_oracle.py - - tests/v1/test_request.py - - tests/v1/test_outputs.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/logits_processors - - pytest -v -s v1/test_oracle.py - - pytest -v -s v1/test_request.py - - pytest -v -s v1/test_outputs.py - - - label: ":amd: (MI300) Metrics, Tracing" # TBD timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3300,7 +2667,7 @@ steps: - pytest -v -s distributed/test_utils.py - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -3393,8 +2760,100 @@ steps: # # ######################################################################################################################################### +#----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------# + +- label: ":amd: (MI355 DPX) Basic Correctness Models %N" # TBD + timeout_in_minutes: 60 + parallelism: 2 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355 DPX) Basic Correctness CuMem" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/memory/cumem + +- label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/memory/sleep_mode + +- label: ":amd: (MI355 DPX) Basic Correctness CPU Offload" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/cpu_offload + +- label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/ + optional: true + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/prefetch_offload + #-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------# +- label: ":amd: (MI355 DPX) Benchmarks CLI" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/benchmarks/ + - rust/src/bench/tests/python_serve_flags.txt + commands: + - pytest -v -s benchmarks/ + - label: ":amd: (MI355) Attention Benchmark Smoke" # TBD timeout_in_minutes: 25 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3412,11 +2871,11 @@ steps: #---------------------------------------------------------- mi355 · compile ----------------------------------------------------------# -- label: ":amd: (MI355) PyTorch Compilation H100 Cases" +- label: ":amd: (MI355 DPX) PyTorch Compilation H100 Cases" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -3452,13 +2911,52 @@ steps: commands: - pytest -v -s compile/h100/test_startup.py -#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------# - -- label: ":amd: (MI355) Sharded RDT Weight Transfer" +- label: ":amd: (MI355) Fusion and Compile FP8" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - tests/compile/passes/ + - tests/compile/correctness_e2e/test_attn_quant.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + commands: + - amd-smi + - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py + - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py + +#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------# + +- label: ":amd: (MI355 DPX) EPLB Algorithm" # TBD + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/eplb + - tests/distributed/test_eplb_algo.py + - tests/distributed/test_eplb_utils.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s distributed/test_eplb_algo.py + - pytest -v -s distributed/test_eplb_utils.py + +- label: ":amd: (MI355 DPX) Sharded RDT Weight Transfer" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -3505,6 +3003,35 @@ steps: - pytest -v -s v1/e2e/general/test_sharded_sampling.py - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py +- label: ":amd: (MI355) MoRI EP Healthy Baseline" + timeout_in_minutes: 10 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - tests/kernels/moe/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/utils/import_utils.py + - vllm/utils/math_utils.py + - vllm/utils/torch_utils.py + - vllm/platforms/ + - vllm/_aiter_ops.py + optional: true + # TODO: Add fault injection, detection and recovery once MoRI supports them. + # This job currently checks healthy EP execution only. + commands: + - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph + - VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]' + - label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3538,6 +3065,49 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py + +- label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + - tests/v1/kv_connector/mooncake_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - examples/disaggregated/mooncake_connector/ + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py + - label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3635,14 +3205,143 @@ steps: - pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py +- label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - csrc/quantization/ + - vllm/compilation/ + - vllm/distributed/ + - vllm/model_executor/ + - vllm/v1/attention/ + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and Llama-3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + +quant_fp8 and +rms_norm and qwen3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and DeepSeek-Coder and + ROCM_AITER_MLA' + +- label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP" # TBD + timeout_in_minutes: 85 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_8 + num_gpus: 8 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - requirements/kv_connectors_rocm.txt + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/models/deepseek_v4/ + - vllm/v1/attention/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # A mirror does not inherit the parent env, so it is repeated here in + # the parent's order. The layouts must stay empty: unset, the script + # defaults to HND, which the V4 indexer rejects. + - >- + VLLM_ROCM_USE_AITER=1 + ENABLE_HMA_FLAG=1 + DP_EP=1 + GPU_MEMORY_UTILIZATION=0.85 + PREFILLER_TP_SIZE=4 + DECODER_TP_SIZE=4 + PREFILL_BLOCK_SIZE=256 + DECODE_BLOCK_SIZE=256 + MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash + VLLM_ENGINE_READY_TIMEOUT_S=1800 + VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8 + PREFILLER_KV_LAYOUT= + DECODER_KV_LAYOUT= + bash v1/kv_connector/nixl_integration/run_accuracy_test.sh + +#---------------------------------------------------------- mi355 · engine -----------------------------------------------------------# + +- label: ":amd: (MI355 DPX) Engine Core" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/engine + - tests/test_sequence + - tests/test_config + - tests/test_logger + - tests/test_vllm_port + - tests/jit_monitor/test_hooks.py + - tests/jit_monitor/test_hooks_gpu.py + - vllm/compilation/ + - vllm/config/ + - vllm/engine/ + - vllm/entrypoints/logger.py + - vllm/envs.py + - vllm/logger.py + - vllm/logging_utils/ + - vllm/platforms/ + - vllm/sequence.py + - vllm/triton_utils/ + - vllm/utils/ + commands: + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py + #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------# -- label: ":amd: (MI355) Entrypoints Integration (API Server) %N" +- label: ":amd: (MI355 DPX) Entrypoints Integration (LLM)" # TBD + timeout_in_minutes: 90 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/llm + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode + - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process + - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests + +- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server) %N" timeout_in_minutes: 65 parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true fast_check: true working_dir: "/vllm-workspace/tests" @@ -3656,11 +3355,11 @@ steps: - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: ":amd: (MI355) Entrypoints Integration (OpenAI API completion)" # TBD +- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx fast_check: true optional: true working_dir: "/vllm-workspace/tests" @@ -3672,11 +3371,11 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness -- label: ":amd: (MI355) Entrypoints Integration (OpenAI API chat_completion)" # TBD +- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API chat_completion)" # TBD timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx fast_check: true optional: true working_dir: "/vllm-workspace/tests" @@ -3687,11 +3386,103 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/chat_completion -- label: ":amd: (MI355) Entrypoints Integration (API Server Generate)" # TBD +- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server Generate)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx + optional: true + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/tool_use + - tests/entrypoints/tool_parsers + - tests/entrypoints/anthropic + - tests/entrypoints/cohere + - tests/entrypoints/generate + commands: + - pytest -v -s tool_use + - pytest -v -s entrypoints/tool_parsers + - pytest -v -s entrypoints/generate + - pytest -v -s entrypoints/anthropic + - pytest -v -s entrypoints/cohere + +- label: ":amd: (MI355 DPX) OpenAI API Correctness" # TBD + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - vllm/entrypoints/openai/ + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/model_loader/ + commands: + - bash ../tools/install_torchcodec_rocm.sh || exit 1 + - pytest -s entrypoints/openai/correctness/ + +- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server) %N (MI355 suite)" + timeout_in_minutes: 65 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + fast_check: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/serve + - tests/entrypoints/scale_out + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion) (MI355 suite)" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/openai + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness + +- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API chat_completion) (MI355 suite)" # TBD + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + fast_check: true + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/openai + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/openai/chat_completion + +- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server Generate) (MI355 suite)" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx fast_check: true optional: true working_dir: "/vllm-workspace/tests" @@ -3709,11 +3500,11 @@ steps: - pytest -v -s entrypoints/anthropic - pytest -v -s entrypoints/cohere -- label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" # TBD +- label: ":amd: (MI355 DPX) Entrypoints Integration (Speech to Text)" # TBD timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3724,11 +3515,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text -- label: ":amd: (MI355) Entrypoints Integration (Multimodal)" +- label: ":amd: (MI355 DPX) Entrypoints Integration (Multimodal)" timeout_in_minutes: 55 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3739,11 +3530,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal -- label: ":amd: (MI355) Entrypoints Integration (Pooling) Shard %N" # TBD +- label: ":amd: (MI355 DPX) Entrypoints Integration (Pooling) Shard %N" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx fast_check: true parallelism: 4 working_dir: "/vllm-workspace/tests" @@ -3757,6 +3548,22 @@ steps: #----------------------------------------------------------- mi355 · evals -----------------------------------------------------------# +- label: ":amd: (MI355 DPX) Multimodal Accuracy Eval (Small Models)" # TBD + timeout_in_minutes: 35 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" + source_file_dependencies: + - vllm/multimodal/ + - vllm/inputs/ + - vllm/v1/core/ + - vllm/platforms/rocm.py + - vllm/model_executor/model_loader/ + commands: + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 + - label: ":amd: (MI355) GPQA Eval (GPT-OSS)" # TBD timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3780,11 +3587,11 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt -- label: ":amd: (MI355) LM Eval Watermarking" +- label: ":amd: (MI355 DPX) LM Eval Watermarking" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -3869,6 +3676,111 @@ steps: - uv pip install --system 'evalscope==1.10.0' - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt +- label: ":amd: (MI355) LM Eval TurboQuant k8v4" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt + +- label: ":amd: (MI355) LM Eval TurboQuant t4nc" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt + +- label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt + +- label: ":amd: (MI355) LM Eval TurboQuant t3nc" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test-amd.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt + - label: ":amd: (MI355) LM Eval Small Models Distributed" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3976,6 +3888,41 @@ steps: #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# +- label: ":amd: (MI355 DPX) Examples" # TBD + timeout_in_minutes: 75 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/examples" + source_file_dependencies: + - vllm/entrypoints + - vllm/multimodal + - examples/ + - vllm/platforms/rocm.py + commands: + # Basic + # Multi-modal models + # Pooling models + # Features demo + - pip install --no-deps tensorizer + - python3 basic/offline_inference/chat.py + - python3 basic/offline_inference/generate.py --model facebook/opt-125m + - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 + - python3 basic/offline_inference/classify.py + - python3 basic/offline_inference/embed.py + - python3 basic/offline_inference/score.py + - python3 generate/multimodal/audio_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 + - python3 pooling/embed/vision_embedding_offline.py --seed 0 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py + - python3 deployment/llm_engine_example.py + - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 + - label: ":amd: (MI355) Examples" # TBD timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4013,11 +3960,11 @@ steps: #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------# -- label: ":amd: (MI355) Attention DiffKV Kernels" +- label: ":amd: (MI355 DPX) Attention DiffKV Kernels" timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4030,29 +3977,6 @@ steps: commands: - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py -- label: ":amd: (MI355) Core Operation Kernels Shard %N" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_1 - parallelism: 3 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/ - - tests/kernels/core - - tests/kernels/test_top_k_per_row.py - - tests/kernels/test_concat_mla_q.py - - tests/kernels/test_rocm_fp32_router_gemm.py - - vllm/model_executor/layers/fused_moe/router/gate_linear.py - - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py - - vllm/model_executor/layers/rotary_embedding/ - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - - tests/kernels/test_fused_qk_norm_rope_gate.py - optional: true - commands: - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - label: ":amd: (MI355) MiniMax Reduce RMS Kernels" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4077,11 +4001,11 @@ steps: commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py -- label: ":amd: (MI355) Kernels" # TBD +- label: ":amd: (MI355 DPX) Kernels" # TBD timeout_in_minutes: 25 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/" source_file_dependencies: - csrc/quantization/fp4/ @@ -4105,11 +4029,11 @@ steps: - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN - pytest -v -s tests/kernels/attention/test_attention_selector.py -- label: ":amd: (MI355) MLA Kernels" +- label: ":amd: (MI355 DPX) MLA Kernels" timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/" @@ -4129,11 +4053,11 @@ steps: - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py -- label: ":amd: (MI355) DeepSeek V4 Fused Kernels" +- label: ":amd: (MI355 DPX) DeepSeek V4 Fused Kernels" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4145,11 +4069,11 @@ steps: commands: - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py -- label: ":amd: (MI355) Attention Kernels Shard %N" # TBD +- label: ":amd: (MI355 DPX) Attention Kernels Shard %N" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx parallelism: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4165,11 +4089,11 @@ steps: commands: - pytest -v -s kernels/attention --ignore=kernels/attention/test_triton_unified_attention_diffkv.py --ignore=kernels/attention/test_rocm_aiter_mla_decode.py --ignore=kernels/attention/test_rocm_aiter_mla_decode_metadata.py --ignore=kernels/attention/test_rocm_aiter_mla_fp8_support.py --ignore=kernels/attention/test_rocm_aiter_mla_op_registration.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355) MoE Kernels Shard %N" # TBD +- label: ":amd: (MI355 DPX) MoE Kernels Shard %N (MI355 suite)" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx parallelism: 5 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4227,7 +4151,37 @@ steps: - vllm/distributed/device_communicators/ - vllm/config commands: - - pytest -v -s kernels/moe/test_moe_layer.py + # MoRI graph cases run in the Fault Tolerance group's healthy EP baseline. + - pytest -v -s kernels/moe/test_moe_layer.py -k "not test_moe_layer_mori_graph" + +- label: ":amd: (MI355) MoRI EP Numerics" + timeout_in_minutes: 20 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/kernels/moe/test_mori_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/moe/modular_kernel_tools/parallel_utils.py + - tests/kernels/utils.py + - tests/utils.py + - tests/conftest.py + - vllm/utils/torch_utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/platforms/ + - vllm/_aiter_ops.py + - requirements/rocm-test.txt + optional: true + commands: + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py - label: ":amd: (MI355) Quantization Kernels Shard %N" timeout_in_minutes: 120 @@ -4254,15 +4208,38 @@ steps: - tests/kernels/quantization/test_rocm_skinny_gemms.py optional: true commands: - - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355) FP8 MoE Kernels" # TBD - timeout_in_minutes: 40 +- label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N" + timeout_in_minutes: 10 + parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: + - tests/kernels/quantization/test_nvfp4_emulation.py + - tests/kernels/quantization/nvfp4_utils.py + - tests/conftest.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py + - vllm/config/kernel.py + - vllm/platforms/ + - vllm/envs.py + - vllm/utils/torch_utils.py + - requirements/rocm-test.txt + optional: true + commands: + - amd-smi + - pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + +- label: ":amd: (MI355 DPX) FP8 MoE Kernels" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + working_dir: "/vllm-workspace/tests" + source_file_dependencies: - csrc/moe/ - vllm/model_executor/layers/fused_moe/ - tests/kernels/moe/test_deepep_moe.py @@ -4303,13 +4280,50 @@ steps: commands: - pytest -v -s kernels/moe/test_deepep_moe.py +#----------------------------------------------------------- mi355 · lora ------------------------------------------------------------# + +- label: ":amd: (MI355 DPX) LoRA Shard %N" # TBD + timeout_in_minutes: 85 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + parallelism: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/lora + - tests/lora + - vllm/platforms/rocm.py + commands: + - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py + #------------------------------------------------------ mi355 · models / basic -------------------------------------------------------# -- label: ":amd: (MI355) GLM5Next" +- label: ":amd: (MI355 DPX) Basic Models (Other)" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/test_terratorch.py + - tests/models/transformers/test_backend.py + - tests/models/test_registry.py + - tests/models/test_hyv4_rocm.py + - tests/models/test_vision.py + - tests/models/test_deepseek_v4_vl_rocm.py + commands: + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py + - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed' + - pytest -v -s models/test_vision.py::test_simple_mrope_vision_model_spatial_merge + +- label: ":amd: (MI355 DPX) GLM5Next" timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4320,11 +4334,11 @@ steps: commands: - pytest -v -s models/glm5next -- label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" # TBD +- label: ":amd: (MI355 DPX) Basic Models (Extra Initialization) Shard %N" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx parallelism: 6 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4340,11 +4354,48 @@ steps: #----------------------------------------------------- mi355 · models / language -----------------------------------------------------# -- label: ":amd: (MI355) Language Models (Hybrid) Shard %N" # TBD +- label: ":amd: (MI355 DPX) Language Models (Standard)" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/language + commands: + - pip freeze | grep -E 'torch' + - pytest -v -s models/language -m 'core_model and (not slow_test)' + +- label: ":amd: (MI355 DPX) Language Models (Extra Standard) Shard %N" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + parallelism: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - tests/models/language/pooling/test_embedding.py + - tests/models/language/generation/test_common.py + - tests/models/language/pooling/test_classification.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pip freeze | grep -E 'torch' + - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355 DPX) Language Models (Hybrid) Shard %N" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx parallelism: 2 optional: true working_dir: "/vllm-workspace/tests" @@ -4359,11 +4410,11 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: ":amd: (MI355) Granite Language Model Compatibility" +- label: ":amd: (MI355 DPX) Granite Language Model Compatibility" timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4379,11 +4430,11 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview' -- label: ":amd: (MI355) Language Models (Extended Generation)" # TBD +- label: ":amd: (MI355 DPX) Language Models (Extended Generation)" # TBD timeout_in_minutes: 95 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -4394,12 +4445,12 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: ":amd: (MI355) Language Models (Extended Pooling) Shard %N" +- label: ":amd: (MI355 DPX) Language Models (Extended Pooling) Shard %N" timeout_in_minutes: 95 parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4408,11 +4459,11 @@ steps: commands: - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: ":amd: (MI355) Language Models (PPL)" # TBD +- label: ":amd: (MI355 DPX) Language Models (PPL)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4439,11 +4490,11 @@ steps: commands: - pytest -v -s models/language/generation_ppl_test -- label: ":amd: (MI355) Language Models (Standard)" # TBD +- label: ":amd: (MI355 DPX) Language Models (Standard) (MI355 suite)" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -4455,11 +4506,40 @@ steps: #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------# -- label: ":amd: (MI355) Multimodal Models (Extended Generation 1)" # TBD +- label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" + - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model + +- label: ":amd: (MI355 DPX) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" + - pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model + - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model + +- label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 1)" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4470,11 +4550,11 @@ steps: - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py -- label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" # TBD +- label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 3)" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4483,11 +4563,11 @@ steps: commands: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' -- label: ":amd: (MI355) Multimodal Models (Extended Pooling)" # TBD +- label: ":amd: (MI355 DPX) Multimodal Models (Extended Pooling)" # TBD timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4496,11 +4576,11 @@ steps: commands: - pytest -v -s models/multimodal/pooling -m 'not core_model' -- label: ":amd: (MI355) Multimodal Models (Standard) 1: qwen2" # TBD +- label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2 (MI355 suite)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4510,11 +4590,11 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model -- label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" # TBD +- label: ":amd: (MI355 DPX) Multimodal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4528,11 +4608,11 @@ steps: #----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------# -- label: ":amd: (MI355) Quantized Models" # TBD +- label: ":amd: (MI355 DPX) Quantized Models" # TBD timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/model_executor/layers/quantization @@ -4568,7 +4648,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi # - label: Quantized MoE Test (B200-MI355) # TBD @@ -4600,11 +4680,11 @@ steps: #--------------------------------------------------------- mi355 · samplers ----------------------------------------------------------# -- label: ":amd: (MI355) Samplers Multimodal Beam Search" +- label: ":amd: (MI355 DPX) Samplers Multimodal Beam Search" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4622,11 +4702,363 @@ steps: #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------# -- label: ":amd: (MI355) E2E Core Large Memory" +- label: ":amd: (MI355 DPX) Spec Decode Draft Model" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/draft_model/ + +- label: ":amd: (MI355 DPX) Spec Decode Eagle 1: DeepSeek + Qwen" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" + +- label: ":amd: (MI355 DPX) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" + + +- label: ":amd: (MI355 DPX) Spec Decode N-Gram + Suffix" # TBD + timeout_in_minutes: 35 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - tests/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/ngram_suffix/ + - python3 spec_decode/test_custom_proposer.py + +- label: ":amd: (MI355 DPX) V1 Core" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true + commands: + # - export HSA_NO_SCRATCH_RECLAIM=1 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/core + +- label: ":amd: (MI355 DPX) V1 Executor + Worker" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s v1/executor + - pytest -v -s v1/worker + - pytest -v -s v1/streaming_input + optional: true + +- label: ":amd: (MI355 DPX) V1 KV Offload" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh + - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload + optional: true + +- label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N" + timeout_in_minutes: 60 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + optional: true + +- label: ":amd: (MI355 DPX) V1 Metrics + LM Eval" timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - export GIT_TERMINAL_PROMPT=0 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/metrics + - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api + - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine + optional: true + + +- label: ":amd: (MI355 DPX) V1 Sample" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s v1/sample + +- label: ":amd: (MI355 DPX) V1 Logits + Oracle" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s v1/logits_processors + - pytest -v -s v1/test_oracle.py + - pytest -v -s v1/test_request.py + - pytest -v -s v1/test_outputs.py + + +- label: ":amd: (MI355 DPX) E2E Core Large Memory" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4638,11 +5070,11 @@ steps: - pytest -v -s v1/e2e/general/test_kv_sharing_fast_prefill.py - pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async -- label: ":amd: (MI355) Extract Hidden States Integration Single GPU" +- label: ":amd: (MI355 DPX) Extract Hidden States Integration Single GPU" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace" @@ -4657,11 +5089,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration -- label: ":amd: (MI355) V1 Attention Shard %N" # TBD +- label: ":amd: (MI355 DPX) V1 Attention Shard %N" # TBD timeout_in_minutes: 125 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx parallelism: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4676,11 +5108,11 @@ steps: commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355) V1 Core" +- label: ":amd: (MI355 DPX) V1 Core (MI355 suite)" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4717,11 +5149,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/core -- label: ":amd: (MI355) V1 Executor + Worker" +- label: ":amd: (MI355 DPX) V1 Executor + Worker (MI355 suite)" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -4760,11 +5192,11 @@ steps: - pytest -v -s v1/worker - pytest -v -s v1/streaming_input -- label: ":amd: (MI355) V1 KV Offload" +- label: ":amd: (MI355 DPX) V1 KV Offload (MI355 suite)" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -4803,12 +5235,12 @@ steps: - pytest -v -s v1/kv_offload - pytest -v -s v1/simple_kv_offload -- label: ":amd: (MI355) V1 KV Connectors Shard %N" +- label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N (MI355 suite)" timeout_in_minutes: 60 parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -4847,11 +5279,11 @@ steps: - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB -- label: ":amd: (MI355) V1 Metrics + LM Eval" +- label: ":amd: (MI355 DPX) V1 Metrics + LM Eval (MI355 suite)" timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -4894,11 +5326,11 @@ steps: - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine -- label: ":amd: (MI355) CUDAGraph" +- label: ":amd: (MI355 DPX) CUDAGraph" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -4924,11 +5356,11 @@ steps: - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py -- label: ":amd: (MI355) V1 Sample" +- label: ":amd: (MI355 DPX) V1 Sample (MI355 suite)" timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -4953,11 +5385,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/sample -- label: ":amd: (MI355) V1 Logits + Oracle" +- label: ":amd: (MI355 DPX) V1 Logits + Oracle (MI355 suite)" timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: /vllm-workspace/tests source_file_dependencies: @@ -5010,12 +5442,12 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not slow_test' v1/spec_decode -- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" +- label: ":amd: (MI355 DPX) Spec Decode AL MTP + Other Acceptance Nightly" timeout_in_minutes: 55 parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -5045,11 +5477,11 @@ steps: #----------------------------------------------------------- mi355 · misc ------------------------------------------------------------# -- label: ":amd: (MI355) Regression" # TBD +- label: ":amd: (MI355 DPX) Regression" # TBD timeout_in_minutes: 25 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -5069,3 +5501,87 @@ steps: commands: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py + +- label: ':amd: (MI355) Quantized MoE' + timeout_in_minutes: 45 + working_dir: /vllm-workspace/ + source_file_dependencies: + - tests/quantization/test_rocm_moe.py + - tests/utils.py + - vllm/model_executor/models/deepseek_v2.py + - vllm/model_executor/models/qwen3_moe.py + - vllm/model_executor/models/gpt_oss.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/v1/worker/gpu/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/quantization/test_rocm_moe.py + dind: false + agent_pool: mi355_1 + num_gpus: 1 + mirror_hardwares: + - amdexperimental + - amdproduction + - amdgfx950nightly + - amdmi355 + optional: true + +- label: ':amd: (MI355) Fusion E2E TP2 AsyncTP BF16 Config Sweep' + timeout_in_minutes: 30 + working_dir: /vllm-workspace/ + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/linear.py + - vllm/model_executor/layers/utils.py + - vllm/distributed/ + - vllm/v1/attention/backends/triton_attn.py + - tests/compile/fusions_e2e/ + - tests/compile/passes/distributed/test_async_tp.py + - tests/compile/rocm/test_async_tp.py + - tests/compile/correctness_e2e/test_async_tp.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/qwen3.py + commands: + - rocm-smi + - pytest -v -s tests/compile/rocm/test_async_tp.py + - pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness + dind: false + agent_pool: mi355_2 + num_gpus: 2 + mirror_hardwares: + - amdexperimental + - amdproduction + - amdgfx950nightly + - amdmi355 + optional: true + +- label: ":amd: (MI355) MRV2 Sampler JIT Warmup" + timeout_in_minutes: 10 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - tests/jit_monitor/ + - tests/models/utils.py + - tests/models/registry.py + - tests/utils.py + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - vllm/model_executor/warmup/ + - vllm/v1/sample/ops/topk_topp_sampler.py + - vllm/utils/jit_monitor.py + - vllm/config/observability.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/jit_monitor/test_no_runtime_jit_rocm.py --timeout=300 diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 43e3113eb3..a5904fe5ac 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -1,27 +1,92 @@ group: Basic Correctness -depends_on: +depends_on: - image-build steps: -- label: ":nvidia: (H200 MIG 18GB) Basic Correctness" +- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Models %N" key: basic-correctness timeout_in_minutes: 68 device: h200_18gb - source_file_dependencies: + parallelism: 2 + source_file_dependencies: &basic-correctness-deps - vllm/ - "!vllm/distributed/kv_transfer/" - - tests/basic_correctness/test_basic_correctness - - tests/basic_correctness/test_cpu_offload - - tests/basic_correctness/test_mem.py + - tests/basic_correctness/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s basic_correctness/test_mem.py - - pytest -v -s basic_correctness/test_basic_correctness.py - - pytest -v -s basic_correctness/test_cpu_offload.py + - pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI300) Basic Correctness" + label: ":amd: (MI355 DPX) Basic Correctness Models %N" dind: false - device: mi300_1 + device: mi355_dpx + timeout_in_minutes: 60 + depends_on: + - image-build-amd + +- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CuMem" + key: basic-correctness-cumem + timeout_in_minutes: 68 + device: h200_18gb + source_file_dependencies: *basic-correctness-deps + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/memory/cumem + mirror: + amd: + label: ":amd: (MI355 DPX) Basic Correctness CuMem" + dind: false + device: mi355_dpx + timeout_in_minutes: 60 + depends_on: + - image-build-amd + +- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Sleep Mode" + key: basic-correctness-sleep-mode + timeout_in_minutes: 68 + device: h200_18gb + source_file_dependencies: *basic-correctness-deps + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/memory/sleep_mode + mirror: + amd: + label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" + dind: false + device: mi355_dpx + timeout_in_minutes: 60 + depends_on: + - image-build-amd + +- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CPU Offload" + key: basic-correctness-cpu-offload + timeout_in_minutes: 68 + device: h200_18gb + source_file_dependencies: *basic-correctness-deps + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/cpu_offload + mirror: + amd: + label: ":amd: (MI355 DPX) Basic Correctness CPU Offload" + dind: false + device: mi355_dpx + timeout_in_minutes: 60 + depends_on: + - image-build-amd + +- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Prefetch Offload" + key: basic-correctness-prefetch-offload + timeout_in_minutes: 68 + device: h200_18gb + source_file_dependencies: *basic-correctness-deps + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s basic_correctness/prefetch_offload + mirror: + amd: + label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload" + dind: false + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index 45a9e02fc8..4b681cd589 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -15,9 +15,9 @@ steps: - pytest -v -s benchmarks/ mirror: amd: - label: ":amd: (MI300) Benchmarks CLI" + label: ":amd: (MI355 DPX) Benchmarks CLI" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 40 depends_on: - image-build-amd @@ -34,3 +34,18 @@ steps: - vllm/v1/attention/ commands: - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k" + mirror: + amd: + label: ":amd: (MI355) Attention Benchmark Smoke" + dind: false + device: mi355_2 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - benchmarks/attention_benchmarks/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k" diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 219a4ca018..221c2e8838 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -67,6 +67,31 @@ steps: - pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py # this runner has 2 GPUs available even though num_devices=2 is not set - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py + mirror: + amd: + label: ":amd: (MI355) Fusion and Compile FP8" + dind: false + device: mi355_2 + num_devices: 2 + working_dir: "/vllm-workspace/" + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - tests/compile/passes/ + - tests/compile/correctness_e2e/test_attn_quant.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + commands: + - amd-smi + # MLA static FP8 and distributed AR/RMS are owned by the existing ROCm jobs. + - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py + - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py - label: ":nvidia: (H100) Fusion E2E Quick" key: fusion-e2e-quick-h100 @@ -88,9 +113,9 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and not +rms_norm and +quant_fp8 and (qwen3 or deepseek)" mirror: amd: - label: ":amd: (MI355) Fusion E2E Quick" + label: ":amd: (MI355 DPX) Fusion E2E Quick" dind: false - device: mi355_1 + device: mi355_dpx working_dir: "/vllm-workspace/" timeout_in_minutes: 30 depends_on: @@ -129,9 +154,9 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" mirror: amd: - label: ":amd: (MI355) Fusion E2E Config Sweep" + label: ":amd: (MI355 DPX) Fusion E2E Config Sweep" dind: false - device: mi355_1 + device: mi355_dpx working_dir: "/vllm-workspace/" timeout_in_minutes: 40 depends_on: @@ -181,6 +206,47 @@ steps: # Run all models and attn backends but only Inductor partition and native custom ops - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E TP2 Quick" + dind: false + device: mi355_2 + timeout_in_minutes: 60 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/ + source_file_dependencies: + - csrc/quantization/ + - vllm/compilation/ + - vllm/distributed/ + - vllm/model_executor/ + - vllm/v1/attention/ + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and Llama-3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + +quant_fp8 and +rms_norm and qwen3 and + ROCM_AITER_UNIFIED_ATTN' + - >- + VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_USE_AITER_CUSTOM_AR=1 + pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py + -k 'test_tp2_ar_rms_fp8_fusions and inductor_partition and + not +quant_fp8 and not +rms_norm and DeepSeek-Coder and + ROCM_AITER_MLA' - label: ":nvidia: (H100) Fusion E2E TP2 AR-RMS Config Sweep" key: fusion-e2e-tp2-ar-rms-config-sweep-h100 @@ -221,6 +287,33 @@ steps: - nvidia-smi # Run just llama3 (fp8 & bf16) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "llama-3" + mirror: + amd: + label: ':amd: (MI355) Fusion E2E TP2 AsyncTP BF16 Config Sweep' + dind: false + device: mi355_2 + num_devices: 2 + working_dir: /vllm-workspace/ + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/linear.py + - vllm/model_executor/layers/utils.py + - vllm/distributed/ + - vllm/v1/attention/backends/triton_attn.py + - tests/compile/fusions_e2e/ + - tests/compile/passes/distributed/test_async_tp.py + - tests/compile/rocm/test_async_tp.py + - tests/compile/correctness_e2e/test_async_tp.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/qwen3.py + commands: + - rocm-smi + - pytest -v -s tests/compile/rocm/test_async_tp.py + - pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness - label: ":nvidia: (B200) Fusion E2E TP2" key: fusion-e2e-tp2-b200 diff --git a/.buildkite/test_areas/crcr_report.yaml b/.buildkite/test_areas/crcr_report.yaml index 0dfd6f50e2..f6b6582893 100644 --- a/.buildkite/test_areas/crcr_report.yaml +++ b/.buildkite/test_areas/crcr_report.yaml @@ -1,13 +1,31 @@ group: CRCR Report +# The report waits for the rest of the build (see crcr-report.sh), so gate it on +# the long pole rather than letting it wait on an agent. A torch-nightly build +# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three +# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in +# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours. +# +# Depending on it means Buildkite does not schedule this step until the build is +# nearly over, so the agent polls briefly instead of idling ~13h. It is a +# scheduling hint, not the correctness mechanism: the script still waits for +# every job, so if this stops being the slowest step the report is late to start +# but still correct. depends_on: - image-build-cpu + - lm-eval-large-models-4xh100 steps: - label: ":satellite: Report nightly results to CRCR" key: crcr-report if: build.env("TORCH_NIGHTLY") == "1" soft_fail: true allow_dependency_failure: true - timeout_in_minutes: 15 + # depends_on defers the start to near the end of the build, so in the normal + # case the script polls for minutes. This only has to cover the case where + # the long pole finishes early and something else is still running, so it is + # sized against the whole build rather than the expected wait. The script's + # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial + # report; this is the backstop that stops the job outliving the build. + timeout_in_minutes: 840 device: cpu-small commands: - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 8ac8e1960a..02ac11879f 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -19,9 +19,9 @@ steps: - pytest -v -s cuda/test_cuda_compatibility_path.py mirror: amd: - label: ":amd: (MI355) CUDA Platform" + label: ":amd: (MI355 DPX) CUDA Platform" dind: false - device: mi355_1 + device: mi355_dpx working_dir: "/vllm-workspace/tests" timeout_in_minutes: 25 depends_on: diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 811301f58d..43abe2ce01 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -42,6 +42,29 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # Preserve all six model/TP pairs with native AITER attention. + - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py - label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy" key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus @@ -130,7 +153,7 @@ steps: label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" dind: false device: mi300_4 - timeout_in_minutes: 60 + timeout_in_minutes: 80 depends_on: - image-build-amd source_file_dependencies: @@ -205,6 +228,25 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy" + dind: false + device: mi355_2 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + - requirements/kv_connectors_rocm.txt + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh - label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance" key: nixlconnector-pd-spec-decode-acceptance-2-gpus @@ -295,6 +337,44 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP" + dind: false + device: mi355_8 + timeout_in_minutes: 85 + depends_on: + - image-build-amd + source_file_dependencies: + - requirements/kv_connectors_rocm.txt + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/models/deepseek_v4/ + - vllm/v1/attention/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # A mirror does not inherit the parent env, so it is repeated here in + # the parent's order. The layouts must stay empty: unset, the script + # defaults to HND, which the V4 indexer rejects. + - >- + VLLM_ROCM_USE_AITER=1 + ENABLE_HMA_FLAG=1 + DP_EP=1 + GPU_MEMORY_UTILIZATION=0.85 + PREFILLER_TP_SIZE=4 + DECODER_TP_SIZE=4 + PREFILL_BLOCK_SIZE=256 + DECODE_BLOCK_SIZE=256 + MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash + VLLM_ENGINE_READY_TIMEOUT_S=1800 + VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8 + PREFILLER_KV_LAYOUT= + DECODER_KV_LAYOUT= + bash v1/kv_connector/nixl_integration/run_accuracy_test.sh - label: ":nvidia: (H200) Kimi-Linear-48B-A3B Disaggregated DP EP" key: kimi-linear-disaggregated diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index d89996c9a2..030351ba57 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -19,6 +19,30 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + - tests/v1/kv_connector/mooncake_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - examples/disaggregated/mooncake_connector/ + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # Same full accuracy/TP sweep, using same-host HIP GPU transport. + - pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py - label: ":nvidia: (L4) Mooncake EC TCP E2E" key: mooncake-ec-tcp-e2e-2-gpus diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 1f2d572922..0ed630daab 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -35,9 +35,9 @@ steps: - pytest -v -s distributed/test_sharded_rdt_trainer.py mirror: amd: - label: ":amd: (MI355) Sharded RDT Weight Transfer" + label: ":amd: (MI355 DPX) Sharded RDT Weight Transfer" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 45 working_dir: /vllm-workspace/tests @@ -67,7 +67,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/worker/worker_base.py - vllm/v1/engine/ - vllm/v1/worker/ @@ -98,7 +98,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/worker/worker_base.py - vllm/v1/engine/ - vllm/v1/worker/ @@ -117,7 +117,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/worker/worker_base.py - vllm/v1/engine/ - vllm/v1/worker/ @@ -141,7 +141,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/worker/worker_base.py - vllm/v1/engine/ - vllm/v1/worker/ @@ -162,7 +162,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/worker/worker_base.py - vllm/v1/engine/ - vllm/v1/worker/ @@ -219,6 +219,41 @@ steps: # default does not fit an L4. - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py + mirror: + amd: + label: ":amd: (MI355) Distributed Torchrun + Examples" + dind: false + device: mi355_4 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - tests/distributed/test_torchrun_example.py + - tests/distributed/test_torchrun_example_moe.py + - examples/rl/ + - examples/features/data_parallel/data_parallel_offline.py + - .buildkite/scripts/install-kv-connectors.sh + - requirements/kv_connectors_rocm.txt + - requirements/test/rocm.in + - requirements/test/rocm.txt + - vllm/platforms/rocm.py + commands: + # Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies. + - uv pip install --system --no-deps 'ray==2.56.1' + - export NCCL_CUMEM_HOST_ENABLE=0 + - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py + - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py + - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py + - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py + - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py + - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py + - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py - label: ":nvidia: (L4) Distributed DP Extended" key: distributed-dp-tests-4-gpus @@ -291,6 +326,23 @@ steps: - export NCCL_CUMEM_HOST_ENABLE=0 # test with torchrun tp=2 and dp=4 with ep - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep + mirror: + amd: + label: ":amd: (MI355) Distributed DP + EP" + dind: false + device: mi355_8 + timeout_in_minutes: 50 + depends_on: + - image-build-amd + source_file_dependencies: + - examples/features/torchrun/torchrun_dp_example_offline.py + - vllm/config/parallel.py + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - vllm/v1/engine/llm_engine.py + - vllm/v1/executor/uniproc_executor.py + - vllm/v1/worker/gpu_worker.py + - vllm/platforms/rocm.py - label: ":nvidia: (A100) Distributed" key: distributed-tests-4xa100 @@ -321,6 +373,7 @@ steps: - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer_m2n.py - pytest -v -s tests/distributed/test_packed_tensor.py mirror: amd: @@ -412,7 +465,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/model_executor/models/ - tests/distributed/ - examples/features/data_parallel/data_parallel_offline.py @@ -429,8 +482,10 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/model_executor/models/ + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py - tests/distributed/ - tests/v1/distributed/test_pp_dp_v2.py commands: @@ -449,7 +504,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/model_executor/models/ - vllm/model_executor/layers/ - vllm/v1/attention/backends/ @@ -471,11 +526,11 @@ steps: - tests/distributed/test_ray_v2_executor.py - tests/distributed/test_ray_v2_executor_e2e.py - tests/distributed/test_pipeline_parallel.py - - tests/basic_correctness/test_basic_correctness.py + - tests/basic_correctness/models/test_basic_correctness.py commands: - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 - export NCCL_CUMEM_HOST_ENABLE=0 - pytest -v -s distributed/test_ray_v2_executor.py - pytest -v -s distributed/test_ray_v2_executor_e2e.py - pytest -v -s distributed/test_pipeline_parallel.py -k "ray" - - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray" + - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray" diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 20bec50b5e..9af5423a4b 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -11,6 +11,28 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 + mirror: + amd: + label: ":amd: (MI355) DeepSeek V2-Lite Sync EPLB Accuracy" + dind: false + device: mi355_4 + timeout_in_minutes: 40 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/distributed/eplb + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/backends/mla/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100 @@ -21,6 +43,25 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 + mirror: + amd: + label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP2" + dind: false + device: mi355_4 + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/eplb + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (B200) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200 @@ -31,6 +72,25 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1 + mirror: + amd: + label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy TP1" + dind: false + device: mi355_2 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/distributed/eplb + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy" key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy @@ -41,6 +101,25 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050 + mirror: + amd: + label: ":amd: (MI355) Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy" + dind: false + device: mi355_4 + timeout_in_minutes: 30 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/eplb + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) DeepSeek V2-Lite Prefetch Offload Accuracy" key: deepseek-v2-lite-prefetch-offload-accuracy-h100 @@ -53,9 +132,9 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030 mirror: amd: - label: ":amd: (MI355) DeepSeek V2-Lite Prefetch Offload Accuracy" + label: ":amd: (MI355 DPX) DeepSeek V2-Lite Prefetch Offload Accuracy" dind: false - device: mi355_1 + device: mi355_dpx working_dir: "/vllm-workspace" timeout_in_minutes: 30 depends_on: diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index e38cd92573..da02f47827 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -29,9 +29,9 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py mirror: amd: - label: ":amd: (MI300) Engine Core" + label: ":amd: (MI355 DPX) Engine Core" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 40 depends_on: - image-build-amd @@ -117,9 +117,9 @@ steps: - pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async mirror: amd: - label: ":amd: (MI355) E2E Core Large Memory" + label: ":amd: (MI355 DPX) E2E Core Large Memory" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 65 working_dir: /vllm-workspace/tests diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 9e0a8399a3..5f8e1d1c35 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -12,6 +12,7 @@ steps: working_dir: "." env: SNAPSHOT_E2E_RUN_LIMIT_S: "1200" + NVIDIA_VISIBLE_DEVICES: "0" source_file_dependencies: - .buildkite/scripts/initialized-snapshot-e2e.sh - .buildkite/test_areas/entrypoints.yaml @@ -38,9 +39,9 @@ steps: - pytest -v -s entrypoints/launchers mirror: amd: - label: ":amd: (MI355) Entrypoints Unit" + label: ":amd: (MI355 DPX) Entrypoints Unit" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 50 depends_on: - image-build-amd @@ -63,9 +64,9 @@ steps: - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests mirror: amd: - label: ":amd: (MI300) Entrypoints Integration (LLM)" + label: ":amd: (MI355 DPX) Entrypoints Integration (LLM)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 90 depends_on: - image-build-amd @@ -88,9 +89,9 @@ steps: - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI300) Entrypoints Integration (API Server) %N" + label: ":amd: (MI355 DPX) Entrypoints Integration (API Server) %N" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -110,9 +111,9 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness mirror: amd: - label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)" + label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -131,9 +132,9 @@ steps: - pytest -v -s entrypoints/openai/chat_completion mirror: amd: - label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)" + label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API chat_completion)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 70 depends_on: - image-build-amd @@ -159,9 +160,9 @@ steps: - pytest -v -s entrypoints/cohere mirror: amd: - label: ":amd: (MI300) Entrypoints Integration (API Server Generate)" + label: ":amd: (MI355 DPX) Entrypoints Integration (API Server Generate)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -179,9 +180,9 @@ steps: - pytest -v -s entrypoints/openai/responses mirror: amd: - label: ":amd: (MI355) Entrypoints Integration (Responses API)" + label: ":amd: (MI355 DPX) Entrypoints Integration (Responses API)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 50 depends_on: - image-build-amd @@ -200,9 +201,9 @@ steps: - pytest -v -s entrypoints/speech_to_text mirror: amd: - label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" + label: ":amd: (MI355 DPX) Entrypoints Integration (Speech to Text)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" depends_on: @@ -222,9 +223,9 @@ steps: - pytest -v -s entrypoints/multimodal mirror: amd: - label: ":amd: (MI355) Entrypoints Integration (Multimodal)" + label: ":amd: (MI355 DPX) Entrypoints Integration (Multimodal)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 55 depends_on: - image-build-amd @@ -243,9 +244,9 @@ steps: - pytest -v -s entrypoints/pooling mirror: amd: - label: ":amd: (MI355) Entrypoints Integration (Pooling)" + label: ":amd: (MI355 DPX) Entrypoints Integration (Pooling)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 65 working_dir: "/vllm-workspace/tests" depends_on: @@ -262,9 +263,9 @@ steps: - pytest -s entrypoints/openai/correctness/ mirror: amd: - label: ":amd: (MI300) OpenAI API Correctness" + label: ":amd: (MI355 DPX) OpenAI API Correctness" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 30 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 485d4f3d31..930a9a448a 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -16,9 +16,9 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: - label: ":amd: (MI300) EPLB Algorithm" + label: ":amd: (MI355 DPX) EPLB Algorithm" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 30 depends_on: - image-build-amd @@ -40,6 +40,19 @@ steps: commands: - pytest -v -s distributed/test_eplb_execute.py - pytest -v -s distributed/test_eplb_spec_decode.py + mirror: + amd: + label: ":amd: (MI355) EPLB Execution" + dind: false + device: mi355_4 + timeout_in_minutes: 35 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/eplb + - tests/distributed/test_eplb_execute.py + - tests/distributed/test_eplb_spec_decode.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) Elastic EP Scaling" key: elastic-ep-scaling-test @@ -51,7 +64,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/compilation/ - tests/distributed/ commands: diff --git a/.buildkite/test_areas/fault_tolerance.yaml b/.buildkite/test_areas/fault_tolerance.yaml index 2e85a1c3f8..a578be14ed 100644 --- a/.buildkite/test_areas/fault_tolerance.yaml +++ b/.buildkite/test_areas/fault_tolerance.yaml @@ -24,3 +24,33 @@ steps: # https://github.com/NVIDIA/nccl/issues/1838 - export NCCL_CUMEM_HOST_ENABLE=0 - pytest -v -s v1/fault_tolerance/test_fault_tolerance_e2e.py + mirror: + amd: + label: ":amd: (MI355) MoRI EP Healthy Baseline" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 10 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - tests/kernels/moe/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/utils/import_utils.py + - vllm/utils/math_utils.py + - vllm/utils/torch_utils.py + - vllm/platforms/ + - vllm/_aiter_ops.py + # TODO: Add fault injection, detection and recovery once MoRI supports them. + # This mirror currently checks healthy EP execution only. + commands: + - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph + - VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]' diff --git a/.buildkite/test_areas/jit_monitor.yaml b/.buildkite/test_areas/jit_monitor.yaml index 3355470c17..27de713fa2 100644 --- a/.buildkite/test_areas/jit_monitor.yaml +++ b/.buildkite/test_areas/jit_monitor.yaml @@ -20,3 +20,24 @@ steps: # instead of running until the build timeout. - export PYTHONFAULTHANDLER=1 - pytest -v -s jit_monitor/test_no_runtime_jit.py --timeout=900 --timeout-method=thread + mirror: + amd: + # Focused MRV2 regression; the broader model/backend matrix is pending. + label: ':amd: (MI355) MRV2 Sampler JIT Warmup' + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 10 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - tests/jit_monitor/ + - tests/models/utils.py + - tests/utils.py + - vllm/v1/worker/gpu/ + - vllm/v1/sample/ops/topk_topp_sampler.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/jit_monitor/test_no_runtime_jit_rocm.py --timeout=300 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 78d57ca455..c53c91cb63 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -15,9 +15,9 @@ steps: - pytest -v -s tests/kernels/ir mirror: amd: - label: ":amd: (MI355) vLLM IR" + label: ":amd: (MI355 DPX) vLLM IR" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 50 working_dir: "/vllm-workspace/" depends_on: @@ -44,9 +44,9 @@ steps: parallelism: 3 mirror: amd: - label: ":amd: (MI355) Core Operation Kernels Shard %N" + label: ":amd: (MI300) Core Operation Kernels Shard %N" dind: false - device: mi355_1 + device: mi300_1 timeout_in_minutes: 40 working_dir: "/vllm-workspace/tests" depends_on: @@ -113,9 +113,9 @@ steps: - pytest -v -s kernels/test_top_k_per_row.py mirror: amd: - label: ":amd: (MI355) DeepSeek V4 Fused Kernels" + label: ":amd: (MI355 DPX) DeepSeek V4 Fused Kernels" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 60 working_dir: /vllm-workspace/tests @@ -144,6 +144,32 @@ steps: - pytest -v -s kernels/test_fused_deepseek_v4_*.py - pytest -v -s models/test_deepseek_v4_mega_moe.py +# Own job so mHC never shares a process with FlashInfer's import-time +# ptxas override. +- label: ":nvidia: (B200) mHC Kernels" + key: kernels-mhc-test-b200 + timeout_in_minutes: 25 + device: b200-k8s + source_file_dependencies: + - csrc/ + - vllm/ + - "!vllm/distributed/kv_transfer/" + - tests/kernels/ + commands: + - pytest -v -s kernels/mhc + +- label: ":nvidia: (B200) FLA Ops Kernels" + key: kernels-fla-ops-test-b200 + timeout_in_minutes: 25 + device: b200-k8s + source_file_dependencies: + - csrc/ + - vllm/ + - "!vllm/distributed/kv_transfer/" + - tests/kernels/ + commands: + - pytest -v -s kernels/fla + # Catch-all for test files at the tests/kernels root. This job collects # the whole root so new files are wired by default. # Files with dedicated jobs elsewhere in this file are excluded via --ignore @@ -151,7 +177,7 @@ steps: # their own jobs / Kernels (B200)). - label: ":nvidia: (B200) Miscellaneous Kernels" key: kernels-root-misc-test-b200 - timeout_in_minutes: 45 + timeout_in_minutes: 30 device: b200-k8s source_file_dependencies: - csrc/ @@ -159,14 +185,14 @@ steps: - "!vllm/distributed/kv_transfer/" - tests/kernels/ commands: - # Isolate mHC from FlashInfer's import-time ptxas override. - - pytest -v -s kernels/test_mhc_kernels.py - pytest -v -s kernels/ --ignore=kernels/attention --ignore=kernels/core + --ignore=kernels/fla --ignore=kernels/helion --ignore=kernels/ir --ignore=kernels/mamba + --ignore=kernels/mhc --ignore=kernels/moe --ignore=kernels/quantization --ignore=kernels/test_concat_mla_q.py @@ -176,7 +202,6 @@ steps: --ignore=kernels/test_kda.py --ignore=kernels/test_bf16x3_router_gemm_cutedsl.py --ignore=kernels/test_ll_bf16_gemm.py - --ignore=kernels/test_mhc_kernels.py --ignore=kernels/test_shuffle_rows.py # BROKEN on main, pending kernel fixes (B200): # test_shuffle_rows.py (1: test_shuffle_rows_edge_cases) @@ -230,9 +255,9 @@ steps: - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py mirror: amd: - label: ":amd: (MI355) Attention DiffKV Kernels" + label: ":amd: (MI355 DPX) Attention DiffKV Kernels" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 35 working_dir: /vllm-workspace/tests @@ -270,9 +295,9 @@ steps: - pytest -v -s kernels/attention/test_mla_cross_layer_kernel_equivalence.py mirror: amd: - label: ":amd: (MI355) MLA Kernels" + label: ":amd: (MI355 DPX) MLA Kernels" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 40 working_dir: /vllm-workspace/ @@ -330,6 +355,8 @@ steps: - vllm/envs.py depends_on: - image-build-amd + commands: + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (L4) MoE Kernels Shard %N" key: kernels-moe-test @@ -392,6 +419,7 @@ steps: - image-build-amd source_file_dependencies: - vllm/platforms/rocm.py + - vllm/third_party/flash_linear_attention/ - label: ":nvidia: (H100) DeepGEMM Kernels" key: kernels-deepgemm-test-h100 @@ -522,6 +550,32 @@ steps: exit 1 ;; esac + mirror: + amd: + label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 10 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/kernels/quantization/test_nvfp4_emulation.py + - tests/kernels/quantization/nvfp4_utils.py + - tests/conftest.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py + - vllm/config/kernel.py + - vllm/platforms/ + - vllm/envs.py + - vllm/utils/torch_utils.py + - requirements/rocm-test.txt + # W4A4 format numerics and replay via emulation. TODO: native packed + # activation/KV-cache layouts and CUDA-only kernels need AMD backends. + commands: + - amd-smi + - pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly" key: b12x-linear-kernels-dgx-spark-nightly @@ -572,18 +626,20 @@ steps: commands: - pip install helion==1.4.0 - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - parallelism: 2 + parallelism: 5 mirror: amd: label: ":amd: (MI250) Helion Kernels Shard %N" dind: false device: mi250_1 timeout_in_minutes: 45 + # Explicit so the base step's count does not multiply the mirror's replicas. + parallelism: 2 depends_on: - image-build-amd source_file_dependencies: - vllm/platforms/rocm.py - - scripts/benchmark_helion_kernels.py + - tools/benchmark_helion_kernels.py - label: ":nvidia: (H100) FP8 MoE Kernels" @@ -684,6 +740,9 @@ steps: - vllm/utils/torch_utils.py - vllm/platforms/ - vllm/_aiter_ops.py + commands: + # MoRI graph cases run in the Fault Tolerance group's healthy EP baseline. + - pytest -v -s kernels/moe/test_moe_layer.py -k "not test_moe_layer_mori_graph" - label: ":nvidia: (B200) FusedMoE Layer Kernels" key: kernels-fusedmoe-layer-test-2-b200s @@ -701,11 +760,99 @@ steps: commands: - pytest -v -s kernels/moe/test_moe_layer.py - pytest -v -s kernels/moe/test_deepep_v2_moe.py + mirror: + amd: + label: ":amd: (MI355) MoRI EP Numerics" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 20 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/kernels/moe/test_mori_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/moe/modular_kernel_tools/parallel_utils.py + - tests/kernels/utils.py + - tests/utils.py + - tests/conftest.py + - vllm/utils/torch_utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/platforms/ + - vllm/_aiter_ops.py + - requirements/rocm-test.txt + # Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the + # DeepEP v2 FlashInfer backend matrix has no direct AMD implementation. + commands: + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py -- label: ":amd: (MI355) FP8 MoE Kernels" +- label: ":amd: (MI355 DPX) Native Quantization Kernels" + key: amd-native-quantization-kernels-mi355 + dind: false + device: mi355_dpx + num_devices: 1 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/rocm/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + - tests/kernels/quantization/test_rocm_mxfp4.py + - tests/kernels/quantization/test_rocm_mxfp8_linear.py + - tests/kernels/quantization/test_rocm_skinny_gemms.py + - tests/quantization/reference_mxfp4.py + - tests/kernels/quant_utils.py + - tests/kernels/utils.py + commands: + - pytest -v -s kernels/quantization/test_rocm_mxfp4.py + - pytest -v -s kernels/quantization/test_rocm_mxfp8_linear.py + - pytest -v -s kernels/quantization/test_rocm_skinny_gemms.py -k wvsplitkrc + +- label: ":amd: (MI355 DPX) FP4 and INT4 MoE Kernels" + key: amd-fp4-moe-kernels-mi355 + dind: false + device: mi355_dpx + num_devices: 1 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + - vllm/config/ + - tests/kernels/moe/test_rocm_aiter_moe.py + - tests/kernels/moe/test_ocp_mx_moe.py + - tests/kernels/moe/test_flydsl_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/utils.py + commands: + - pytest -v -s kernels/moe/test_rocm_aiter_moe.py -k 'mi350_mxfp4 or quant_scheme_support_matches_gfx950' + - pytest -v -s kernels/moe/test_ocp_mx_moe.py::test_rocm_mxfp4_moe_oracle -k AITER_MXFP4 + - pytest -v -s kernels/moe/test_flydsl_moe.py + +- label: ":amd: (MI355 DPX) FP8 MoE Kernels" key: amd-fp8-moe-kernels-mi355 dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 40 working_dir: /vllm-workspace/tests @@ -732,10 +879,10 @@ steps: - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py -- label: ":amd: (MI355) Kernels" +- label: ":amd: (MI355 DPX) Kernels" key: amd-kernels-mi355 dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 25 working_dir: /vllm-workspace/ diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 3450b70c28..c67cc08b94 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -68,9 +68,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt mirror: amd: - label: ":amd: (MI355) LM Eval Watermarking" + label: ":amd: (MI355 DPX) LM Eval Watermarking" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 45 working_dir: /vllm-workspace/tests @@ -114,6 +114,27 @@ steps: commands: - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4 + mirror: + amd: + label: ":amd: (MI355) LM Eval Large Models FP8" + dind: false + device: mi355_4 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - .buildkite/lm-eval-harness/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_USE_DEEP_GEMM=0 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4 - label: ":nvidia: (B200) LM Eval Small Models" key: lm-eval-small-models-1xb200 @@ -125,6 +146,28 @@ steps: - vllm/model_executor/layers/quantization commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt + mirror: + amd: + label: ":amd: (MI355 DPX) LM Eval Small Models" + dind: false + device: mi355_dpx + timeout_in_minutes: 45 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - tests/evals/gsm8k/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-small.txt - label: ":nvidia: (B200) LM Eval Small Models Distributed" key: lm-eval-small-models-distributed-2xb200 @@ -286,6 +329,7 @@ steps: - csrc/ - tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml + - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml - tests/evals/gsm8k/configs/models-spec-decode-rocm.txt - tests/evals/gsm8k/test_gsm8k_correctness.py - tests/evals/gsm8k/gsm8k_eval.py @@ -342,6 +386,33 @@ steps: - vllm/third_party/flash_linear_attention/ops/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval Qwen3.5 Models" + dind: false + device: mi355_2 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_5_mtp.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/transformers_utils/configs/qwen3_5_moe.py + - vllm/model_executor/models/qwen2.py + - vllm/model_executor/models/qwen3.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/third_party/flash_linear_attention/ops/ + - tests/evals/gsm8k/configs/models-qwen35-mi355.txt + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml + - tests/evals/gsm8k/test_gsm8k_correctness.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt - label: ":nvidia: (H200) LM Eval DeepSeek R1 TP" key: lm-eval-deepseek-r1-tp-8xh200 @@ -523,7 +594,7 @@ steps: - label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY" key: lm-eval-humming-h100-act optional: true - timeout_in_minutes: 70 + timeout_in_minutes: 90 device: h100 num_devices: 1 source_file_dependencies: @@ -534,7 +605,7 @@ steps: - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - # - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt - label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY" key: lm-eval-humming-h100-deepep @@ -587,9 +658,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt -- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant KV Cache" - key: lm-eval-turboquant-kv-cache - timeout_in_minutes: 55 +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k8v4" + key: lm-eval-turboquant-k8v4 + timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: - vllm/model_executor/layers/quantization/turboquant/ @@ -597,7 +668,155 @@ steps: - vllm/v1/attention/ops/triton_turboquant_decode.py - vllm/v1/attention/ops/triton_turboquant_store.py commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant k8v4" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t4nc" + key: lm-eval-turboquant-t4nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant t4nc" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k3v4nc" + key: lm-eval-turboquant-k3v4nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t3nc" + key: lm-eval-turboquant-t3nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant t3nc" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt - label: ":nvidia: (H100) GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xh100 @@ -775,9 +994,9 @@ steps: - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt mirror: amd: - label: ":amd: (MI355) MRCR Eval Small Models" + label: ":amd: (MI355 DPX) MRCR Eval Small Models" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 40 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 28f695f391..8321bc7a4f 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -14,9 +14,9 @@ steps: parallelism: 4 mirror: amd: - label: ":amd: (MI300) LoRA Shard %N" + label: ":amd: (MI355 DPX) LoRA Shard %N" dind: false - device: mi300_1 + device: mi355_dpx working_dir: "/vllm-workspace/tests" timeout_in_minutes: 85 source_file_dependencies: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 6709dbb565..f4b07ffd97 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -29,9 +29,9 @@ steps: - pytest -v -s v1/sample mirror: amd: - label: ":amd: (MI300) V1 Sample" + label: ":amd: (MI355 DPX) V1 Sample" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 70 depends_on: - image-build-amd @@ -51,9 +51,9 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: - label: ":amd: (MI300) V1 Logits + Oracle" + label: ":amd: (MI355 DPX) V1 Logits + Oracle" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 70 depends_on: - image-build-amd @@ -96,9 +96,9 @@ steps: - pytest -v -s -m 'not cpu_test' v1/core mirror: amd: - label: ":amd: (MI300) V1 Core" + label: ":amd: (MI355 DPX) V1 Core" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -116,9 +116,9 @@ steps: - pytest -v -s v1/streaming_input mirror: amd: - label: ":amd: (MI300) V1 Executor + Worker" + label: ":amd: (MI355 DPX) V1 Executor + Worker" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -135,9 +135,9 @@ steps: - pytest -v -s v1/simple_kv_offload mirror: amd: - label: ":amd: (MI300) V1 KV Offload" + label: ":amd: (MI355 DPX) V1 KV Offload" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -155,9 +155,9 @@ steps: - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI300) V1 KV Connectors Shard %N" + label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -182,9 +182,9 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: - label: ":amd: (MI300) V1 Metrics + LM Eval" + label: ":amd: (MI355 DPX) V1 Metrics + LM Eval" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -211,6 +211,7 @@ steps: - vllm/utils/ - vllm/v1/ - tests/v1 + - tests/watermarking device: cpu-small commands: # split the test to avoid interference @@ -222,6 +223,7 @@ steps: - pytest -v -s -m 'cpu_test' v1/kv_connector/unit - pytest -v -s -m 'cpu_test' v1/ec_connector/unit - pytest -v -s -m 'cpu_test' v1/metrics + - pytest -v -s watermarking - label: ":nvidia: (H200 MIG 18GB) Extract Hidden States Integration" key: extract-hidden-states-integration @@ -239,9 +241,9 @@ steps: - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration mirror: amd: - label: ":amd: (MI355) Extract Hidden States Integration Single GPU" + label: ":amd: (MI355 DPX) Extract Hidden States Integration Single GPU" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 45 working_dir: /vllm-workspace @@ -328,9 +330,9 @@ steps: working_dir: "/vllm-workspace/tests" # optional mirror: amd: - label: ":amd: (MI355) Regression" + label: ":amd: (MI355 DPX) Regression" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 25 depends_on: - image-build-amd @@ -370,9 +372,9 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 mirror: amd: - label: ":amd: (MI300) Examples" + label: ":amd: (MI355 DPX) Examples" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 75 source_file_dependencies: - vllm/entrypoints @@ -472,9 +474,9 @@ steps: - pytest -v -s utils_ mirror: amd: - label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker" + label: ":amd: (MI355 DPX) Async Engine, Inputs, Utils, Worker" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 50 depends_on: - image-build-amd @@ -493,6 +495,7 @@ steps: source_file_dependencies: &cpu-async-engine-deps - vllm/assets/ - vllm/config/ + - vllm/device_allocator/ - vllm/engine/arg_utils.py - vllm/entrypoints/chat_utils.py - vllm/entrypoints/mcp/ @@ -518,6 +521,7 @@ steps: - vllm/v1/ - tests/test_envs.py - tests/test_outputs.py + - tests/test_pcp_dp.py - tests/test_pooling_params.py - tests/test_ray_env.py - tests/test_sampling_params.py @@ -533,11 +537,14 @@ steps: - tests/parser - tests/transformers_utils - tests/config + - tests/device_allocator device: cpu-small commands: - python3 standalone_tests/lazy_imports.py + - pytest -v -s device_allocator - pytest -v -s test_envs.py - pytest -v -s test_outputs.py + - pytest -v -s test_pcp_dp.py - pytest -v -s test_pooling_params.py - pytest -v -s test_ray_env.py - pytest -v -s test_sampling_params.py @@ -665,9 +672,9 @@ steps: - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test mirror: amd: - label: ":amd: (MI355) Acceptance Length (Large Models)" + label: ":amd: (MI355 DPX) Acceptance Length (Large Models)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 45 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 17818cc152..a23359b768 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -33,7 +33,7 @@ steps: label: ":amd: (MI300) Model Executor" dind: false device: mi300_1 - timeout_in_minutes: 60 + timeout_in_minutes: 75 depends_on: - image-build-amd source_file_dependencies: @@ -48,3 +48,8 @@ steps: - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export PYTHONFAULTHANDLER=1 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 930c2dd590..83a08cfc24 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -16,9 +16,9 @@ steps: - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset mirror: amd: - label: ":amd: (MI355) Basic Models (Initialization)" + label: ":amd: (MI355 DPX) Basic Models (Initialization)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 30 depends_on: - image-build-amd @@ -39,9 +39,9 @@ steps: parallelism: 14 mirror: amd: - label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" + label: ":amd: (MI355 DPX) Basic Models (Extra Initialization) Shard %N" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 60 parallelism: 10 working_dir: "/vllm-workspace/tests" @@ -70,9 +70,9 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py mirror: amd: - label: ":amd: (MI300) Basic Models (Other)" + label: ":amd: (MI355 DPX) Basic Models (Other)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -109,6 +109,23 @@ steps: commands: # The native NVIDIA Kimi K3 kernels require the SM100 family. - pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py + mirror: + amd: + label: ":amd: (MI355) Kimi K3" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + commands: + # The ROCm KDA/AttnRes kernels are gated on gfx950, so this has to be MI355. + # The two kernels/ files the NVIDIA job adds are CUDA-only (they import + # cuda.bindings.driver), so they are deliberately left out here. + - pytest -v -s models/kimi_k3 - label: ":nvidia: (H200) GLM5Next" key: glm5next-unit-tests @@ -121,9 +138,9 @@ steps: - pytest -v -s models/glm5next mirror: amd: - label: ":amd: (MI355) GLM5Next" + label: ":amd: (MI355 DPX) GLM5Next" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 35 working_dir: "/vllm-workspace/tests" depends_on: @@ -145,13 +162,15 @@ steps: - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py mirror: amd: - label: ":amd: (MI300) Qwen4 Exp" + label: ":amd: (MI355 DPX) Qwen4 Exp" dind: false - device: mi300_1 + device: mi355_dpx depends_on: - image-build-amd commands: - pytest -v -s models/qwen4_exp/test_qsa_amd.py + # PLE and QSA import conflicting HC registrations in a shared process. + - pytest -v -s models/qwen4_exp/test_ple.py - label: ":computer: (CPU) Qwen4 Exp" key: qwen4-exp-unit-tests-cpu diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index c8473e7f0b..fa8013ec54 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -16,9 +16,9 @@ steps: - pytest -v -s models/language -m 'core_model and (not slow_test)' mirror: amd: - label: ":amd: (MI300) Language Models (Standard)" + label: ":amd: (MI355 DPX) Language Models (Standard)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -40,9 +40,9 @@ steps: parallelism: 2 mirror: amd: - label: ":amd: (MI300) Language Models (Extra Standard) Shard %N" + label: ":amd: (MI355 DPX) Language Models (Extra Standard) Shard %N" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 40 depends_on: - image-build-amd @@ -82,9 +82,9 @@ steps: parallelism: 2 mirror: amd: - label: ":amd: (MI355) Language Models (Hybrid) Shard %N" + label: ":amd: (MI355 DPX) Language Models (Hybrid) Shard %N" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -113,9 +113,9 @@ steps: - pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview' mirror: amd: - label: ":amd: (MI355) Granite Language Model Compatibility" + label: ":amd: (MI355 DPX) Granite Language Model Compatibility" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 80 working_dir: /vllm-workspace/tests @@ -151,9 +151,9 @@ steps: - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' mirror: amd: - label: ":amd: (MI355) Language Models (Extended Generation)" + label: ":amd: (MI355 DPX) Language Models (Extended Generation)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 95 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 779801a2ea..e07f38a7bd 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -15,9 +15,9 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: - label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" + label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 65 depends_on: - image-build-amd @@ -36,9 +36,9 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: amd: - label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" + label: ":amd: (MI355 DPX) Multimodal Models (Standard) 2: qwen3 + gemma" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 55 depends_on: - image-build-amd @@ -78,9 +78,9 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: - label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" + label: ":amd: (MI355 DPX) Multimodal Models (Standard) 4: other + whisper" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 70 depends_on: - image-build-amd @@ -125,9 +125,9 @@ steps: - pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI355) Multimodal Processor Shard %N" + label: ":amd: (MI355 DPX) Multimodal Processor Shard %N" dind: false - device: mi355_1 + device: mi355_dpx parallelism: 4 timeout_in_minutes: 115 depends_on: @@ -146,9 +146,9 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 mirror: amd: - label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" + label: ":amd: (MI355 DPX) Multimodal Accuracy Eval (Small Models)" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 35 depends_on: - image-build-amd @@ -212,9 +212,9 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI355) Multimodal Models (Extended Generation 2) Shard %N" + label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 2) Shard %N" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 100 depends_on: - image-build-amd @@ -231,9 +231,9 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' mirror: amd: - label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" + label: ":amd: (MI355 DPX) Multimodal Models (Extended Generation 3)" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 90 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 6b901d8f49..d9653e84ee 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -78,9 +78,9 @@ steps: - pytest -v -s plugins_tests/gguf mirror: amd: - label: ":amd: (MI355) GGUF Plugin" + label: ":amd: (MI355 DPX) GGUF Plugin" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" depends_on: diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 8204fb9670..38f0f617bc 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -89,9 +89,9 @@ steps: - "find compile/h100/ -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" mirror: amd: - label: ":amd: (MI355) PyTorch Compilation H100 Cases" + label: ":amd: (MI355 DPX) PyTorch Compilation H100 Cases" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" depends_on: @@ -192,10 +192,14 @@ steps: - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" mirror: amd: - label: ":amd: (MI355) PyTorch Fullgraph" + label: ":amd: (MI355 DPX) PyTorch Fullgraph" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 90 + # Remove once the ROCm Triton pin includes triton-lang/triton#11567. + # Avoid overlapping LLVM codegen/LLD by disabling background precompilation. + env: + TORCHINDUCTOR_PRECOMPILATION_TIMEOUT_SECONDS: "0" depends_on: - image-build-amd diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index 702bbe7024..d1f47cf2a7 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -19,12 +19,12 @@ steps: - uv pip install --system conch-triton-kernels # The SM90-only checkpoint currently contains a removed weight_chan_scale # parameter. It was not exercised by the previous L4 job. - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT mirror: amd: - label: ":amd: (MI355) Quantization Shard %N" + label: ":amd: (MI300) Quantization Shard %N" dind: false - device: mi355_1 + device: mi300_1 timeout_in_minutes: 115 working_dir: "/vllm-workspace/tests" depends_on: @@ -43,7 +43,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions" @@ -76,6 +76,32 @@ steps: - vllm/v1/attention/backends/flashinfer.py commands: - pytest -s -v tests/quantization/test_blackwell_moe.py + mirror: + amd: + label: ":amd: (MI355) Quantized MoE" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/quantization/test_rocm_moe.py + - tests/utils.py + - vllm/model_executor/models/deepseek_v2.py + - vllm/model_executor/models/qwen3_moe.py + - vllm/model_executor/models/gpt_oss.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/v1/worker/gpu/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/quantization/test_rocm_moe.py - label: ":nvidia: (H200 MIG 35GB) Quantized Models" device: h200_35gb @@ -90,9 +116,9 @@ steps: - pytest -v -s models/quantization mirror: amd: - label: ":amd: (MI355) Quantized Models" + label: ":amd: (MI355 DPX) Quantized Models" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 80 working_dir: "/vllm-workspace/tests" depends_on: diff --git a/.buildkite/test_areas/ray_compat.yaml b/.buildkite/test_areas/ray_compat.yaml index dc8571b260..a177ec7131 100644 --- a/.buildkite/test_areas/ray_compat.yaml +++ b/.buildkite/test_areas/ray_compat.yaml @@ -18,9 +18,9 @@ steps: - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh mirror: amd: - label: ":amd: (MI355) Ray Dependency Compatibility Check" + label: ":amd: (MI355 DPX) Ray Dependency Compatibility Check" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 25 working_dir: "/" depends_on: diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index a679b405ed..c5899a9c59 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -43,9 +43,9 @@ steps: - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" mirror: amd: - label: ":amd: (MI355) Rust Frontend OpenAI Coverage" + label: ":amd: (MI355 DPX) Rust Frontend OpenAI Coverage" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 35 depends_on: - image-build-amd @@ -98,9 +98,9 @@ steps: - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info" mirror: amd: - label: ":amd: (MI355) Rust Frontend Serve/Admin Coverage" + label: ":amd: (MI355 DPX) Rust Frontend Serve/Admin Coverage" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 30 depends_on: - image-build-amd @@ -133,9 +133,9 @@ steps: - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: - label: ":amd: (MI355) Rust Frontend Core Correctness" + label: ":amd: (MI355 DPX) Rust Frontend Core Correctness" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 25 depends_on: - image-build-amd @@ -163,9 +163,9 @@ steps: - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice" mirror: amd: - label: ":amd: (MI355) Rust Frontend Tool Use" + label: ":amd: (MI355 DPX) Rust Frontend Tool Use" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 30 depends_on: - image-build-amd @@ -188,7 +188,7 @@ steps: - vllm/distributed/ - "!vllm/distributed/kv_transfer/" - vllm/engine/ - - vllm/executor/ + - vllm/v1/executor/ - vllm/v1/engine/ - vllm/v1/worker/ - tests/utils.py @@ -204,3 +204,27 @@ steps: - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info" + mirror: + amd: + label: ":amd: (MI355) Rust Frontend Distributed" + dind: false + device: mi355_4 + timeout_in_minutes: 45 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + depends_on: + - image-build-amd + source_file_dependencies: + - rust/ + - vllm/distributed/ + - "!vllm/distributed/kv_transfer/" + - vllm/engine/ + - vllm/v1/executor/ + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/utils.py + - tests/v1/distributed/test_dense_dp_world_size.py + - tests/v1/distributed/test_external_lb_dp.py + - tests/v1/distributed/test_hybrid_lb_dp.py + - tests/v1/distributed/test_internal_lb_dp.py + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 084583c8a7..bc811883ee 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -53,9 +53,9 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers/test_beam_search.py::test_beam_search_passes_multimodal_data mirror: amd: - label: ":amd: (MI355) Samplers Multimodal Beam Search" + label: ":amd: (MI355 DPX) Samplers Multimodal Beam Search" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 45 working_dir: /vllm-workspace/tests diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index f60d745df3..8a41a4c993 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -44,9 +44,9 @@ steps: - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" mirror: amd: - label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" + label: ":amd: (MI355 DPX) Spec Decode Eagle 1: DeepSeek + Qwen" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -71,9 +71,9 @@ steps: - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" mirror: amd: - label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" + label: ":amd: (MI355 DPX) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 depends_on: - image-build-amd @@ -157,9 +157,9 @@ steps: - python3 spec_decode/test_custom_proposer.py mirror: amd: - label: ":amd: (MI300) Spec Decode N-Gram + Suffix" + label: ":amd: (MI355 DPX) Spec Decode N-Gram + Suffix" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 35 depends_on: - image-build-amd @@ -172,22 +172,24 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py -- label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model" +- label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model Shard %N" key: spec-decode-draft-model timeout_in_minutes: 60 device: h200_18gb + parallelism: 4 source_file_dependencies: - vllm/v1/spec_decode/ - vllm/v1/worker/gpu/spec_decode/ - tests/v1/e2e/spec_decode/ commands: - - pytest -v -s v1/e2e/spec_decode/draft_model/ + - pytest -v -s v1/e2e/spec_decode/draft_model/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI300) Spec Decode Draft Model" + label: ":amd: (MI355 DPX) Spec Decode Draft Model Shard %N" dind: false - device: mi300_1 + device: mi355_dpx timeout_in_minutes: 60 + parallelism: 2 depends_on: - image-build-amd source_file_dependencies: @@ -288,7 +290,8 @@ steps: - vllm/model_executor/models/laguna_dflash.py - tests/v1/e2e/spec_decode/ commands: - - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ + # DFlash2 NVFP4 (27B) does not fit a 35GB MIG slice; runs on B200 below. + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2" mirror: amd: label: ":amd: (MI300) Spec Decode AL DFlash Nightly" @@ -314,6 +317,21 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py +- label: ":nvidia: (B200) Spec Decode AL DFlash2 Nightly" + key: spec-decode-dflash2-nightly-b200 + timeout_in_minutes: 60 + device: b200-k8s + optional: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "dflash2" + - label: ":nvidia: (H200 MIG 35GB) Spec Decode AL DSpark Nightly" key: spec-decode-dspark-nightly timeout_in_minutes: 60 @@ -374,9 +392,9 @@ steps: - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi mirror: amd: - label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" + label: ":amd: (MI355 DPX) Spec Decode AL MTP + Other Acceptance Nightly" dind: false - device: mi355_1 + device: mi355_dpx timeout_in_minutes: 55 depends_on: - image-build-amd diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index b7a9fdb4e0..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,47 +0,0 @@ -[run] -# Track the installed vllm package (this is what actually gets imported during tests) -# Use wildcard pattern to match the installed location -source = - vllm - */dist-packages/vllm - */site-packages/vllm -omit = - */tests/* - */test_* - */__pycache__/* - */build/* - */dist/* - */vllm.egg-info/* - */third_party/* - */examples/* - */benchmarks/* - */docs/* - -[paths] -# Map all possible vllm locations to a canonical "vllm" path -# This ensures coverage.combine properly merges data from different test runs -source = - vllm - /vllm-workspace/src/vllm - /vllm-workspace/vllm - */site-packages/vllm - */dist-packages/vllm - -[report] -exclude_lines = - pragma: no cover - def __repr__ - if self.debug: - if settings.DEBUG - raise AssertionError - raise NotImplementedError - if 0: - if __name__ == .__main__.: - class .*\bProtocol\): - @(abc\.)?abstractmethod - -[html] -directory = htmlcov - -[xml] -output = coverage.xml diff --git a/.gemini/config.yaml b/.gemini/config.yaml deleted file mode 100644 index 2499d3f095..0000000000 --- a/.gemini/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github -have_fun: false # Just review the code -code_review: - comment_severity_threshold: HIGH # Reduce quantity of comments - pull_request_opened: - summary: false # Don't summarize the PR in a separate comment diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f64cf5f561..256707fc9a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -50,7 +50,7 @@ # Rust Frontend /rust/ @BugenZhao @njhill /rust/src/bench @esmeetu -/build_rust.sh @BugenZhao @njhill +/tools/build_rust.sh @BugenZhao @njhill /rust-toolchain.toml @BugenZhao @njhill /.buildkite/test_areas/rust* @BugenZhao @njhill @@ -134,7 +134,7 @@ mkdocs.yaml @hmellor # Linting -.markdownlint.yaml @hmellor +/tools/pre_commit/.markdownlint.yaml @hmellor .pre-commit-config.yaml @hmellor /tools/pre_commit @hmellor diff --git a/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to .github/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/codecov.yml b/.github/codecov.yml similarity index 100% rename from codecov.yml rename to .github/codecov.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4823f788dc..24dcf826fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: hooks: - id: markdownlint-cli2 language_version: lts - args: [--fix] + args: [--config, tools/pre_commit/.markdownlint.yaml, --fix] exclude: (^|/)CLAUDE\.md$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 @@ -39,7 +39,7 @@ repos: rev: 745eface02aef23e168a8afb6b5737818efbea95 # v0.11.0.1 hooks: - id: shellcheck - args: ["-s", "bash", "--format=gcc"] + args: ["-s", "bash", "--format=gcc", "--rcfile=tools/pre_commit/.shellcheckrc"] - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.11.1 hooks: diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 43c994c2d3..0000000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/CMakeLists.txt b/CMakeLists.txt index dbf4ca90e2..02e098f2d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,8 +460,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") DEFINED CMAKE_CUDA_COMPILER_VERSION AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0) - list(APPEND VLLM_STABLE_EXT_SRC "csrc/libtorch_stable/all_reduce_mhc.cu") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS "9.0a;10.0f;10.1f;10.3f;10.7f;11.0f;12.0f;12.1f" "${CUDA_ARCHS}") @@ -777,22 +775,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "(requires SM90+ and CUDA >= 12.0).") endif() - # Only build AllSpark kernels if we are building for at least some compatible archs. - cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}") - if (ALLSPARK_ARCHS) - set(ALLSPARK_SRCS - "csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu" - "csrc/libtorch_stable/quantization/gptq_allspark/allspark_qgemm_w8a16.cu") - set_gencode_flags_for_srcs( - SRCS "${ALLSPARK_SRCS}" - CUDA_ARCHS "${ALLSPARK_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${ALLSPARK_SRCS}") - message(STATUS "Building AllSpark kernels for archs: ${ALLSPARK_ARCHS}") - else() - message(STATUS "Not building AllSpark kernels as no compatible archs found" - " in CUDA target architectures") - endif() - # # CUTLASS scaled_mm kernels (moved from _C to _C_stable_libtorch) # diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index bae0149f10..b8e5f5be4c 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -630,10 +630,6 @@ def get_tokenizer( pretrained_model_name_or_path ): pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - kwargs["use_fast"] = False if tokenizer_mode == "mistral": try: from vllm.tokenizers.mistral import MistralTokenizer diff --git a/benchmarks/kernels/benchmark_marlin.py b/benchmarks/kernels/benchmark_marlin.py index 751211993a..f2330a4ff0 100644 --- a/benchmarks/kernels/benchmark_marlin.py +++ b/benchmarks/kernels/benchmark_marlin.py @@ -6,10 +6,6 @@ import torch.utils.benchmark as benchmark from benchmark_shapes import WEIGHT_SHAPES from vllm import _custom_ops as ops -from vllm.model_executor.layers.quantization.utils.allspark_utils import ( - ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - ALLSPARK_SUPPORTED_QUANT_TYPES, -) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, @@ -31,7 +27,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import ( from vllm.model_executor.layers.quantization.utils.quant_utils import ( gptq_pack, gptq_quantize_weights, - quantize_weights, ) from vllm.scalar_type import ScalarType, scalar_types from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -62,9 +57,6 @@ def bench_run( return repack_supported = group_size in MARLIN_SUPPORTED_GROUP_SIZES - allspark_supported = ( - quant_type in ALLSPARK_SUPPORTED_QUANT_TYPES and group_size == -1 - ) def gen_marlin_params(): # Marlin quant @@ -104,35 +96,6 @@ def bench_run( q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n) return q_w_gptq - def gen_allspark_params(): - qw_reorder = s_reorder = zp_reorder = sm_count = sm_version = ( - CUBLAS_M_THRESHOLD - ) = None - nonlocal allspark_supported - if allspark_supported: - properties = torch.cuda.get_device_properties(b.device.index) - sm_count = properties.multi_processor_count - sm_version = properties.major * 10 + properties.minor - - supported_arch = sm_version >= 80 and sm_version < 90 - allspark_supported = allspark_supported and supported_arch - if supported_arch: - w_ref, qw, s, zp = quantize_weights(b, quant_type, group_size, has_zp) - qw = qw.to(torch.uint8) - - qw_reorder, s_reorder, zp_reorder = ops.allspark_repack_weight( - qw, s, zp, has_zp - ) - CUBLAS_M_THRESHOLD = ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD - return ( - qw_reorder, - s_reorder, - zp_reorder, - sm_count, - sm_version, - CUBLAS_M_THRESHOLD, - ) - ( marlin_w_ref, marlin_q_w, @@ -141,9 +104,6 @@ def bench_run( marlin_zp, ) = gen_marlin_params() q_w_gptq = gen_repack_params() - qw_reorder, s_reorder, zp_reorder, sm_count, sm_version, CUBLAS_M_THRESHOLD = ( - gen_allspark_params() - ) # Prepare marlin_workspace = MarlinWorkspace( @@ -167,17 +127,9 @@ def bench_run( "marlin_workspace": marlin_workspace, # GPTQ params "q_w_gptq": q_w_gptq, - # AllSpark W8A16 params - "qw_reorder": qw_reorder, - "s_reorder": s_reorder, - "zp_reorder": zp_reorder, - "sm_count": sm_count, - "sm_version": sm_version, - "CUBLAS_M_THRESHOLD": CUBLAS_M_THRESHOLD, # Kernels "marlin_gemm": ops.marlin_gemm, "gptq_marlin_repack": ops.gptq_marlin_repack, - "allspark_w8a16_gemm": ops.allspark_w8a16_gemm, } min_run_time = 1 @@ -227,17 +179,6 @@ def bench_run( ).blocked_autorange(min_run_time=min_run_time) ) - if allspark_supported: - results.append( - benchmark.Timer( - stmt="output = allspark_w8a16_gemm(a, qw_reorder, s_reorder, zp_reorder, size_n, group_size, sm_count, sm_version, CUBLAS_M_THRESHOLD, False, True)", # noqa: E501 - globals=globals, - label=label, - sub_label=sub_label, - description="allspark_w8a16_gemm_fp32", - ).blocked_autorange(min_run_time=min_run_time) - ) - def main(args): print("Benchmarking models:") diff --git a/benchmarks/kernels/benchmark_multimem_reduce_scatter.py b/benchmarks/kernels/benchmark_multimem_reduce_scatter.py new file mode 100644 index 0000000000..0f0966eff4 --- /dev/null +++ b/benchmarks/kernels/benchmark_multimem_reduce_scatter.py @@ -0,0 +1,480 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Benchmark production MNNVL reduce-scatter backends on SM100/SM103. + +The multimem candidate includes the device-to-device copy into its persistent +symmetric input buffer. Timings use pointer-distinct CUDA graphs and report the +median of the per-sample maximum across ranks. + +Run the default BF16 message-size sweep with: + + .venv/bin/python -m torch.distributed.run --standalone --nproc-per-node=8 \ + benchmarks/kernels/benchmark_multimem_reduce_scatter.py +""" + +import argparse +import json +import os +import statistics +from collections.abc import Callable +from dataclasses import dataclass + +import torch +import torch.distributed as dist + +import vllm._custom_ops as ops +from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce + +_DTYPES = { + "bf16": torch.bfloat16, + "f16": torch.float16, + "f32": torch.float32, +} + + +@dataclass +class Candidate: + name: str + runs: list[Callable[[], None]] + outputs: list[torch.Tensor] + + +def parse_size(value: str) -> int: + suffixes = {"k": 1 << 10, "m": 1 << 20, "g": 1 << 30} + value = value.strip().lower() + multiplier = suffixes.get(value[-1], 1) + number = value[:-1] if multiplier != 1 else value + return int(float(number) * multiplier) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--message-bytes", + type=parse_size, + nargs="+", + default=[ + 16 << 20, + 32 << 20, + 64 << 20, + 68 << 20, + 72 << 20, + 76 << 20, + 96 << 20, + ], + help="Full input bytes on each rank. K/M/G suffixes are accepted.", + ) + parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16") + parser.add_argument( + "--blocks", + type=int, + nargs="+", + default=[8], + help="Maximum CTAs used by each multimem candidate.", + ) + parser.add_argument("--num-workspaces", type=int, default=4) + parser.add_argument("--warmup-replays", type=int, default=8) + parser.add_argument("--samples", type=int, default=31) + parser.add_argument( + "--cold-l2", + action="store_true", + help="Flush approximately twice L2 capacity before each replay.", + ) + return parser.parse_args() + + +def make_lamport_run( + comm: CustomAllreduce, + input_tensor: torch.Tensor, + output: torch.Tensor, +) -> Callable[[], None]: + def run() -> None: + ops.mnnvl_lamport_reduce_scatter( + comm._ptr, + input_tensor, + output, + comm.mnnvl_lamport_rs_local_ptr, + comm.mnnvl_lamport_rs_epoch_ptr, + comm.mnnvl_buffer_size, + ) + + return run + + +def make_multimem_run( + comm: CustomAllreduce, + input_tensor: torch.Tensor, + output: torch.Tensor, + blocks: int, +) -> Callable[[], None]: + def run() -> None: + ops.mnnvl_multimem_reduce_scatter( + comm._ptr, + input_tensor, + output, + comm.mnnvl_multimem_rs_local_ptr, + comm.mnnvl_multimem_rs_multicast_ptr, + comm.mnnvl_multimem_rs_buffer_size, + blocks, + ) + + return run + + +def make_nccl_run( + input_tensor: torch.Tensor, + output: torch.Tensor, + device_group: dist.ProcessGroup, +) -> Callable[[], None]: + def run() -> None: + dist.reduce_scatter_single(output, input_tensor, group=device_group) + + return run + + +def build_candidates( + comm: CustomAllreduce, + inputs: list[torch.Tensor], + local_elements: int, + blocks_values: list[int], + device_group: dist.ProcessGroup, +) -> list[Candidate]: + candidates: list[Candidate] = [] + + lamport_outputs = [ + input_tensor.new_empty(local_elements) for input_tensor in inputs + ] + candidates.append( + Candidate( + "lamport", + [ + make_lamport_run(comm, input_tensor, output) + for input_tensor, output in zip(inputs, lamport_outputs) + ], + lamport_outputs, + ) + ) + + for blocks in blocks_values: + outputs = [input_tensor.new_empty(local_elements) for input_tensor in inputs] + candidates.append( + Candidate( + f"multimem_b{blocks}", + [ + make_multimem_run(comm, input_tensor, output, blocks) + for input_tensor, output in zip(inputs, outputs) + ], + outputs, + ) + ) + + nccl_outputs = [input_tensor.new_empty(local_elements) for input_tensor in inputs] + candidates.append( + Candidate( + "nccl", + [ + make_nccl_run(input_tensor, output, device_group) + for input_tensor, output in zip(inputs, nccl_outputs) + ], + nccl_outputs, + ) + ) + return candidates + + +def check_correctness( + candidates: list[Candidate], + inputs: list[torch.Tensor], + rank: int, + world_size: int, + cpu_group: dist.ProcessGroup, + device_group: dist.ProcessGroup, + graphs: dict[str, list[torch.cuda.CUDAGraph]] | None = None, +) -> None: + expected_outputs = [] + for input_tensor in inputs: + expected = input_tensor.float().clone() + dist.all_reduce(expected, group=device_group) + expected_outputs.append(expected.to(input_tensor.dtype).chunk(world_size)[rank]) + + if inputs[0].dtype == torch.float32: + rtol, atol = 1e-5, 1e-5 + elif inputs[0].dtype == torch.float16: + rtol, atol = 1e-2, 1e-2 + else: + rtol, atol = 5e-2, 1.25e-1 + + for candidate in candidates: + for workspace_index, expected in enumerate(expected_outputs): + dist.barrier(group=cpu_group) + if graphs is None: + candidate.runs[workspace_index]() + else: + graphs[candidate.name][workspace_index].replay() + torch.accelerator.synchronize() + error = None + try: + torch.testing.assert_close( + candidate.outputs[workspace_index], + expected, + rtol=rtol, + atol=atol, + ) + except AssertionError as assertion_error: + error = assertion_error + correct = torch.tensor( + error is None, + dtype=torch.int32, + device=candidate.outputs[workspace_index].device, + ) + dist.all_reduce(correct, op=dist.ReduceOp.MIN, group=device_group) + if not correct.item(): + location = f"{candidate.name} workspace {workspace_index}" + detail = str(error) if error is not None else "failed on another rank" + raise AssertionError( + f"{location} produced an incorrect shard: {detail}" + ) + dist.barrier(group=cpu_group) + + +def capture_graph( + run: Callable[[], None], + cpu_group: dist.ProcessGroup, +) -> tuple[torch.cuda.CUDAGraph, torch.cuda.Stream]: + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + dist.barrier(group=cpu_group) + with torch.cuda.stream(stream): + for _ in range(3): + run() + stream.synchronize() + dist.barrier(group=cpu_group) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + run() + torch.cuda.current_stream().wait_stream(stream) + dist.barrier(group=cpu_group) + return graph, stream + + +def capture_candidates( + candidates: list[Candidate], + cpu_group: dist.ProcessGroup, +) -> tuple[dict[str, list[torch.cuda.CUDAGraph]], list[object]]: + graphs = {} + keepalive: list[object] = [] + for candidate in candidates: + bundles = [capture_graph(run, cpu_group) for run in candidate.runs] + graphs[candidate.name] = [graph for graph, _ in bundles] + keepalive.extend(bundles) + return graphs, keepalive + + +def benchmark_graphs( + graphs: dict[str, list[torch.cuda.CUDAGraph]], + warmup_replays: int, + samples: int, + device_group: dist.ProcessGroup, + device_barrier: Callable[[], None], + flush: torch.Tensor | None, +) -> dict[str, float]: + names = list(graphs) + for round_index in range(warmup_replays): + for candidate_index in range(len(names)): + name = names[(round_index + candidate_index) % len(names)] + device_barrier() + graphs[name][round_index % len(graphs[name])].replay() + torch.accelerator.synchronize() + + timings = {name: [] for name in names} + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + for sample_index in range(samples): + for candidate_index in range(len(names)): + name = names[(sample_index + candidate_index) % len(names)] + if flush is not None: + flush.zero_() + device_barrier() + start.record() + graphs[name][sample_index % len(graphs[name])].replay() + end.record() + end.synchronize() + + elapsed = torch.tensor( + start.elapsed_time(end) * 1000, + dtype=torch.float64, + device=torch.accelerator.current_device_index(), + ) + dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group) + timings[name].append(elapsed.item()) + + return {name: statistics.median(values) for name, values in timings.items()} + + +def benchmark_size( + args: argparse.Namespace, + comm: CustomAllreduce, + message_bytes: int, + dtype: torch.dtype, + device: torch.device, + rank: int, + world_size: int, + device_group: dist.ProcessGroup, + cpu_group: dist.ProcessGroup, + device_barrier: Callable[[], None], +) -> dict[str, object]: + element_size = dtype.itemsize + if message_bytes % (element_size * world_size) != 0: + raise ValueError("Message bytes must divide evenly into TP shards") + num_elements = message_bytes // element_size + local_elements = num_elements // world_size + if local_elements * element_size % 16 != 0: + raise ValueError("Each output shard must be a multiple of 16 bytes") + + inputs = [] + for workspace_index in range(args.num_workspaces): + generator = torch.Generator(device=device) + generator.manual_seed(1000 + rank * 101 + workspace_index) + input_tensor = torch.empty(num_elements, dtype=dtype, device=device) + input_tensor.uniform_(-1.0, 1.0, generator=generator) + inputs.append(input_tensor) + + candidates = build_candidates( + comm, + inputs, + local_elements, + args.blocks, + device_group, + ) + torch.accelerator.synchronize() + dist.barrier(group=cpu_group) + check_correctness( + candidates, + inputs, + rank, + world_size, + cpu_group, + device_group, + ) + graphs, graph_keepalive = capture_candidates(candidates, cpu_group) + check_correctness( + candidates, + inputs, + rank, + world_size, + cpu_group, + device_group, + graphs, + ) + + flush = None + if args.cold_l2: + l2_bytes = torch.cuda.get_device_properties(device).L2_cache_size + flush = torch.empty(2 * l2_bytes, dtype=torch.uint8, device=device) + times = benchmark_graphs( + graphs, + args.warmup_replays, + args.samples, + device_group, + device_barrier, + flush, + ) + graph_keepalive.clear() + + bus_bytes = message_bytes * (world_size - 1) / world_size + fastest = min(times, key=times.__getitem__) + return { + "message_bytes": message_bytes, + "shard_bytes": message_bytes // world_size, + "latency_us": times, + "logical_bus_gbps": { + name: bus_bytes / (latency_us * 1e3) for name, latency_us in times.items() + }, + "fastest": fastest, + } + + +def main() -> None: + args = parse_args() + if not args.message_bytes or min(args.message_bytes) <= 0: + raise ValueError("--message-bytes values must be positive") + if not args.blocks or min(args.blocks) <= 0 or max(args.blocks) > 8: + raise ValueError("--blocks values must be in [1, 8]") + if min(args.num_workspaces, args.samples) <= 0: + raise ValueError("Workspace and sample counts must be positive") + + local_rank = int(os.environ["LOCAL_RANK"]) + torch.accelerator.set_device_index(local_rank) + dist.init_process_group("nccl") + device_group = dist.group.WORLD + cpu_group = dist.new_group(backend="gloo") + rank = dist.get_rank(device_group) + world_size = dist.get_world_size(device_group) + device = torch.device("cuda", local_rank) + capability = torch.cuda.get_device_capability(device) + if world_size not in (2, 4, 8) or capability not in ((10, 0), (10, 3)): + raise RuntimeError( + "This benchmark targets TP2/TP4/TP8 SM100/SM103, " + f"got TP{world_size} SM{capability}" + ) + + max_message_bytes = max(args.message_bytes) + comm = CustomAllreduce( + group=cpu_group, + device=device, + max_mnnvl_reduce_scatter_size=max_message_bytes, + max_mnnvl_multimem_reduce_scatter_size=max_message_bytes, + ) + if comm.disabled: + raise RuntimeError("The production custom all-reduce backend is unavailable") + comm._init_mnnvl_multimem_reduce_scatter_buffer() + if not comm.mnnvl_multimem_rs_multicast_ptr: + raise RuntimeError("The production MNNVL multimem backend is unavailable") + + dtype = _DTYPES[args.dtype] + warmup = torch.zeros(1, dtype=torch.float32, device=device) + dist.all_reduce(warmup, group=device_group) + sync = torch.zeros_like(warmup) + + def device_barrier() -> None: + dist.all_reduce(sync, group=device_group) + + results = [ + benchmark_size( + args, + comm, + message_bytes, + dtype, + device, + rank, + world_size, + device_group, + cpu_group, + device_barrier, + ) + for message_bytes in args.message_bytes + ] + if rank == 0: + metadata = { + "world_size": world_size, + "gpu": torch.cuda.get_device_name(device), + "sm": f"{capability[0]}{capability[1]}", + "num_sms": torch.cuda.get_device_properties(device).multi_processor_count, + "dtype": args.dtype, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "blocks": args.blocks, + "num_workspaces": args.num_workspaces, + "cold_l2": args.cold_l2, + } + print(json.dumps({"metadata": metadata, "results": results}, indent=2)) + + comm.close() + dist.barrier(group=cpu_group) + dist.destroy_process_group(cpu_group) + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/kernels/benchmark_per_token_group_quant.py b/benchmarks/kernels/benchmark_per_token_group_quant.py index fda3d2a652..afc31104f7 100644 --- a/benchmarks/kernels/benchmark_per_token_group_quant.py +++ b/benchmarks/kernels/benchmark_per_token_group_quant.py @@ -117,7 +117,18 @@ if __name__ == "__main__": args = parse_args() warmup_iters, bench_iters = args.warmup_iters, args.bench_iters - shapes = [(32, 128), (64, 256), (16, 512)] + shapes = [ + (32, 128), + (64, 256), + (16, 512), + # Serving-sized activations, where this op is memory-bound and its + # schedule dominates: a prefill chunk and a decode batch at two + # common hidden sizes. + (1024, 6144), + (8192, 6144), + (1024, 7168), + (8192, 7168), + ] group_sizes = [64, 128] dtypes = ["fp8", "int8"] if args.dtype == "both" else [args.dtype] diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index cca2a98cee..414fbc5e23 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -173,6 +173,18 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA) set(AMX_FP8_SUPPORTED FALSE) message(STATUS "AMX-FP8 disabled by ENABLE_AMX_FP8=OFF") endif() + + # AVX10.2 arrived in GCC 15, so the target attribute, the fp8 convert + # intrinsics and __builtin_cpu_supports("avx10.2") are all hard errors on + # the 12.3+ the backend otherwise supports. Probe instead of assuming. + check_cxx_compiler_flag("-mavx10.2" COMPILER_SUPPORTS_AVX10_2_FLAG) + if (COMPILER_SUPPORTS_AVX10_2_FLAG) + set(AVX10_2_SUPPORTED TRUE) + else() + set(AVX10_2_SUPPORTED FALSE) + message(STATUS "AVX10.2 disabled: compiler does not support -mavx10.2 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") + endif() + list(APPEND CXX_COMPILE_FLAGS_AVX2 "-mavx2") elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) @@ -629,6 +641,12 @@ if (ENABLE_X86_ISA) message(STATUS "AMX-FP8 (Diamond Rapids) enabled") endif() + # For the sgl-kernels AVX10.2 fp8 quantize paths + if (AVX10_2_SUPPORTED) + target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX10_2") + message(STATUS "AVX10.2 enabled") + endif() + # AVX512F define_extension_target( _C_AVX512 diff --git a/cmake/external_projects/flashkda.cmake b/cmake/external_projects/flashkda.cmake index 5ac4f4b415..fadee58efb 100644 --- a/cmake/external_projects/flashkda.cmake +++ b/cmake/external_projects/flashkda.cmake @@ -13,7 +13,7 @@ else() FetchContent_Declare( flashkda GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git - GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02 + GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b GIT_PROGRESS TRUE GIT_SUBMODULES cutlass ) diff --git a/cmake/external_projects/qutlass.cmake b/cmake/external_projects/qutlass.cmake index cd3665c889..b6df33c170 100644 --- a/cmake/external_projects/qutlass.cmake +++ b/cmake/external_projects/qutlass.cmake @@ -142,9 +142,9 @@ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS) target_compile_definitions(_qutlass_C PRIVATE QUTLASS_MINIMAL_BUILD=1 + QUTLASS_DISABLE_PYBIND=1 TARGET_CUDA_ARCH=${QUTLASS_TARGET_CC} CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1 - TORCH_TARGET_VERSION=0x020B000000000000ULL USE_CUDA) set_property(SOURCE ${QUTLASS_SOURCES} APPEND PROPERTY COMPILE_OPTIONS diff --git a/cmake/external_projects/triton_kernels.cmake b/cmake/external_projects/triton_kernels.cmake index 8312c1f7ca..1906e6da5c 100644 --- a/cmake/external_projects/triton_kernels.cmake +++ b/cmake/external_projects/triton_kernels.cmake @@ -12,8 +12,8 @@ if (DEFINED ENV{TRITON_KERNELS_SRC_DIR}) else() if (VLLM_TARGET_DEVICE STREQUAL "rocm") set(TRITON_GIT "https://github.com/ROCm/triton.git") - # Pinned from release/internal/3.6.x - set(TRITON_KERNELS_TAG "0f380657dbf3ee86eb57558ff71df24f03b5d4e7") + # Pinned from release/internal/3.8.x + set(TRITON_KERNELS_TAG "669b31acc1dd1b3fd93286afd5db67f65d9f7557") else() set(TRITON_GIT "https://github.com/triton-lang/triton.git") set(TRITON_KERNELS_TAG "v3.5.1") diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp index ea0b4e49b2..13e430fb41 100644 --- a/csrc/cpu/cpu_attn.cpp +++ b/csrc/cpu/cpu_attn.cpp @@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache( input.block_table = block_table.data_ptr(); input.alibi_slopes = alibi_slopes.has_value() ? alibi_slopes->data_ptr() : nullptr; - input.s_aux = s_aux.has_value() ? s_aux->data_ptr() : nullptr; + // Attention sinks may be bf16 (native path) or fp32. Anything that is not + // bf16 must be provided as fp32 and is executed in full float precision. + if (s_aux.has_value()) { + TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 || + s_aux->scalar_type() == at::ScalarType::Float, + "cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ", + s_aux->scalar_type(), " must be bfloat16 or float32"); + input.s_aux = s_aux->data_ptr(); + input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16; + } else { + input.s_aux = nullptr; + input.s_aux_is_bf16 = false; + } input.dynamic_causal = dynamic_causal.has_value() ? dynamic_causal->data_ptr() : nullptr; input.scale = scale; diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index 4ade47eff3..d914e15063 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -879,7 +879,11 @@ struct AttentionInput { int32_t* seq_lens; int32_t* block_table; float* alibi_slopes; - c10::BFloat16* s_aux; + // Attention sinks pointer. May reference bf16 or fp32 data depending on + // `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is + // provided as fp32 and executed in full float precision. + const void* s_aux; + bool s_aux_is_bf16; bool* dynamic_causal; float scale; bool causal; @@ -1565,7 +1569,8 @@ class AttentionMainLoop { const float scale = input->scale; const float softcap_scale = input->softcap; const float* alibi_slopes = input->alibi_slopes; - const c10::BFloat16* s_aux = input->s_aux; + const void* s_aux = input->s_aux; + const bool s_aux_is_bf16 = input->s_aux_is_bf16; const bool* dynamic_causal = input->dynamic_causal; const bool is_dynamic_causal = dynamic_causal != nullptr; @@ -1773,9 +1778,6 @@ class AttentionMainLoop { const float* curr_alibi_slopes = (alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx : nullptr); - const c10::BFloat16* curr_s_aux = - (s_aux != nullptr ? s_aux + q_head_start_idx : nullptr); - // copy the Q tile to q_buffer, the logical layout of q_buffer is // [actual_q_token_num, curr_q_heads_per_kv, head_dim] { @@ -1787,10 +1789,23 @@ class AttentionMainLoop { if (use_sink) { alignas(64) float s_aux_fp32[16]; - // All other platforms have BF16Vec16 available - vec_op::BF16Vec16 vec_bf16(curr_s_aux); - vec_op::FP32Vec16 vec_fp32(vec_bf16); - vec_fp32.save(s_aux_fp32); + // Sinks may be provided as bf16 or fp32. Either way they are + // loaded into fp32: bf16 sinks are up-converted (native + // BF16Vec16 path, available on all platforms), while fp32 + // sinks are loaded directly and executed in full precision. + if (s_aux_is_bf16) { + const c10::BFloat16* curr_s_aux = + static_cast(s_aux) + + q_head_start_idx; + vec_op::BF16Vec16 vec_bf16(curr_s_aux); + vec_op::FP32Vec16 vec_fp32(vec_bf16); + vec_fp32.save(s_aux_fp32); + } else { + const float* curr_s_aux = + static_cast(s_aux) + q_head_start_idx; + vec_op::FP32Vec16 vec_fp32(curr_s_aux); + vec_fp32.save(s_aux_fp32); + } float* __restrict__ curr_sum_buffer = sum_buffer; float* __restrict__ curr_max_buffer = max_buffer; diff --git a/csrc/cpu/sgl-kernels/common.h b/csrc/cpu/sgl-kernels/common.h index dc0769450e..acf7591281 100644 --- a/csrc/cpu/sgl-kernels/common.h +++ b/csrc/cpu/sgl-kernels/common.h @@ -532,10 +532,17 @@ void zero_buffer(T* data, int64_t size) { #endif #if defined(__x86_64__) +// CPU_CAPABILITY_AVX10_2 is defined by cmake only when the compiler accepts +// -mavx10.2; without it the builtin below is not merely absent but a hard +// error, so the AVX512 paths have to stand in. bool avx10_2_available() { +#ifdef CPU_CAPABILITY_AVX10_2 // __builtin_cpu_supports returns the masked feature bit rather than 0/1. static const bool available = __builtin_cpu_supports("avx10.2"); return available; +#else + return false; +#endif } #endif diff --git a/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp index 854566d682..2014a29fa2 100644 --- a/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp +++ b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp @@ -772,6 +772,7 @@ inline __m128i cvtfp32_fp8e4m3(__m512& src) { return _mm512_cvtepi32_epi8(packed); } +#ifdef CPU_CAPABILITY_AVX10_2 __attribute__((target("avx10.2"))) inline __m128i cvtfp32_fp8e4m3_avx10_2(__m512& src) { __m256i f16_vec = _mm512_cvt_roundps_ph(src, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); return _mm256_cvtph_hf8(_mm256_castsi256_ph(f16_vec)); @@ -805,6 +806,7 @@ __attribute__((target("avx10.2"))) void quantize_bf16_block_avx10_2( _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), fp8_vec2); } } +#endif // CPU_CAPABILITY_AVX10_2 std::tuple _quantize_fp8e4m3_bf16_per_tensor_no_scale(const at::Tensor& t) { constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max @@ -861,10 +863,12 @@ std::tuple _quantize_fp8e4m3_bf16_per_tensor_no_scale(co scale_data[c] = scale_val; // Step 3: Scale and clamp using AVX512 (reuse the same loop structure) +#ifdef CPU_CAPABILITY_AVX10_2 if (avx10_2_available()) { quantize_bf16_block_avx10_2(channel_src, quant_dst, elements_per_channel, scale_reciprocal); continue; } +#endif i = 0; const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); const __m512 quant_max_vec = _mm512_set1_ps(quant_max); @@ -939,10 +943,12 @@ _quantize_fp8e4m3_bf16_per_tensor_with_scale(const at::Tensor& t, at::Tensor& sc const at::BFloat16* src_data = t_bf16.data_ptr() + c * elements_per_channel; at::Float8_e4m3fn* quant_t_data = quant_t.data_ptr() + c * elements_per_channel; +#ifdef CPU_CAPABILITY_AVX10_2 if (avx10_2_available()) { quantize_bf16_block_avx10_2(src_data, quant_t_data, elements_per_channel, scale_reciprocal); continue; } +#endif int64_t i = 0; // Process 32 elements at a time using AVX512 diff --git a/csrc/custom_all_gather_reduce_scatter.cuh b/csrc/custom_all_gather_reduce_scatter.cuh index 0190af728e..d70ad75d25 100644 --- a/csrc/custom_all_gather_reduce_scatter.cuh +++ b/csrc/custom_all_gather_reduce_scatter.cuh @@ -7,6 +7,10 @@ namespace vllm { constexpr int kMnnvlLamportAgThreads = 128; constexpr int kMnnvlLamportRsThreads = 256; constexpr int kMnnvlLamportConcurrentPollMaxPacks = 8192; +constexpr int kMnnvlMultimemRsThreads = 1024; +constexpr int kMnnvlMultimemRsBlockLimit = 8; +constexpr int kMnnvlMultimemRsVectorBytes = 16; +constexpr int kMnnvlMultimemRsUnroll = 8; using CopyPack = array_t; @@ -48,6 +52,127 @@ __global__ void __launch_bounds__(512, 1) barrier_at_end(sg, self_sg, rank); } +// Multimem reduce-scatter reduces through an NVLS multicast mapping, which +// has no AMD equivalent. The whole path is compiled out on ROCm and the host +// entry point rejects the call instead of launching anything. +#if !defined(USE_ROCM) + +template +DINLINE void multimem_load_reduce_16(uint32_t (&result)[4], const T* address) { + #if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + if constexpr (std::is_same::value) { + asm volatile( + "multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.bf16x2 " + "{%0,%1,%2,%3}, [%4];" + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]) + : "l"(address) + : "memory"); + } else if constexpr (std::is_same::value) { + asm volatile( + "multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.f16x2 " + "{%0,%1,%2,%3}, [%4];" + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]) + : "l"(address) + : "memory"); + } else { + static_assert(std::is_same::value); + asm volatile( + "multimem.ld_reduce.relaxed.sys.global.add.v4.f32 " + "{%0,%1,%2,%3}, [%4];" + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]) + : "l"(address) + : "memory"); + } + #else + asm volatile("trap;"); + #endif +} + +DINLINE void store_global_16(void* address, const uint32_t (&value)[4]) { + asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};" + : + : "l"(address), "r"(value[0]), "r"(value[1]), "r"(value[2]), + "r"(value[3]) + : "memory"); +} + +DINLINE void mnnvl_multimem_publish_flag(FlagType* flag_addr, FlagType flag) { + #if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + asm volatile("multimem.st.release.sys.global.u32 [%0], %1;" + : + : "l"(flag_addr), "r"(flag) + : "memory"); + // The flag is stored through the multicast alias and polled through the + // local unicast alias in the same grid. + asm volatile("fence.proxy.alias;" ::: "memory"); + #else + asm volatile("trap;"); + #endif +} + +template +__global__ void mnnvl_multimem_barrier_kernel(Signal* local_signal, + Signal* multicast_signal, + int rank) { + FlagType flag = local_signal->_flag[0] + 1; + FlagType* multicast_counters = + ready ? multicast_signal->start[0] : multicast_signal->end[0]; + FlagType* local_counters = + ready ? local_signal->start[0] : local_signal->end[0]; + // Replicate this rank's counter into every backing allocation. + mnnvl_multimem_publish_flag(&multicast_counters[rank], flag); + #pragma unroll + for (int peer = 0; peer < ngpus; ++peer) { + while (ld_flag_acquire(&local_counters[peer]) != flag); + } + local_signal->_flag[0] = flag; +} + +template +__global__ void __launch_bounds__(kMnnvlMultimemRsThreads, 1) + mnnvl_multimem_reduce_scatter_kernel(const T* __restrict__ multicast_input, + T* __restrict__ result, int rank, + int packs_per_rank) { + constexpr int kThreadsPerWarp = 32; + constexpr int kWarpsPerCta = kMnnvlMultimemRsThreads / kThreadsPerWarp; + constexpr int kPacksPerWarpIteration = + kThreadsPerWarp * kMnnvlMultimemRsUnroll; + + int lane; + asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane)); + int warp = blockIdx.x * kWarpsPerCta + threadIdx.x / kThreadsPerWarp; + int num_warps = gridDim.x * kWarpsPerCta; + int pack_offset = warp * kPacksPerWarpIteration + lane; + int pack_stride = num_warps * kPacksPerWarpIteration; + auto* rank_input = reinterpret_cast(multicast_input) + + rank * packs_per_rank * kMnnvlMultimemRsVectorBytes; + auto* rank_output = reinterpret_cast(result); + + while (pack_offset < packs_per_rank) { + uint32_t reduced[kMnnvlMultimemRsUnroll][4]; + #pragma unroll + for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) { + int pack = pack_offset + u * kThreadsPerWarp; + if (pack < packs_per_rank) { + multimem_load_reduce_16( + reduced[u], reinterpret_cast( + rank_input + pack * kMnnvlMultimemRsVectorBytes)); + } + } + #pragma unroll + for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) { + int pack = pack_offset + u * kThreadsPerWarp; + if (pack < packs_per_rank) { + store_global_16(rank_output + pack * kMnnvlMultimemRsVectorBytes, + reduced[u]); + } + } + pack_offset += pack_stride; + } +} + +#endif // !defined(USE_ROCM) + template union LamportPack { P packed; diff --git a/csrc/custom_all_reduce.cuh b/csrc/custom_all_reduce.cuh index 385a0e0ae7..4bfa3b71dc 100644 --- a/csrc/custom_all_reduce.cuh +++ b/csrc/custom_all_reduce.cuh @@ -339,6 +339,12 @@ class CustomAllreduce { void mnnvl_lamport_reduce_scatter(cudaStream_t stream, T* input, T* output, void* local_buffer, uint32_t* epochs, int size, int stage_size_bytes); + template + void mnnvl_multimem_reduce_scatter(cudaStream_t stream, + const T* multicast_input, T* output, + Signal* local_signal, + Signal* multicast_signal, int size, + int block_limit); ~CustomAllreduce() { for (auto [_, ptr] : ipc_handles_) { diff --git a/csrc/libtorch_stable/all_reduce_mhc.cu b/csrc/libtorch_stable/all_reduce_mhc.cu deleted file mode 100644 index dc58371156..0000000000 --- a/csrc/libtorch_stable/all_reduce_mhc.cu +++ /dev/null @@ -1,212 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -#include "torch_utils.h" -#include "custom_all_gather_reduce_scatter.cuh" -#include -#include - -namespace { -using Tensor = torch::stable::Tensor; -using Pack = vllm::packed_t::P; -constexpr int kHidden = 5120; -constexpr int kRanks = 4; -constexpr int kThreads = 128; -constexpr int kCluster = 5; -constexpr int kPacks = kHidden / Pack::size; - -__global__ __launch_bounds__(kThreads) - __cluster_dims__(1, kCluster, 1) void all_reduce_mhc_kernel( - const Pack* input, const Pack* residual, const float* post, - const float* comb, const float* pre, const Pack* weight, Pack* output, - Pack* normalized, Pack* local, Pack* multicast, uint32_t* epochs, - int rank, int stage_size, float eps) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 - cudaGridDependencySynchronize(); - const int token = blockIdx.x; - const int column = blockIdx.y * kThreads + threadIdx.x; - const int idx = token * kPacks + column; - const int size = gridDim.x * kPacks; - const int stage = epochs[0] % 3; - const int dirty = vllm::mnnvl_lamport_dirty_stage(stage); - const int dirty_size = epochs[2 + dirty]; - Pack value = input[idx]; - vllm::store_multimem_lamport_payload( - multicast + stage * stage_size + rank * size + idx, - vllm::sanitize_lamport_payload(value)); - // Multi-token forwards benefit from launching dependents before mHC finishes. - if (gridDim.x > 1) cudaTriggerProgrammaticLaunchCompletion(); - vllm::lamport_cta_arrive(epochs + 1); - for (int i = idx; i < dirty_size; i += size) { - local[dirty * stage_size + i] = vllm::lamport_sentinel(); - } - Pack peers[kRanks]; - bool pending; - do { - vllm::LamportPack packets[kRanks]; - #pragma unroll - for (int peer = 0; peer < kRanks; ++peer) { - auto ptr = local + stage * stage_size + idx + peer * size; - asm volatile("ld.relaxed.sys.global.v4.u32 {%0,%1,%2,%3}, [%4];" - : "=r"(packets[peer].words[0]), "=r"(packets[peer].words[1]), - "=r"(packets[peer].words[2]), "=r"(packets[peer].words[3]) - : "l"(ptr) - : "memory"); - } - pending = false; - #pragma unroll - for (int peer = 0; peer < kRanks; ++peer) { - pending |= vllm::is_lamport_dirty(packets[peer]); - peers[peer] = packets[peer].packed; - } - } while (pending); - auto sum = vllm::upcast(peers[0]); - #pragma unroll - for (int r = 1; r < kRanks; ++r) { - vllm::packed_assign_add(sum, vllm::upcast(peers[r])); - } - // Keep the BF16 boundary between the collective and mHC post. - auto reduced = vllm::upcast(vllm::downcast(sum)); - decltype(sum) streams[4]; - #pragma unroll - for (int r = 0; r < 4; ++r) { - streams[r] = vllm::upcast(residual[(token * 4 + r) * kPacks + column]); - } - decltype(sum) collapsed{}; - #pragma unroll - for (int c = 0; c < 4; ++c) { - decltype(sum) mixed; - #pragma unroll - for (int e = 0; e < Pack::size; ++e) { - // Match post*x followed by the four residual FMAs. - float acc = __fmul_rn(post[token * 4 + c], reduced.data[e]); - #pragma unroll - for (int r = 0; r < 4; ++r) { - acc = fmaf(comb[token * 16 + r * 4 + c], streams[r].data[e], acc); - } - mixed.data[e] = acc; - } - Pack rounded = vllm::downcast(mixed); - output[(token * 4 + c) * kPacks + column] = rounded; - auto as_float = vllm::upcast(rounded); - #pragma unroll - for (int e = 0; e < Pack::size; ++e) { - collapsed.data[e] = - fmaf(pre[token * 4 + c], as_float.data[e], collapsed.data[e]); - } - } - // RMSNorm consumes the rounded BF16 collapse in the unfused path. - collapsed = vllm::upcast(vllm::downcast(collapsed)); - float sqr = 0; - #pragma unroll - for (int e = 0; e < Pack::size; ++e) { - sqr = fmaf(collapsed.data[e], collapsed.data[e], sqr); - } - #pragma unroll - for (int delta = 16; delta; delta /= 2) { - sqr += __shfl_xor_sync(0xffffffff, sqr, delta); - } - __shared__ float warp_sums[kCluster][kThreads / 32]; - auto cluster = cooperative_groups::this_cluster(); - const int lane = threadIdx.x % 32; - if (lane < kCluster) { - *cluster.map_shared_rank(&warp_sums[blockIdx.y][threadIdx.x / 32], lane) = - sqr; - } - cluster.sync(); - float total = 0; - #pragma unroll - for (int c = 0; c < kCluster; ++c) { - #pragma unroll - for (int w = 0; w < kThreads / 32; ++w) total += warp_sums[c][w]; - } - const float scale = rsqrtf(__fadd_rn(total / kHidden, eps)); - auto weights = vllm::upcast(weight[column]); - #pragma unroll - for (int e = 0; e < Pack::size; ++e) { - collapsed.data[e] = collapsed.data[e] * scale * weights.data[e]; - } - normalized[idx] = vllm::downcast(collapsed); - if (idx == 0) { - while (*reinterpret_cast(epochs + 1) < - gridDim.x * gridDim.y) { - } - epochs[2 + stage] = kRanks * size; - epochs[0] = vllm::mnnvl_lamport_next_stage(stage); - epochs[1] = 0; - } - if (gridDim.x == 1) cudaTriggerProgrammaticLaunchCompletion(); -#else - asm volatile("trap;"); -#endif -} - -void all_reduce_mhc(Tensor input, Tensor residual, Tensor post, Tensor comb, - Tensor pre, Tensor weight, Tensor output, Tensor normalized, - int64_t local, int64_t multicast, Tensor epochs, - int64_t rank, int64_t stage_bytes, double eps) { - using torch::headeronly::ScalarType; - const torch::stable::accelerator::DeviceGuard guard(input.get_device_index()); - const int64_t n = input.size(0); - STD_TORCH_CHECK(input.dim() == 2 && input.size(1) == kHidden && n > 0 && - n <= 16); - for (const auto& t : {input, residual, weight, output, normalized}) { - STD_TORCH_CHECK(t.is_contiguous() && - t.scalar_type() == ScalarType::BFloat16); - STD_TORCH_CHECK(t.get_device_index() == input.get_device_index()); - STD_TORCH_CHECK(reinterpret_cast(t.const_data_ptr()) % 16 == 0); - } - for (const auto& t : {post, comb, pre}) { - STD_TORCH_CHECK(t.is_contiguous() && t.scalar_type() == ScalarType::Float); - STD_TORCH_CHECK(t.get_device_index() == input.get_device_index()); - } - STD_TORCH_CHECK(residual.numel() == n * 4 * kHidden && - output.numel() == residual.numel() && - normalized.numel() == input.numel() && - weight.numel() == kHidden); - STD_TORCH_CHECK(post.numel() == n * 4 && pre.numel() == n * 4 && - comb.numel() == n * 16); - STD_TORCH_CHECK(epochs.is_contiguous() && - epochs.scalar_type() == ScalarType::Int && - epochs.numel() >= 5 && - epochs.get_device_index() == input.get_device_index()); - STD_TORCH_CHECK(local && multicast && rank >= 0 && rank < kRanks && - stage_bytes % 16 == 0 && - stage_bytes >= n * kHidden * 2 * kRanks); - STD_TORCH_CHECK(get_device_prop()->major == 10); - cudaLaunchAttribute attr{}; - attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; - attr.val.programmaticStreamSerializationAllowed = 1; - cudaLaunchConfig_t config{ - .gridDim = dim3(n, kCluster), - .blockDim = dim3(kThreads), - .dynamicSmemBytes = 0, - .stream = get_current_cuda_stream(input.get_device_index()), - .attrs = &attr, - .numAttrs = 1}; - STD_CUDA_CHECK(cudaLaunchKernelEx( - &config, all_reduce_mhc_kernel, - static_cast(input.const_data_ptr()), - static_cast(residual.const_data_ptr()), - static_cast(post.const_data_ptr()), - static_cast(comb.const_data_ptr()), - static_cast(pre.const_data_ptr()), - static_cast(weight.const_data_ptr()), - static_cast(output.mutable_data_ptr()), - static_cast(normalized.mutable_data_ptr()), - reinterpret_cast(local), reinterpret_cast(multicast), - static_cast(epochs.mutable_data_ptr()), static_cast(rank), - static_cast(stage_bytes / sizeof(Pack)), static_cast(eps))); -} -} // namespace - -STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, mhc) { - mhc.def( - "all_reduce_mhc(Tensor input, Tensor residual, Tensor post, Tensor comb, " - "Tensor pre, Tensor weight, Tensor! output, Tensor! normalized, " - "int local, int multicast, Tensor! epochs, int rank, int stage_bytes, " - "float eps) -> ()"); -} -STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, mhc) { - mhc.impl("all_reduce_mhc", TORCH_BOX(&all_reduce_mhc)); -} diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 4b8f4c23a5..9130adf87d 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -549,16 +549,20 @@ __global__ void concat_and_cache_ds_mla_kernel( scalar_t* kv_cache_16bit = reinterpret_cast(&kv_cache[dst_idx_start]); - // The last warp handles the RoPE part + // Zero the reserved RoPE tail for NoPE rows. if (threadIdx.x >= 64) { // Each thread handles two elements of RoPE const int8_t pe_idx_start = (threadIdx.x - 64) * 2; - const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start; - // Vectorized load of two 16-bit values, performed as one 32-bit load - const int32_t vals = *reinterpret_cast(&k_pe[src_idx]); // RoPE values start after the packed 8-bit NoPE values and the // 32-bit scales const int64_t dst_idx = kv_lora_rank / 2 + 8 + pe_idx_start; + if (pe_dim == 0) { + *reinterpret_cast(&kv_cache_16bit[dst_idx]) = 0; + return; + } + const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start; + // Vectorized load of two 16-bit values, performed as one 32-bit load + const int32_t vals = *reinterpret_cast(&k_pe[src_idx]); // Vectorized store of two 16-bit values, performed as one 32-bit store *reinterpret_cast(&kv_cache_16bit[dst_idx]) = vals; return; @@ -600,7 +604,7 @@ __global__ void concat_and_cache_ds_mla_kernel( } // Now all threads in the block scale and write their elements - // NoPE data is packed in the first kv_lora_rank/2 bytes (first 256 bytes) + // NoPE data occupies the first kv_lora_rank bytes (512 bytes). const int64_t dst_idx_base = dst_idx_start + (threadIdx.x * 8); uint8_t result[8]; @@ -936,7 +940,8 @@ void concat_and_cache_mla( if (kv_cache_dtype == "fp8_ds_mla") { STD_TORCH_CHECK(kv_lora_rank == 512, "kv_lora_rank must be 512 for fp8_ds_mla"); - STD_TORCH_CHECK(pe_dim == 64, "pe_dim must be 64 for fp8_ds_mla"); + STD_TORCH_CHECK(pe_dim == 64 || pe_dim == 0, + "pe_dim must be 64 or 0 for fp8_ds_mla"); STD_TORCH_CHECK(kv_cache.size(2) == 656 / kv_cache.element_size(), "kv_cache.size(2) must be 656 bytes for fp8_ds_mla"); STD_TORCH_CHECK(kv_c.element_size() == 2, diff --git a/csrc/libtorch_stable/custom_all_gather_reduce_scatter.cu b/csrc/libtorch_stable/custom_all_gather_reduce_scatter.cu index 88aed49d18..2e360e90b5 100644 --- a/csrc/libtorch_stable/custom_all_gather_reduce_scatter.cu +++ b/csrc/libtorch_stable/custom_all_gather_reduce_scatter.cu @@ -191,6 +191,55 @@ void CustomAllreduce::mnnvl_lamport_reduce_scatter(cudaStream_t stream, #undef MNNVL_LAMPORT_RS_LAUNCH } +template +void CustomAllreduce::mnnvl_multimem_reduce_scatter( + cudaStream_t stream, const T* multicast_input, T* output, + Signal* local_signal, Signal* multicast_signal, int size, int block_limit) { +#if defined(USE_ROCM) + throw std::runtime_error( + "MNNVL multimem reduce-scatter requires NVLS multicast and is not " + "supported on ROCm"); +#else + if (size <= 0) + throw std::runtime_error( + "MNNVL multimem reduce-scatter requires a non-empty input"); + int size_bytes = size * sizeof(T); + if (size_bytes % (kMnnvlMultimemRsVectorBytes * world_size_) != 0) + throw std::runtime_error( + "MNNVL multimem reduce-scatter requires each output shard byte size " + "to be a multiple of 16"); + if (block_limit <= 0 || block_limit > kMnnvlMultimemRsBlockLimit) + throw std::runtime_error( + "MNNVL multimem reduce-scatter block limit is out of range"); + + int packs_per_rank = size_bytes / kMnnvlMultimemRsVectorBytes / world_size_; + int packs_per_block = kMnnvlMultimemRsThreads * kMnnvlMultimemRsUnroll; + int blocks = std::min( + block_limit, (packs_per_rank + packs_per_block - 1) / packs_per_block); + + #define MNNVL_MULTIMEM_RS_CASE(ngpus) \ + case ngpus: \ + mnnvl_multimem_barrier_kernel \ + <<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \ + mnnvl_multimem_reduce_scatter_kernel \ + <<>>( \ + multicast_input, output, rank_, packs_per_rank); \ + mnnvl_multimem_barrier_kernel \ + <<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \ + break; + + switch (world_size_) { + MNNVL_MULTIMEM_RS_CASE(2) + MNNVL_MULTIMEM_RS_CASE(4) + MNNVL_MULTIMEM_RS_CASE(8) + default: + throw std::runtime_error( + "MNNVL multimem reduce-scatter only supports num gpus in (2,4,8)"); + } + #undef MNNVL_MULTIMEM_RS_CASE +#endif +} + } // namespace vllm using fptr_t = int64_t; @@ -360,3 +409,62 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp, "bfloat16"); } } + +void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp, + torch::stable::Tensor& out, + fptr_t _local_buffer, + fptr_t _multicast_buffer, + int64_t stage_sz_bytes, + int64_t block_limit) { + auto fa = reinterpret_cast(_fa); + const torch::stable::accelerator::DeviceGuard device_guard( + inp.get_device_index()); + const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index()); + + STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type())); + STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel())); + STD_TORCH_CHECK(inp.numel() > 0); + STD_TORCH_CHECK(_is_weak_contiguous(out)); + STD_TORCH_CHECK(_is_weak_contiguous(inp)); + auto input_size = inp.numel() * inp.element_size(); + STD_TORCH_CHECK(input_size <= stage_sz_bytes); + auto local_buffer = reinterpret_cast(_local_buffer); + auto multicast_buffer = reinterpret_cast(_multicast_buffer); + STD_TORCH_CHECK(local_buffer != nullptr); + STD_TORCH_CHECK(multicast_buffer != nullptr); + auto* local_signal = reinterpret_cast( + static_cast(local_buffer) - sizeof(vllm::Signal)); + auto* multicast_signal = reinterpret_cast( + static_cast(multicast_buffer) - sizeof(vllm::Signal)); + STD_CUDA_CHECK(cudaMemcpyAsync(local_buffer, inp.const_data_ptr(), input_size, + cudaMemcpyDeviceToDevice, stream)); + switch (out.scalar_type()) { + case torch::headeronly::ScalarType::Float: { + fa->mnnvl_multimem_reduce_scatter( + stream, reinterpret_cast(multicast_buffer), + reinterpret_cast(out.mutable_data_ptr()), local_signal, + multicast_signal, inp.numel(), block_limit); + break; + } + case torch::headeronly::ScalarType::Half: { + fa->mnnvl_multimem_reduce_scatter( + stream, reinterpret_cast(multicast_buffer), + reinterpret_cast(out.mutable_data_ptr()), local_signal, + multicast_signal, inp.numel(), block_limit); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case torch::headeronly::ScalarType::BFloat16: { + fa->mnnvl_multimem_reduce_scatter( + stream, reinterpret_cast(multicast_buffer), + reinterpret_cast(out.mutable_data_ptr()), local_signal, + multicast_signal, inp.numel(), block_limit); + break; + } +#endif + default: + throw std::runtime_error( + "MNNVL multimem reduce-scatter only supports float32, float16 and " + "bfloat16"); + } +} diff --git a/csrc/libtorch_stable/custom_all_gather_reduce_scatter_ops.cpp b/csrc/libtorch_stable/custom_all_gather_reduce_scatter_ops.cpp index 198b78d56d..bf7d3f8abd 100644 --- a/csrc/libtorch_stable/custom_all_gather_reduce_scatter_ops.cpp +++ b/csrc/libtorch_stable/custom_all_gather_reduce_scatter_ops.cpp @@ -17,6 +17,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ag_rs) { custom_ag_rs.def( "mnnvl_lamport_reduce_scatter(int fa, Tensor inp, Tensor! out, int " "local_buffer, int epoch_buffer, int stage_sz_bytes) -> ()"); + custom_ag_rs.def( + "mnnvl_multimem_reduce_scatter(int fa, Tensor inp, Tensor! out, int " + "local_buffer, int multicast_buffer, int stage_sz_bytes, int " + "block_limit) -> ()"); } STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) { @@ -26,4 +30,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) { custom_ag_rs.impl("custom_reduce_scatter", TORCH_BOX(&custom_reduce_scatter)); custom_ag_rs.impl("mnnvl_lamport_reduce_scatter", TORCH_BOX(&mnnvl_lamport_reduce_scatter)); + custom_ag_rs.impl("mnnvl_multimem_reduce_scatter", + TORCH_BOX(&mnnvl_multimem_reduce_scatter)); } diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 6b17815a07..fba4220d73 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -175,9 +175,6 @@ torch::stable::Tensor awq_dequantize(torch::stable::Tensor _kernel, // DSV3 fused A GEMM: conditionally compiled so declaration and impl // registration are in the source file (dsv3_fused_a_gemm.cu) -// AllSpark ops: declarations are in the source files -// (allspark_repack.cu and allspark_qgemm_w8a16.cu) - #endif // CPU tensor -> CUDA UVA view (shared CUDA/ROCm) @@ -530,6 +527,10 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp, torch::stable::Tensor& out, fptr_t local_buffer, fptr_t epoch_buffer, int64_t stage_sz_bytes); +void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp, + torch::stable::Tensor& out, + fptr_t local_buffer, fptr_t multicast_buffer, + int64_t stage_sz_bytes, int64_t block_limit); void dispose(fptr_t _fa); int64_t meta_size(); void register_buffer(fptr_t _fa, const std::vector& fake_ipc_ptrs); diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_qgemm_w8a16.cu b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_qgemm_w8a16.cu deleted file mode 100644 index 96dc3ecfc8..0000000000 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_qgemm_w8a16.cu +++ /dev/null @@ -1,1024 +0,0 @@ -#include "allspark_utils.cuh" - -#include -#include -#include -#include - -#include - -#include "core/registration.h" -#include "libtorch_stable/torch_utils.h" - -torch::stable::Tensor as_g_workspace; - -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 - -torch::stable::Tensor allspark_w8a16_gemm( - torch::stable::Tensor const& a, torch::stable::Tensor const& b_qweight, - torch::stable::Tensor const& b_scales, - std::optional const& b_qzeros, int64_t n, - int64_t group_size, int64_t sm_count, int64_t sm_version, - int64_t CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) { - STD_TORCH_CHECK_NOT_IMPLEMENTED( - false, "allspark_w8a16_gemm(..) requires CUDA_ARCH >= 8.0"); - return torch::stable::empty({1, 1}); -} - -#else -namespace allspark { -/* - * GemmTile manage data movement from Global Memory to Shared Memory - * requiring N % 8 == 0, K % 16 == 0 by loading uint - * BN is obtained by padding the original N to a multiple of 32 - * weight B is rearranged as N32K16 order, - * i.e. a initial data block of size 32(n)x16(k) is reordered as n8k4n4k4, - * in order to put data loaded by the same thread of 32x16 data block together - * continuously (see - * https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type) - */ -template -struct GmemTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK { - // element num loaded by a LDG inst. - static constexpr int LDG_ELEMENT_CNT_A = 8; - static constexpr int LDG_ELEMENT_CNT_B = 16; - static constexpr int WARP_SIZE = 32; - static constexpr int M_SIZE_ONE_LOAD = (BLOCK * LDG_ELEMENT_CNT_A) / 32; - static constexpr int N_SIZE_ONE_LOAD = (BLOCK * LDG_ELEMENT_CNT_B) / 32; - - __device__ GmemTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK( - const SM8x_GEMM_W8A16_Splitk_Params& k_params, - const uint32_t& A_smem_addr, const uint32_t& BQ_smem_addr, - const uint32_t& A_stage_stride, const uint32_t& BQ_stage_stride) - : params(k_params), - A_smem_base_addr(A_smem_addr), - BQ_smem_base_addr(BQ_smem_addr), - A_smem_stage_stride(A_stage_stride), - BQ_smem_stage_stride(BQ_stage_stride) { - this_block_A_base_ptr = params.A_ptr + blockIdx.x * Mtile * params.K + - blockIdx.z * params.SplitK; - // here B is rearranged as N32K16 order, i.e. 4 continuous N-direction - // 8(N)x16(K) size data blocks are packed together - this_block_B_base_ptr = params.B_ptr + blockIdx.y * Ntile * params.K + - blockIdx.z * params.SplitK * 4; - - const auto lane_id = threadIdx.x % WARP_SIZE; - - // For matrix A, a block load/store Mtile(row) x 32(col) elements in - // multiple iters, 8x4 warp load/store 8(row) x 32(col) elements per iter - const auto Aldg_row_base_idx = threadIdx.x / 4; - Aldg_col_idx = (threadIdx.x % 4) * LDG_ELEMENT_CNT_A; - const int Aldg_base_offset = Aldg_row_base_idx * params.K + Aldg_col_idx; - - // For matrix B, a block load/store elements of (Ntile / 4) row x 128 col - // elements of N32K16 packing in multiple iters, 4x8 warp load/store 4(row) - // * 128(col) per iter - Bldg_col_idx = (threadIdx.x % 8) * LDG_ELEMENT_CNT_B; - const auto Bldg_row_base_idx = threadIdx.x / 8; - const int Bldg_base_offset = - Bldg_row_base_idx * params.K * 4 + Bldg_col_idx; - - this_block_A_base_ptr += Aldg_base_offset; - this_block_B_base_ptr += Bldg_base_offset; - - const int sts_a_base_offset = - (threadIdx.x / 4) * 32 + - ((lane_id % 4) ^ ((lane_id / 4) % 4) ^ ((lane_id / 4) / 4)) * - LDG_ELEMENT_CNT_A; - const int sts_bq_base_offset = - Bldg_row_base_idx * 32 * 4 + - ((threadIdx.x % 8) ^ (((threadIdx.x / 8) % 2) * 4)) * LDG_ELEMENT_CNT_B; - - A_smem_base_addr += sts_a_base_offset * sizeof(FType); - BQ_smem_base_addr += sts_bq_base_offset * sizeof(uint8_t); - - A_ldg_guard = 0; - B_ldg_guard = 0; - #pragma unroll - for (int i = 0; i < (Mtile + M_SIZE_ONE_LOAD - 1) / M_SIZE_ONE_LOAD; ++i) { - auto m_idx = blockIdx.x * Mtile + Aldg_row_base_idx + i * M_SIZE_ONE_LOAD; - if (m_idx < params.M) { - A_ldg_guard |= (1u << i); - } - } - - const int N_padded = (params.N + 31) / 32 * 32; - #pragma unroll - for (int i = 0; i < (Ntile + N_SIZE_ONE_LOAD - 1) / N_SIZE_ONE_LOAD; ++i) { - auto n_idx = blockIdx.y * Ntile + (Bldg_row_base_idx / 8) * 32 + - i * N_SIZE_ONE_LOAD; - if (n_idx < N_padded) { - B_ldg_guard |= (1u << i); - } - } - } - - __device__ void ldgsts_first_ktiles(const int& first_k_tile, - const int& k_tiles) { - // load first k_tile - // load A - const int A_src_size = Aldg_col_idx < first_k_tile ? 16 : 0; - #pragma unroll - for (int i = 0; i < (Mtile + M_SIZE_ONE_LOAD - 1) / M_SIZE_ONE_LOAD; ++i) { - cp_async<16>( - A_smem_base_addr + (i * M_SIZE_ONE_LOAD * 32) * sizeof(FType), - this_block_A_base_ptr + i * M_SIZE_ONE_LOAD * params.K, A_src_size, - (A_ldg_guard & (1u << i)) != 0); - } - - // load B - const int B_src_size = (Bldg_col_idx / 4) < first_k_tile ? 16 : 0; - #pragma unroll - for (int i = 0; i < (Ntile + N_SIZE_ONE_LOAD - 1) / N_SIZE_ONE_LOAD; ++i) { - cp_async<16>( - BQ_smem_base_addr + (i * N_SIZE_ONE_LOAD * 32) * sizeof(uint8_t), - this_block_B_base_ptr + i * N_SIZE_ONE_LOAD * params.K, B_src_size, - (B_ldg_guard & (1u << i)) != 0); - } - - cp_async_commit_group(); - this_block_A_base_ptr += first_k_tile; - this_block_B_base_ptr += (first_k_tile * 4); - - // load second to (N-stage - 1) k_tiles - for (int stage_idx = 1; stage_idx < NStage - 1; ++stage_idx) { - if (stage_idx < k_tiles) { - #pragma unroll - for (int i = 0; i < (Mtile + M_SIZE_ONE_LOAD - 1) / M_SIZE_ONE_LOAD; - ++i) { - cp_async<16>(A_smem_base_addr + stage_idx * A_smem_stage_stride + - (i * M_SIZE_ONE_LOAD * 32) * sizeof(FType), - this_block_A_base_ptr + i * M_SIZE_ONE_LOAD * params.K, - 16, (A_ldg_guard & (1u << i)) != 0); - } - - #pragma unroll - for (int i = 0; i < (Ntile + N_SIZE_ONE_LOAD - 1) / N_SIZE_ONE_LOAD; - ++i) { - cp_async<16>(BQ_smem_base_addr + stage_idx * BQ_smem_stage_stride + - (i * N_SIZE_ONE_LOAD * 32) * sizeof(uint8_t), - this_block_B_base_ptr + i * N_SIZE_ONE_LOAD * params.K, - 16, (B_ldg_guard & (1u << i)) != 0); - } - - this_block_A_base_ptr += 32; - this_block_B_base_ptr += (32 * 4); - } - cp_async_commit_group(); - } - } - - __device__ void ldgsts(const int& sts_stage_idx) { - const int a_stage_offset = sts_stage_idx * A_smem_stage_stride; - const int bq_stage_offset = sts_stage_idx * BQ_smem_stage_stride; - #pragma unroll - for (int i = 0; i < (Mtile + M_SIZE_ONE_LOAD - 1) / M_SIZE_ONE_LOAD; ++i) { - cp_async<16>(A_smem_base_addr + a_stage_offset + - (i * M_SIZE_ONE_LOAD * 32) * sizeof(FType), - this_block_A_base_ptr + i * M_SIZE_ONE_LOAD * params.K, 16, - (A_ldg_guard & (1u << i)) != 0); - } - - #pragma unroll - for (int i = 0; i < (Ntile + N_SIZE_ONE_LOAD - 1) / N_SIZE_ONE_LOAD; ++i) { - cp_async<16>(BQ_smem_base_addr + bq_stage_offset + - (i * N_SIZE_ONE_LOAD * 32) * sizeof(uint8_t), - this_block_B_base_ptr + i * N_SIZE_ONE_LOAD * params.K, 16, - (B_ldg_guard & (1u << i)) != 0); - } - - cp_async_commit_group(); - this_block_A_base_ptr += 32; - this_block_B_base_ptr += (32 * 4); - } - - const FType* this_block_A_base_ptr = nullptr; - const QType* this_block_B_base_ptr = nullptr; - - int Aldg_col_idx; - int Bldg_col_idx; - - uint32_t A_ldg_guard; - uint32_t B_ldg_guard; - - uint32_t A_smem_base_addr, BQ_smem_base_addr; - const uint32_t A_smem_stage_stride, BQ_smem_stage_stride; - - const SM8x_GEMM_W8A16_Splitk_Params& params; -}; - -/* - * requiring N % 8 == 0 - */ -template -struct ComputeTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK { - static constexpr int WARP_SIZE = 32; - static constexpr int WARP_CNT = BLOCK / WARP_SIZE; - static constexpr int WARP_NTILE = Ntile / WARP_CNT; - static constexpr int WARP_NITER = WARP_NTILE / 8; // hmma16816 - static_assert(WARP_NTILE == 32 or WARP_NTILE == 64, - "now only support WARP_NTILE = 32 or 64!"); - - __device__ ComputeTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK( - const SM8x_GEMM_W8A16_Splitk_Params& k_params, - const uint32_t& A_smem_addr, const uint32_t& BQ_smem_addr, - const uint32_t& A_stage_stride, const uint32_t& BQ_stage_stride) - : params(k_params), - A_smem_base_addr(A_smem_addr), - BQ_smem_base_addr(BQ_smem_addr), - A_smem_stage_stride(A_stage_stride), - BQ_smem_stage_stride(BQ_stage_stride) { - warp_id = threadIdx.x / WARP_SIZE; - lane_id = threadIdx.x % WARP_SIZE; - - load_a_base_offset[0] = - (lane_id % 16) * 32 + - ((lane_id / 16) ^ (lane_id % 4) ^ ((lane_id / 4) % 2)) * 8; - load_a_base_offset[1] = - (lane_id % 16) * 32 + - ((lane_id / 16 + 2) ^ (lane_id % 4) ^ ((lane_id / 4) % 2)) * 8; - - load_b_base_offset[0] = - (lane_id / 4 + warp_id * (WARP_NTILE / 4)) * 32 * 4 + - (lane_id % 4) * 16 + ((lane_id / 4) % 2) * 16 * 4; - load_b_base_offset[1] = - (lane_id / 4 + warp_id * (WARP_NTILE / 4)) * 32 * 4 + - (lane_id % 4) * 16 + (((lane_id / 4) % 2) ^ 1) * 16 * 4; - - sts_c_base_offset = warp_id * Mtile * WARP_NTILE + - (lane_id / 4) * WARP_NTILE + (lane_id % 4) * 2; - - if (EnableFuse) { - this_block_C_base_ptr = - params.C_ptr + blockIdx.x * Mtile * params.N + blockIdx.y * Ntile; - } else { - this_block_C_base_ptr = - params.C_split_ptr + blockIdx.z * params.M * params.N + - blockIdx.x * Mtile * params.N + blockIdx.y * Ntile; - } - int store_thds_in_row = WARP_NTILE / 8; - store_c_row_base_idx = lane_id / store_thds_in_row; - store_c_col_idx = warp_id * WARP_NTILE + (lane_id % store_thds_in_row) * 8; - store_c_base_offset = store_c_row_base_idx * params.N + store_c_col_idx; - - #pragma unroll - for (int i = 0; i < Mtile / 16; ++i) { - #pragma unroll - for (int j = 0; j < WARP_NITER; ++j) { - #pragma unroll - for (int k = 0; k < 4; ++k) { - C_frag[i][j][k] = 0.f; - } - } - } - params_n_idx = - blockIdx.y * Ntile + warp_id * WARP_NTILE + (lane_id / 4) * 4; - } - - __device__ void lds(const int& smem_stage_idx, const int& reg_buf_idx, - const int& k_phase_idx) { - uint32_t A_smem_addr = - A_smem_base_addr + A_smem_stage_stride * smem_stage_idx; - uint32_t B_smem_addr = - BQ_smem_base_addr + BQ_smem_stage_stride * smem_stage_idx; - - #pragma unroll - for (int i = 0; i < Mtile / 16; ++i) { - ldsm_4(A_frag[reg_buf_idx][i][0], A_frag[reg_buf_idx][i][1], - A_frag[reg_buf_idx][i][2], A_frag[reg_buf_idx][i][3], - A_smem_addr + (load_a_base_offset[k_phase_idx] + i * 16 * 32) * - sizeof(FType)); - } - #pragma unroll - for (int i = 0; i < WARP_NTILE / 32; ++i) { - lds128(BQ_frag[reg_buf_idx][4 * i + 0], BQ_frag[reg_buf_idx][4 * i + 1], - BQ_frag[reg_buf_idx][4 * i + 2], BQ_frag[reg_buf_idx][4 * i + 3], - B_smem_addr + (load_b_base_offset[k_phase_idx] + i * 32 * 32) * - sizeof(uint8_t)); - } - - // dequant B - #pragma unroll - for (int i = 0; i < WARP_NITER / 2; ++i) { - cvt_8bx4_to_16bx4_bias128(BQ_frag[reg_buf_idx][2 * i], - BF_frag[reg_buf_idx][2 * i]); - if (has_zp) { - BF_frag[reg_buf_idx][2 * i][0] = - __hsub2(BF_frag[reg_buf_idx][2 * i][0], num2num2(B_zero[i].x)); - BF_frag[reg_buf_idx][2 * i][1] = - __hsub2(BF_frag[reg_buf_idx][2 * i][1], num2num2(B_zero[i].x)); - } - - BF_frag[reg_buf_idx][2 * i][0] = - __hmul2(BF_frag[reg_buf_idx][2 * i][0], num2num2(B_scale[i].x)); - BF_frag[reg_buf_idx][2 * i][1] = - __hmul2(BF_frag[reg_buf_idx][2 * i][1], num2num2(B_scale[i].x)); - - cvt_8bx4_to_16bx4_bias128(BQ_frag[reg_buf_idx][2 * i + 1], - BF_frag[reg_buf_idx][2 * i + 1]); - if (has_zp) { - BF_frag[reg_buf_idx][2 * i + 1][0] = - __hsub2(BF_frag[reg_buf_idx][2 * i + 1][0], num2num2(B_zero[i].y)); - BF_frag[reg_buf_idx][2 * i + 1][1] = - __hsub2(BF_frag[reg_buf_idx][2 * i + 1][1], num2num2(B_zero[i].y)); - } - - BF_frag[reg_buf_idx][2 * i + 1][0] = - __hmul2(BF_frag[reg_buf_idx][2 * i + 1][0], num2num2(B_scale[i].y)); - BF_frag[reg_buf_idx][2 * i + 1][1] = - __hmul2(BF_frag[reg_buf_idx][2 * i + 1][1], num2num2(B_scale[i].y)); - } - } - - __device__ void ldg_params() { - const int N_padded = (params.N + 31) / 32 * 32; - // load B scale and zero_point - #pragma unroll - for (int i = 0; i < WARP_NTILE / 32; ++i) { - ldg64_ca(B_scale[2 * i + 0], B_scale[2 * i + 1], - params.B_scale_ptr + params_n_idx + i * 32, - (params_n_idx + i * 32) < N_padded); - if (has_zp) { - ldg64_ca(B_zero[2 * i + 0], B_zero[2 * i + 1], - params.B_zero_ptr + params_n_idx + i * 32, - (params_n_idx + i * 32) < N_padded); - } - } - } - - __device__ void mma(const int& reg_buf_idx) { - #pragma unroll - for (int m_idx = 0; m_idx < Mtile / 16; ++m_idx) { - #pragma unroll - for (int n_idx = 0; n_idx < WARP_NITER; ++n_idx) { - hmma16816_f32( - C_frag[m_idx][n_idx], A_frag[reg_buf_idx][m_idx], - reinterpret_cast(BF_frag[reg_buf_idx][n_idx])); - } - } - } - - __device__ void fused_splitk_reduce() { - // need splitk-reduce if enable splitk - if (gridDim.z > 1) { - auto blk_red_idx = blockIdx.x * gridDim.y + blockIdx.y; - // Wait for all previous blocks in the splitk direction to accumulate the - // results into C_tmp - if (threadIdx.x == 0) { - uint32_t* red_count_ptr = params.red_count_ptr + blk_red_idx; - uint32_t count; - do { - // make sure the ld.cg inside the do-wile loop - __threadfence_block(); - asm volatile("ld.global.cg.b32 %0, [%1];" - : "=r"(count) - : "l"(red_count_ptr)); - } while (count != blockIdx.z); - } - __syncthreads(); - - auto C_tmp_base_offset = blk_red_idx * Mtile * Ntile + threadIdx.x * 4; - if (blockIdx.z != 0) { - // expecting that temporary register here reuses the previous A&B frag - // register - float temp_frag[Mtile / 16][WARP_NITER][4]; - #pragma unroll - for (int m_idx = 0; m_idx < Mtile / 16; ++m_idx) { - #pragma unroll - for (int n_idx = 0; n_idx < WARP_NITER; ++n_idx) { - int offset = - C_tmp_base_offset + (m_idx * WARP_NITER + n_idx) * BLOCK * 4; - *reinterpret_cast(temp_frag[m_idx][n_idx]) = - *reinterpret_cast(params.C_tmp_ptr + offset); - } - } - #pragma unroll - for (int m_idx = 0; m_idx < Mtile / 16; ++m_idx) { - #pragma unroll - for (int n_idx = 0; n_idx < WARP_NITER; ++n_idx) { - #pragma unroll - for (int idx = 0; idx < 4; ++idx) { - C_frag[m_idx][n_idx][idx] += temp_frag[m_idx][n_idx][idx]; - } - } - } - } - - // first splitk - 1 blocks need to write partial results into C_tmp - if (blockIdx.z != gridDim.z - 1) { - #pragma unroll - for (int m_idx = 0; m_idx < Mtile / 16; ++m_idx) { - #pragma unroll - for (int n_idx = 0; n_idx < WARP_NITER; ++n_idx) { - int offset = - C_tmp_base_offset + (m_idx * WARP_NITER + n_idx) * BLOCK * 4; - asm volatile( - "{st.global.cg.v4.b32 [%0], {%1, %2, %3, %4};}\n" - : - : "l"(params.C_tmp_ptr + offset), "f"(C_frag[m_idx][n_idx][0]), - "f"(C_frag[m_idx][n_idx][1]), "f"(C_frag[m_idx][n_idx][2]), - "f"(C_frag[m_idx][n_idx][3])); - } - } - __threadfence(); - __syncthreads(); - if (threadIdx.x == 0) { - uint32_t* red_count_ptr = params.red_count_ptr + blk_red_idx; - atomicInc(red_count_ptr, gridDim.z); - } - } - } - } - - __device__ void stg(char* smem) { - if (EnableFuse) { - if (blockIdx.z != gridDim.z - 1) return; - } - uint32_t* C_sts_ptr = - reinterpret_cast(smem + sts_c_base_offset * sizeof(FType)); - // C_tile sts - #pragma unroll - for (int m_idx = 0; m_idx < Mtile / 16; ++m_idx) { - #pragma unroll - for (int n_idx = 0; n_idx < WARP_NITER; ++n_idx) { - #pragma unroll - for (int k_idx = 0; k_idx < 2; ++k_idx) { - FType low16 = MarlinScalarType2::float2num( - C_frag[m_idx][n_idx][k_idx * 2]); - FType high16 = MarlinScalarType2::float2num( - C_frag[m_idx][n_idx][k_idx * 2 + 1]); - uint32_t tmp = (reinterpret_cast(low16) & 0xffff) | - (reinterpret_cast(high16) << 16); - int sts_offset = - m_idx * 16 * (WARP_NTILE / 2) + - (((lane_id / (32 / WARP_NITER)) + n_idx) % WARP_NITER) * (8 / 2) + - k_idx * 8 * (WARP_NTILE / 2); - C_sts_ptr[sts_offset] = tmp; - } - } - } - - __syncthreads(); - - FType* C_base_ptr = this_block_C_base_ptr + store_c_base_offset; - // C_tile lds and stg - auto m_base_idx = store_c_row_base_idx + blockIdx.x * Mtile; - bool n_guard = (store_c_col_idx + blockIdx.y * Ntile) < params.N; - if (WARP_NTILE == 32) { - int lds_c_base_offset = warp_id * Mtile * WARP_NTILE + - (lane_id / 4) * WARP_NTILE + - ((lane_id % 4 + lane_id / 8) % 4) * 8; - uint4* C_lds_ptr = - reinterpret_cast(smem + lds_c_base_offset * sizeof(FType)); - #pragma unroll - for (int i = 0; i < (Mtile / 16) * (WARP_NITER / 2); ++i) { - uint4 stg_reg = C_lds_ptr[i * 8 * 4]; - stg128(stg_reg.x, stg_reg.y, stg_reg.z, stg_reg.w, - C_base_ptr + i * 8 * params.N, - (m_base_idx + i * 8) < params.M && n_guard); - } - } else if (WARP_NTILE == 64) { - int lds_c_base_offset = - warp_id * Mtile * WARP_NTILE + (lane_id / 8) * WARP_NTILE; - #pragma unroll - for (int i = 0; i < (Mtile / 16) * (WARP_NITER / 2); ++i) { - int lds_c_offset = lds_c_base_offset + i * 4 * WARP_NTILE + - ((lane_id % 8 + lane_id / 8 + (i % 2) * 4) % 8) * 8; - uint4 stg_reg = - *reinterpret_cast(smem + lds_c_offset * sizeof(FType)); - stg128(stg_reg.x, stg_reg.y, stg_reg.z, stg_reg.w, - C_base_ptr + i * 4 * params.N, - (m_base_idx + i * 4) < params.M && n_guard); - } - } - } - - const SM8x_GEMM_W8A16_Splitk_Params& params; - - int load_a_base_offset[2]; - int load_b_base_offset[2]; - int sts_c_base_offset; - - int store_c_base_offset; - - int store_c_row_base_idx, store_c_col_idx; - FType* this_block_C_base_ptr = nullptr; - - int params_n_idx; - const uint32_t A_smem_base_addr, BQ_smem_base_addr; - const uint32_t A_smem_stage_stride, BQ_smem_stage_stride; - - int lane_id; - int warp_id; - // first 2 denotes double buffer, second dim denotes M direction - uint32_t A_frag[2][Mtile / 16][4]; - - typename HalfType::T2 B_scale[WARP_NITER / 2]; - typename HalfType::T2 B_zero[WARP_NITER / 2]; - uint32_t BQ_frag[2][WARP_NITER]; - // first 2 denotes double buffer, second dim denotes N direction, last 2 - // denotes K direction - typename HalfType::T2 BF_frag[2][WARP_NITER][2]; - // first dim denotes M direction, second dim denotes N direction - float C_frag[Mtile / 16][WARP_NITER][4]; -}; - -/* - * @brief W8A16 Perchannel Quantization GEMM, - * requires N % 8 == 0, K % 16 == 0 - * accumulator precision: FP32 - * @tparam FType: DataType for A, B_scale, B_zero, and C, supports half or - * nv_bfloat16 - * @tparam QType: DataType for B, support uint8(bias128) - * @tparam Mtile: M-dimensional size of the gemm block tile, supports 16, 32, - * 48 or 64 - * @tparam Ntile: N-dimensional size of the gemm block tile, supports 128 or - * 256 - * @tparam NStage: Num of stages for async copy - * @tparam BLOCK: BLOCK size - * @tparam EnableFuse: If true, use fused splitk-reduce, otherwise use - * non-fused splitk-reduce - * @tparam has_zp: whether to use zero_point - * - * @fparam params struct consists of following parameters: - * @param A_ptr: Matrix A value ptr, A = (M, K) - * @param B_ptr: Matrix B value ptr, B = (N32_align, K) (N32K16 special - * format), N32_align = (N + 32 - 1) / 32 * 32 - * @param B_scale_ptr: B_scale value ptr, B_scale = (N32_align,) (N32K16 - * special format) - * @param B_zero_ptr: B_zero value ptr, B_zero = (N32_align,) (N32K16 - * special format) - * @param C_ptr: Matrix C value ptr, C = (M, N) - * @param M: dimnesion m - * @param N: dimnesion n - * @param K: dimnesion k - * @param SplitK: split size along K-dimension - * @param C_split_ptr: Matrix C_split value ptr, used only in non-fused - * splitk-reduce - * @param C_tmp_ptr: Matrix C_tmp value ptr, used only in fused - * splitk-reduce - * @param red_count_ptr: 1-D red_count value ptr, used only in fused - * splitk-reduce - */ -template -__global__ void __launch_bounds__(BLOCK) - ampere_hgemm_W8A16_perc_f16_f16_MtilexNtilex32_hmma16816_multistage_AN_BTN32K16_CN_splitk_kernel( - const SM8x_GEMM_W8A16_Splitk_Params params) { - // A smem size = 64 * 32 * 2B/elem * 4(stage) = 16KB - // B smem size = 128 * 32 * 1B/elem * 4(stage) = 16KB - constexpr int smem_size_one_stage = Mtile * 32 * 2 + Ntile * 32; - __shared__ char smem[NStage * smem_size_one_stage]; - char* A_smem = smem; - char* BQ_smem = smem + Mtile * 32 * 2 * NStage; - - uint32_t A_smem_addr = smem_u32addr(A_smem); - uint32_t BQ_smem_addr = smem_u32addr(BQ_smem); - uint32_t A_smem_stage_stride = Mtile * 32 * 2; - uint32_t BQ_smem_stage_stride = Ntile * 32; - - // initialize the data move process from GM to SMEM for this block - GmemTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK< - FType, QType, Mtile, Ntile, NStage, BLOCK> - gmem_tile(params, A_smem_addr, BQ_smem_addr, A_smem_stage_stride, - BQ_smem_stage_stride); - - int sts_stage_idx = 0; - int lds_stage_idx = 0; - - auto tb_k_slice = blockIdx.z * params.SplitK + params.SplitK <= params.K - ? params.SplitK - : params.K - blockIdx.z * params.SplitK; - int k_tiles = (tb_k_slice + 31) / 32; - int first_k_tile = tb_k_slice - (k_tiles - 1) * 32; - - // load first three tiles to shared memory - gmem_tile.ldgsts_first_ktiles(first_k_tile, k_tiles); - sts_stage_idx += (NStage - 2); - ComputeTile_W8A16_PerC_MtilexNtilex32_multistage_SM8x_SplitK< - FType, QType, Mtile, Ntile, BLOCK, EnableFuse, has_zp> - compute_tile(params, A_smem_addr, BQ_smem_addr, A_smem_stage_stride, - BQ_smem_stage_stride); - compute_tile.ldg_params(); - cp_asyc_wait_group(); - __syncthreads(); - - compute_tile.lds(lds_stage_idx, 0, 0); - int reg_buf_idx = 1; - - // main loop - for (; k_tiles > NStage - 1; --k_tiles) { - // load next A&B tile - sts_stage_idx = sts_stage_idx < NStage - 1 ? sts_stage_idx + 1 : 0; - gmem_tile.ldgsts(sts_stage_idx); - - #pragma unroll - for (int k_phase_idx = 0; k_phase_idx < 2; k_phase_idx++) { - // dequantize next B tile - if (k_phase_idx == 1) { - cp_asyc_wait_group(); - __syncthreads(); - lds_stage_idx = lds_stage_idx < NStage - 1 ? lds_stage_idx + 1 : 0; - } - - compute_tile.lds(lds_stage_idx, reg_buf_idx, (k_phase_idx + 1) % 2); - - compute_tile.mma(reg_buf_idx ^ 1); - reg_buf_idx ^= 1; - } - } - - // last NStage-1 tiles - for (; k_tiles > 0; --k_tiles) { - cp_async_commit_group(); - #pragma unroll - for (int k_phase_idx = 0; k_phase_idx < 2; k_phase_idx++) { - // dequantize next B tile - if (k_phase_idx == 1) { - cp_asyc_wait_group(); - __syncthreads(); - lds_stage_idx = lds_stage_idx < NStage - 1 ? lds_stage_idx + 1 : 0; - } - - compute_tile.lds(lds_stage_idx, reg_buf_idx, (k_phase_idx + 1) % 2); - - compute_tile.mma(reg_buf_idx ^ 1); - reg_buf_idx ^= 1; - } - } - - if (EnableFuse) { - compute_tile.fused_splitk_reduce(); - } - compute_tile.stg(smem); -} - - #define __CALL_IF(MTILE, NTILE, NUM_THREADS, ENABLE_FUSE, HAS_ZP) \ - else if (Mtile == MTILE && Ntile == NTILE && BLOCK == NUM_THREADS && \ - enable_fuse == ENABLE_FUSE && has_zp == HAS_ZP) { \ - ampere_hgemm_W8A16_perc_f16_f16_MtilexNtilex32_hmma16816_multistage_AN_BTN32K16_CN_splitk_kernel< \ - FType, QType, MTILE, NTILE, 4, NUM_THREADS, ENABLE_FUSE, HAS_ZP> \ - <<>>(params); \ - } - -template -void ampere_hgemm_W8A16_perc_f16_f16_MtilexNtilex32_mma16816_multistage_AN_BTN32K16_CN_splitk( - const FType* A, const QType* B, const FType* B_scale, const FType* B_zero, - FType* C, const int M, const int N, const int K, void* workspace, - const int sm_version, const BlockTileSplitkParams& fused_gemm_params, - cudaStream_t stream) { - int Mtile = fused_gemm_params.Mtile; - int grid_x = (M + Mtile - 1) / Mtile; - int Ntile = fused_gemm_params.Ntile; - int grid_y = (N + Ntile - 1) / Ntile; - int SplitK = fused_gemm_params.SplitK; - int grid_z = (K + SplitK - 1) / SplitK; - - int BLOCK = (Ntile == 256) ? 256 : 128; - - dim3 grid(grid_x, grid_y, grid_z); - dim3 block(BLOCK); - - bool enable_fuse = fused_gemm_params.EnableFuse; - bool has_zp = B_zero != nullptr; - if (enable_fuse) { - float* C_tmp = reinterpret_cast(workspace); - uint32_t* red_count = reinterpret_cast( - (char*)workspace + grid_x * Mtile * grid_y * Ntile * sizeof(float)); - CHECK_CUDA(cudaMemsetAsync(red_count, 0, grid_x * grid_y * sizeof(uint32_t), - stream)); - SM8x_GEMM_W8A16_Splitk_Params params{ - A, B, B_scale, B_zero, C, M, N, - K, SplitK, 0, -1, nullptr, C_tmp, red_count}; - - if (false) { - } - // Select the template parameters for kernel launch - // according to the above settings. Tuning is not supported. - __CALL_IF(16, 256, 256, true, false) - __CALL_IF(32, 256, 256, true, false) - __CALL_IF(48, 256, 256, true, false) - __CALL_IF(64, 128, 128, true, false) - __CALL_IF(64, 256, 256, true, false) - __CALL_IF(16, 256, 256, true, true) - __CALL_IF(32, 256, 256, true, true) - __CALL_IF(48, 256, 256, true, true) - __CALL_IF(64, 128, 128, true, true) - __CALL_IF(64, 256, 256, true, true) - } else { - FType* C_split = reinterpret_cast(workspace); - SM8x_GEMM_W8A16_Splitk_Params params{ - A, B, B_scale, B_zero, C, M, N, - K, SplitK, 0, -1, C_split, nullptr, nullptr}; - - if (false) { - } - // Select the template parameters for kernel launch - // according to the above settings. Tuning is not supported. - __CALL_IF(16, 256, 256, false, false) - __CALL_IF(32, 256, 256, false, false) - __CALL_IF(48, 256, 256, false, false) - __CALL_IF(64, 128, 128, false, false) - __CALL_IF(64, 256, 256, false, false) - __CALL_IF(16, 256, 256, false, true) - __CALL_IF(32, 256, 256, false, true) - __CALL_IF(48, 256, 256, false, true) - __CALL_IF(64, 128, 128, false, true) - __CALL_IF(64, 256, 256, false, true) - - // SplitK reduce - f16_gemm_splitk_reduce(C_split, C, M, N, grid_z, stream); - } -} - -size_t allspark_qgemm_w8a16_perc_n32k16_ampere_workspace_size( - int m, int n, int k, int sm_count, - BlockTileSplitkParams& fused_gemm_params) { - // Determine the block tile and splitk strategy - int m16_times = (m + 16 - 1) / 16; - int Mtile = m16_times <= 4 ? m16_times * 16 : 64; - int grid_x = (m + Mtile - 1) / Mtile; - int Ntile = - (float(grid_x * ((n + 127) / 128)) / sm_count > 10) || (Mtile < 64) ? 256 - : 128; - int grid_y = (n + Ntile - 1) / Ntile; - int grid_z; - - // split-k - const float SPLIT_THRESHOLD = 0.8; - int n_slice; - for (n_slice = 1; n_slice < k / 256; ++n_slice) { - int n_block = grid_x * grid_y * n_slice; - if (n_block >= sm_count * SPLIT_THRESHOLD && - (n_block % sm_count == 0 || n_block % sm_count >= sm_count * 0.5)) { - break; - } - } - - int k_slice = - (k / n_slice) % 32 == 0 ? k / n_slice : k / n_slice / 32 * 32 + 32; - grid_z = (k + k_slice - 1) / k_slice; - bool enable_fuse = float(grid_x * grid_y) / sm_count >= 0.5 ? 1 : 0; - - size_t ws_size; - if (enable_fuse) { - ws_size = grid_x * Mtile * grid_y * Ntile * sizeof(float) // For C_tmp - + grid_x * grid_y * sizeof(uint32_t); // For red_count - } else { - ws_size = grid_z * m * n * sizeof(__half); - } - - fused_gemm_params.Mtile = Mtile; - fused_gemm_params.Ntile = Ntile; - fused_gemm_params.SplitK = k_slice; - fused_gemm_params.EnableFuse = enable_fuse; - return ws_size; -} - -// restore from N32K16 order to original N-major order -// K % 16 == 0, N % 8 == 0 -// each block process 64(k) * 32(n) result elements -template -__global__ void restore_N32_K16_dequantize_rhs_w8a16_perc_kernel( - const QT* qdata, const FT* scales, const FT* zeros, FT* fdata, - const int N_32align, const int N, const int K) { - __shared__ FT smem[64 * 32]; - auto warp_id = threadIdx.x / 32; - auto lane_id = threadIdx.x % 32; - const auto src_row_idx = blockIdx.x * 8 + lane_id / 4; - const int src_col_idx = - blockIdx.y * 64 * 4 + warp_id * 16 * 4 + (lane_id % 4) * 16; - const int src_offset = src_row_idx * K * 4 + src_col_idx; - auto params_nidx = blockIdx.x * 32 + (lane_id / 4) * 4; - - QT qval_reg[16]; - const QT* pdata = qdata + src_offset; - if (src_col_idx < (K * 4)) { - *(reinterpret_cast(qval_reg)) = - *(reinterpret_cast(qdata + src_offset)); - } - FT scale_reg[4]; - *(reinterpret_cast(scale_reg)) = - *(reinterpret_cast(scales + params_nidx)); - FT zero_reg[4]; - if (zeros != nullptr) { - *(reinterpret_cast(zero_reg)) = - *(reinterpret_cast(zeros + params_nidx)); - } - FT fval_reg[16]; - - const int sts_base_offset = - (warp_id * 16 + (lane_id % 4) * 2) * 32 + lane_id / 4; - #pragma unroll - for (int ni = 0; ni < 4; ++ni) { - cvt_8bx4_to_16bx4_bias128( - *reinterpret_cast(&qval_reg[ni * 4]), - reinterpret_cast::T2*>(&(fval_reg[ni * 4]))); - #pragma unroll - for (int ki = 0; ki < 4; ++ki) { - if (zeros != nullptr) { - fval_reg[ni * 4 + ki] = __hsub(fval_reg[ni * 4 + ki], zero_reg[ni]); - } - fval_reg[ni * 4 + ki] = __hmul(fval_reg[ni * 4 + ki], scale_reg[ni]); - int sts_offset = sts_base_offset + ((ki / 2) * 8 + (ki % 2)) * 32 + - ((ni + lane_id % 4) % 4) * 8; - smem[sts_offset] = fval_reg[ni * 4 + ki]; - } - } - __syncthreads(); - - const int lds_base_offset = - (threadIdx.x / 4) * 32 + ((threadIdx.x % 4 + threadIdx.x / 8) % 4) * 8; - #pragma unroll - for (int i = 0; i < 2; ++i) { - *reinterpret_cast(fval_reg + i * 8) = - *reinterpret_cast(smem + lds_base_offset + i * 32 * 32); - } - - const auto dst_row_base_kidx = blockIdx.y * 64 + threadIdx.x / 4; - const auto dst_col_nidx = blockIdx.x * 32 + (threadIdx.x % 4) * 8; - #pragma unroll - for (int i = 0; i < 2; ++i) { - int dst_row_kidx = dst_row_base_kidx + i * 32; - int dst_offset = dst_row_kidx * N + dst_col_nidx; - if (dst_row_kidx < K && dst_col_nidx < N) { - *reinterpret_cast(fdata + dst_offset) = - *reinterpret_cast(fval_reg + i * 8); - } - } -} - -template -void restore_N32_K16_dequantize_rhs_w8a16(const QT* qdata, const FT* scales, - const FT* zeros, FT* fdata, - const int N_32align, const int N, - const int K, const int GroupSize, - cudaStream_t stream) { - STD_TORCH_CHECK(N % 8 == 0 && K % 16 == 0 && N_32align % 32 == 0, - "Unsupported shape"); - if (GroupSize == -1) { - const int BLOCK = 128; - dim3 grid(N_32align / 32, ((K / 16) + 3) / 4); - restore_N32_K16_dequantize_rhs_w8a16_perc_kernel - <<>>(qdata, scales, zeros, fdata, N_32align, N, - K); - } - // TODO: Support SubChannel - else { - STD_TORCH_CHECK(false, "Now only support PerChannel"); - } -} - -template -void w8a16_gemm_dq_cublas(const FT* in, const QT* rhs_qdata_ptr, - const FT* rhs_scales_ptr, const FT* rhs_zeros_ptr, - FT* out, void* workspace, const int M, - const int N_32align, const int N, const int K, - const int group_size, cudaStream_t stream, - cublasHandle_t handle) { - static_assert( - std::is_same::value || std::is_same::value, - "only float16 and bfloat16 is supported"); - // Dequant - FT* rhs_fdata_ptr = static_cast(workspace); - restore_N32_K16_dequantize_rhs_w8a16(rhs_qdata_ptr, rhs_scales_ptr, - rhs_zeros_ptr, rhs_fdata_ptr, N_32align, - N, K, group_size, stream); - // cuBLAS GEMM - int lda = K; - int ldb = N; - int ldc = N; - const float alpha = 1.0f; - const float beta = 0.0f; - cudaDataType_t cuda_type; - if (std::is_same::value) { - cuda_type = CUDA_R_16F; - } else { - cuda_type = CUDA_R_16BF; - } - CHECK_CUBLAS(cublasGemmEx(handle, CUBLAS_OP_N, CUBLAS_OP_N, N, M, K, &alpha, - rhs_fdata_ptr, cuda_type, ldb, in, cuda_type, lda, - &beta, out, cuda_type, ldc, CUDA_R_32F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); -} - -template -void allspark_qgemm_w8a16_perc_ampere( - const FType* A, const QType* B, const FType* B_scale, const FType* B_zero, - FType* C, const int M, const int N_32align, const int N, const int K, - void* workspace, const BlockTileSplitkParams& fused_gemm_params, - const int group_size, int CUBLAS_M_THRESHOLD, const int sm_version, - cudaStream_t stream, cublasHandle_t handle) { - if (M > CUBLAS_M_THRESHOLD) { - w8a16_gemm_dq_cublas(A, B, B_scale, B_zero, C, workspace, M, - N_32align, N, K, group_size, stream, - handle); - } else { - ampere_hgemm_W8A16_perc_f16_f16_MtilexNtilex32_mma16816_multistage_AN_BTN32K16_CN_splitk< - FType, QType>(A, B, B_scale, B_zero, C, M, N, K, workspace, sm_version, - fused_gemm_params, stream); - } -} - -} // namespace allspark - -torch::stable::Tensor allspark_w8a16_gemm( - torch::stable::Tensor const& a, torch::stable::Tensor const& b_qweight, - torch::stable::Tensor const& b_scales, - std::optional const& b_qzeros, int64_t n, - int64_t group_size, int64_t sm_count, int64_t sm_version, - int64_t CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) { - // Verify device and strides - STD_TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); - STD_TORCH_CHECK(a.is_contiguous(), "A is not contiguous"); - - STD_TORCH_CHECK(b_qweight.device().is_cuda(), "b_qweight is not on GPU"); - STD_TORCH_CHECK(b_qweight.is_contiguous(), "b_qweight is not contiguous"); - - STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); - STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); - - if (has_zp) { - STD_TORCH_CHECK(b_qzeros.value().device().is_cuda(), - "b_qzeros is not on GPU"); - STD_TORCH_CHECK(b_qzeros.value().is_contiguous(), - "b_qzeros is not contiguous"); - } - - int m = a.size(0); - int n_32align = (n + 32 - 1) / 32 * 32; - int k = a.size(1); - - // Verify shape - STD_TORCH_CHECK(b_qweight.size(0) == n_32align, - "Shape mismatch: b_qweight.size(0) = ", b_qweight.size(0), - ", n_32align = ", n_32align); - STD_TORCH_CHECK(b_qweight.size(1) == k, - "Shape mismatch: b_qweight.size(1) = ", b_qweight.size(1), - ", k = ", k); - - STD_TORCH_CHECK(group_size == -1, "Currently only supports group_size = -1"); - - const torch::stable::accelerator::DeviceGuard device_guard( - a.get_device_index()); - const void* a_ptr = reinterpret_cast(a.data_ptr()); - const uint8_t* b_ptr = reinterpret_cast(b_qweight.data_ptr()); - const void* b_scale_ptr = reinterpret_cast(b_scales.data_ptr()); - const void* b_zero_ptr = nullptr; - if (b_qzeros.has_value()) { - b_zero_ptr = reinterpret_cast(b_qzeros.value().data_ptr()); - } - - auto c = - torch::stable::empty({m, n}, a.scalar_type(), std::nullopt, a.device()); - void* c_ptr = reinterpret_cast(c.mutable_data_ptr()); - - cudaStream_t stream = get_current_cuda_stream(); - cublasHandle_t handle = get_current_cuda_blas_handle(); - - allspark::BlockTileSplitkParams fused_gemm_params; - - size_t ws_size = 0; - if (m > CUBLAS_M_THRESHOLD) { - ws_size = k * n * 2; // sizeof(f16)==2 - } else { - ws_size = allspark::allspark_qgemm_w8a16_perc_n32k16_ampere_workspace_size( - m, n, k, sm_count, fused_gemm_params); - } - - if (as_g_workspace.numel() < - ws_size) { // ws_options: kChar, so numel() is bytes - as_g_workspace = torch::stable::empty({static_cast(ws_size)}, - torch::headeronly::ScalarType::Char, - std::nullopt, a.device()); - } - void* ws = reinterpret_cast(as_g_workspace.data_ptr()); - - if (a.scalar_type() == torch::headeronly::ScalarType::Half) { - allspark::allspark_qgemm_w8a16_perc_ampere<__half, uint8_t>( - reinterpret_cast(a_ptr), b_ptr, - reinterpret_cast(b_scale_ptr), - reinterpret_cast(b_zero_ptr), - reinterpret_cast<__half*>(c_ptr), m, n_32align, n, k, ws, - fused_gemm_params, group_size, CUBLAS_M_THRESHOLD, sm_version, stream, - handle); - } else if (a.scalar_type() == torch::headeronly::ScalarType::BFloat16) { - allspark::allspark_qgemm_w8a16_perc_ampere<__nv_bfloat16, uint8_t>( - reinterpret_cast(a_ptr), b_ptr, - reinterpret_cast(b_scale_ptr), - reinterpret_cast(b_zero_ptr), - reinterpret_cast<__nv_bfloat16*>(c_ptr), m, n_32align, n, k, ws, - fused_gemm_params, group_size, CUBLAS_M_THRESHOLD, sm_version, stream, - handle); - } - - return c; -} - -#endif - -STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { - m.impl("allspark_w8a16_gemm", TORCH_BOX(&allspark_w8a16_gemm)); -} diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu deleted file mode 100644 index 9a00f66241..0000000000 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu +++ /dev/null @@ -1,176 +0,0 @@ -#include "allspark_utils.cuh" - -#include -#include -#include - -#include "core/registration.h" -#include "libtorch_stable/torch_utils.h" - -namespace allspark { - -// Rearrange B to facilitate Ampere Tensor Core load data -// reorder B from (K, N) to (N_32align / 4, K * 4) -// K % 16 == 0, N % 16 == 0, N_32align % 32 == 0 -template -__global__ void __launch_bounds__(128) - rearrange_kn_weight_as_n32k16_order_ldg16_kernel( - const uint8_t* B, const FType* B_scale, const FType* B_zero, - uint8_t* B_result, FType* B_scale_result, FType* B_zero_result, - const int K, const int N, const int N_32align) { - const auto lane_id = threadIdx.x % 32; - const auto warp_id = threadIdx.x / 32; - - if (blockIdx.x != gridDim.x - 1) { - // Load B - // per block process 64(k) * 128(n) B elements - // per warp process 16(k) * 128 B elements - const int src_row_base_idx = - blockIdx.x * 64 + warp_id * 16 + ((lane_id % 8) / 2) * 2; - const int src_col_idx = - blockIdx.y * 128 + (lane_id / 8) * 32 + (lane_id % 2) * 16; - uint8_t B_frag[4][16]; -#pragma unroll - for (int i = 0; i < 4; ++i) { - int src_row_idx = src_row_base_idx + (i / 2) * 8 + (i % 2); - int src_offset = src_row_idx * N + src_col_idx; - bool guard = src_row_idx < K && src_col_idx < N; - ldg128_cg_0(*reinterpret_cast(B_frag[i]), - *(reinterpret_cast(B_frag[i]) + 1), - *(reinterpret_cast(B_frag[i]) + 2), - *(reinterpret_cast(B_frag[i]) + 3), B + src_offset, - guard); - } - - // reorder B - uint8_t B_reorder_frag[8][8]; -#pragma unroll - for (int i = 0; i < 4; ++i) { -#pragma unroll - for (int j = 0; j < 16; ++j) { - int dst_i = j % 8; - int dst_j = i + (j / 8) * 4; - B_reorder_frag[dst_i][dst_j] = B_frag[i][j]; - } - } - - // Store B - const auto dst_row_base_idx = blockIdx.y * (128 / 4) + (lane_id / 8) * 8; - const int dst_col_idx = - blockIdx.x * (64 * 4) + warp_id * 64 + (lane_id % 8) * 8; - for (int i = 0; i < 8; ++i) { - int dst_row_idx = dst_row_base_idx + i; - int dst_offset = dst_row_idx * K * 4 + dst_col_idx; - bool guard = (dst_row_base_idx < N_32align / 4) && (dst_col_idx < K * 4); - if (guard) { - *reinterpret_cast(B_result + dst_offset) = - *reinterpret_cast(B_reorder_frag[i]); - } - } - } else { - // Load B_scale and B_zero - FType b_scale_reg, b_zero_reg; - auto src_offset = blockIdx.y * 128 + threadIdx.x; - ldg16_cg_0(b_scale_reg, B_scale + src_offset, src_offset < N); - if (B_zero != nullptr) - ldg16_cg_0(b_zero_reg, B_zero + src_offset, src_offset < N); - int dst_offset = - blockIdx.y * 128 + warp_id * 32 + (lane_id % 8) * 4 + lane_id / 8; - if (dst_offset < N_32align) { - B_scale_result[dst_offset] = b_scale_reg; - if (B_zero != nullptr) B_zero_result[dst_offset] = b_zero_reg; - } - } -} - -template -void rearrange_kn_weight_as_n32k16_order_ldg16( - const uint8_t* B, const FType* B_scale, const FType* B_zero, - uint8_t* B_result, FType* B_scale_result, FType* B_zero_result, - const int64_t K, const int64_t N, const int64_t N_32align, - cudaStream_t stream) { - if (N % 16 != 0 || K % 16 != 0) { - std::cerr << "Now only support N and K is multiples of 16" << std::endl; - } - const int BLOCK = 128; - int grid_x = (K + 64 - 1) / 64 + 1; - int grid_y = (N + 128 - 1) / 128; - dim3 grid(grid_x, grid_y); - - rearrange_kn_weight_as_n32k16_order_ldg16_kernel - <<>>(B, B_scale, B_zero, B_result, B_scale_result, - B_zero_result, K, N, N_32align); -} -} // namespace allspark - -void rearrange_kn_weight_as_n32k16_order( - torch::stable::Tensor const& b_qweight, - torch::stable::Tensor const& b_scales, - std::optional const& b_zeros, bool has_zp, - torch::stable::Tensor& b_qweight_reorder, - torch::stable::Tensor& b_scales_reorder, - std::optional const& b_zeros_reorder, - const int64_t K, const int64_t N, const int64_t N_32align) { - // Verify device and strides - STD_TORCH_CHECK(b_qweight.device().is_cuda(), "b_qweight is not on GPU"); - STD_TORCH_CHECK(b_qweight.is_contiguous(), "b_qweight is not contiguous"); - - STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); - STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); - - STD_TORCH_CHECK(b_qweight_reorder.device().is_cuda(), - "b_qweight_reorder is not on GPU"); - STD_TORCH_CHECK(b_qweight_reorder.is_contiguous(), - "b_qweight_reorder is not contiguous"); - - STD_TORCH_CHECK(b_scales_reorder.device().is_cuda(), - "b_scales_reorder is not on GPU"); - STD_TORCH_CHECK(b_scales_reorder.is_contiguous(), - "b_scales_reorder is not contiguous"); - - if (has_zp) { - STD_TORCH_CHECK(b_zeros.value().device().is_cuda(), - "b_zeros is not on GPU"); - STD_TORCH_CHECK(b_zeros.value().is_contiguous(), - "b_zeros is not contiguous"); - - STD_TORCH_CHECK(b_zeros_reorder.value().device().is_cuda(), - "b_zeros_reorder is not on GPU"); - STD_TORCH_CHECK(b_zeros_reorder.value().is_contiguous(), - "b_zeros_reorder is not contiguous"); - } - - const uint8_t* matB = reinterpret_cast(b_qweight.data_ptr()); - const void* b_scale = b_scales.data_ptr(); - const void* b_zero = has_zp ? b_zeros.value().data_ptr() : nullptr; - - uint8_t* matB_reorder = - reinterpret_cast(b_qweight_reorder.mutable_data_ptr()); - void* b_scale_reorder = b_scales_reorder.mutable_data_ptr(); - void* b_zero_reorder = - has_zp ? b_zeros_reorder.value().mutable_data_ptr() : nullptr; - - const torch::stable::accelerator::DeviceGuard device_guard( - b_qweight.get_device_index()); - cudaStream_t stream = get_current_cuda_stream(); - if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) { - allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__half>( - matB, reinterpret_cast(b_scale), - reinterpret_cast(b_zero), matB_reorder, - reinterpret_cast<__half*>(b_scale_reorder), - reinterpret_cast<__half*>(b_zero_reorder), K, N, N_32align, stream); - } else if (b_scales.scalar_type() == - torch::headeronly::ScalarType::BFloat16) { - allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__nv_bfloat16>( - matB, reinterpret_cast(b_scale), - reinterpret_cast(b_zero), matB_reorder, - reinterpret_cast<__nv_bfloat16*>(b_scale_reorder), - reinterpret_cast<__nv_bfloat16*>(b_zero_reorder), K, N, N_32align, - stream); - } -} - -STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { - m.impl("rearrange_kn_weight_as_n32k16_order", - TORCH_BOX(&rearrange_kn_weight_as_n32k16_order)); -} diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh deleted file mode 100644 index 628f6551ea..0000000000 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh +++ /dev/null @@ -1,387 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh" -using marlin::MarlinScalarType2; - -namespace allspark { - -#define CHECK_CUDA(cmd) \ - do { \ - cudaError_t cuda_status = cmd; \ - if (cuda_status != cudaSuccess) { \ - std::string err_str = cudaGetErrorString(cuda_status); \ - std::cerr << "Failed: " << __FILE__ << ":" << __LINE__ << " " \ - << err_str; \ - exit(-1); \ - } \ - } while (0) - -#define CHECK_CUBLAS(cmd) \ - do { \ - cublasStatus_t cublas_status = cmd; \ - if (cublas_status != CUBLAS_STATUS_SUCCESS) { \ - std::cerr << "Failed: " << __FILE__ << ":" << __LINE__ << " " \ - << cublas_status << std::endl; \ - exit(-1); \ - } \ - } while (0) - -template -struct SM8x_GEMM_W8A16_Splitk_Params { - const FType* A_ptr; - const QType* B_ptr; - const FType* B_scale_ptr; - const FType* B_zero_ptr; - FType* C_ptr; - int M; - int N; - int K; - int SplitK; - int GroupCnt; - int GroupSize; - FType* C_split_ptr; // for non-fused splitk reduce - float* C_tmp_ptr; // for fused splitk reduce - uint32_t* red_count_ptr; // for fused splitk reduce -}; - -struct alignas(16) BlockTileSplitkParams { - int Mtile; - int Ntile; - int SplitK; - bool EnableFuse; -}; - -template -__global__ void f16_gemm_splitk_reduce_kernel(const FType* C_split, FType* C, - uint32_t n, uint32_t n_matrix, - uint32_t matrix_size) { - auto idx = blockIdx.x * BLOCK + threadIdx.x; - - if (idx >= matrix_size) { - return; - } - - float sum = 0.f; - - int n_mat = N_MATRIX > 0 ? N_MATRIX : (int)n_matrix; - for (int i = 0; i < n_mat; ++i) { - sum += MarlinScalarType2::num2float(C_split[idx + i * matrix_size]); - } - - C[idx] = MarlinScalarType2::float2num(sum); -} - -template -void f16_gemm_splitk_reduce(const FType* C_split, FType* C, const uint32_t m, - const uint32_t n, const uint32_t n_matrix, - cudaStream_t stream) { - const int BLOCK = 128; - uint32_t matrix_size = m * n; - int grid = (matrix_size + BLOCK - 1) / BLOCK; - - void (*kernel)(const FType*, FType*, uint32_t, uint32_t, uint32_t) = nullptr; - - switch (n_matrix) { - case 4: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 5: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 6: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 7: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 8: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 9: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 10: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 11: - kernel = f16_gemm_splitk_reduce_kernel; - break; - case 12: - kernel = f16_gemm_splitk_reduce_kernel; - break; - default: - kernel = f16_gemm_splitk_reduce_kernel; - break; - } - - kernel<<>>(C_split, C, n, n_matrix, matrix_size); -} - -template -struct HalfType; -template <> -struct HalfType { - using T1 = __half; - using T2 = __half2; -}; -template <> -struct HalfType<__nv_bfloat16> { - using T1 = __nv_bfloat16; - using T2 = __nv_bfloat162; -}; - -// convert 64-bit pointer to 32-bit smem addr -__device__ __forceinline__ uint32_t smem_u32addr(const void* smem_ptr) { - uint32_t addr; - asm("{.reg .u64 u64addr;\n" - " cvta.to.shared.u64 u64addr, %1;\n" - " cvt.u32.u64 %0, u64addr;}\n" - : "=r"(addr) - : "l"(smem_ptr)); - - return addr; -} - -template -__device__ __forceinline__ void ldg16_cg_0(T& r0, const void* ptr, bool guard) { - static_assert(sizeof(T) == 2, "ldg16_cg_0: invalid T"); - - asm volatile( - "{.reg .pred p;\n" - " setp.ne.b32 p, %2, 0;\n" - " @!p mov.b16 %0, 0;\n" -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \ - __CUDA_ARCH__ >= 750 - " @p ld.global.cg.L2::128B.b16 {%0}, [%1];}\n" -#else - " @p ld.global.ca.b16 {%0}, [%1];}\n" -#endif - : "=h"(reinterpret_cast(r0)) - : "l"(ptr), "r"((int)guard)); -} - -template -__device__ __forceinline__ void ldg64_ca(T& r0, T& r1, const void* ptr, - bool guard) { - static_assert(sizeof(T) == 4, "ldg64_ca: invalid T"); - - asm volatile( - "{.reg .pred p;\n" - " setp.ne.b32 p, %3, 0;\n" -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \ - __CUDA_ARCH__ >= 750 - " @p ld.global.ca.L2::128B.v2.b32 {%0, %1}, [%2];}\n" -#else - " @p ld.global.ca.v2.b32 {%0, %1}, [%2];}\n" -#endif - : "=r"(reinterpret_cast(r0)), - "=r"(reinterpret_cast(r1)) - : "l"(ptr), "r"((int)guard)); -} - -template -__device__ __forceinline__ void ldg128_cg_0(T& r0, T& r1, T& r2, T& r3, - const void* ptr, bool guard) { - static_assert(sizeof(T) == 4, "ldg128_cg_0: invalid T"); - - asm volatile( - "{.reg .pred p;\n" - " setp.ne.b32 p, %5, 0;\n" - " @!p mov.b32 %0, 0;\n" - " @!p mov.b32 %1, 0;\n" - " @!p mov.b32 %2, 0;\n" - " @!p mov.b32 %3, 0;\n" -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4 && \ - __CUDA_ARCH__ >= 750 - " @p ld.global.cg.L2::128B.v4.b32 {%0, %1, %2, %3}, [%4];}\n" -#else - " @p ld.global.cg.v4.b32 {%0, %1, %2, %3}, [%4];}\n" -#endif - : "=r"(reinterpret_cast(r0)), - "=r"(reinterpret_cast(r1)), - "=r"(reinterpret_cast(r2)), - "=r"(reinterpret_cast(r3)) - : "l"(ptr), "r"((int)guard)); -} - -template -__device__ __forceinline__ void lds128(T& reg0, T& reg1, T& reg2, T& reg3, - const uint32_t addr) { - static_assert(sizeof(T) == 4, "lds128: invalid T"); - - asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];\n" - : "=r"(reinterpret_cast(reg0)), - "=r"(reinterpret_cast(reg1)), - "=r"(reinterpret_cast(reg2)), - "=r"(reinterpret_cast(reg3)) - : "r"(addr)); -} - -template -__device__ __forceinline__ void stg128(const T& r0, const T& r1, const T& r2, - const T& r3, const void* ptr, - bool guard) { - static_assert(sizeof(T) == 4, "stg128: invalid T"); - - asm volatile( - "{.reg .pred p;\n" - " setp.ne.b32 p, %1, 0;\n" - " @p st.global.v4.b32 [%0], {%2, %3, %4, %5};}\n" - : - : "l"(ptr), "r"((int)guard), "r"(reinterpret_cast(r0)), - "r"(reinterpret_cast(r1)), - "r"(reinterpret_cast(r2)), - "r"(reinterpret_cast(r3))); -} - -template -__device__ __forceinline__ void ldsm_4(T& r0, T& r1, T& r2, T& r3, - const uint32_t& addr) { - static_assert(sizeof(T) == 4, "ldsm_4: invalid T"); -#if (__CUDA_ARCH__ >= 750) && (__CUDACC_VER_MAJOR__ >= 11) - asm volatile( - "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(reinterpret_cast(r0)), - "=r"(reinterpret_cast(r1)), - "=r"(reinterpret_cast(r2)), - "=r"(reinterpret_cast(r3)) - : "r"(addr)); -#endif -} - -template -__device__ __forceinline__ void hmma16816_f32(float (&d)[4], - const uint32_t (&a)[4], - const uint32_t (&b)[2]); - -template <> -__device__ __forceinline__ void hmma16816_f32<__half>(float (&d)[4], - const uint32_t (&a)[4], - const uint32_t (&b)[2]) { -#if (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11) - asm volatile( - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, " - "{%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" - : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1])); -#endif -} - -template <> -__device__ __forceinline__ void hmma16816_f32<__nv_bfloat16>( - float (&d)[4], const uint32_t (&a)[4], const uint32_t (&b)[2]) { -#if (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11) - asm volatile( - "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, " - "{%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" - : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1])); -#endif -} - -template -__device__ __forceinline__ void cp_async(const uint32_t smem_addr, - const void* gmem_ptr, - const int src_in_bytes, bool guard) { - static_assert( - (SIZE_IN_BYTES == 4 || SIZE_IN_BYTES == 8 || SIZE_IN_BYTES == 16), - "Size is not supported"); -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800 - asm volatile( - "{.reg.pred p;\n" - " setp.ne.b32 p, %4, 0;\n" - #if __CUDACC_VER_MINOR__ >= 4 - " @p cp.async.cg.shared.global.L2::256B [%0], [%1], %2, %3;}\n" - #else - " @p cp.async.cg.shared.global [%0], [%1], %2, %3;}\n" - #endif - ::"r"(smem_addr), - "l"(gmem_ptr), "n"(SIZE_IN_BYTES), "r"(src_in_bytes), "r"((int)guard)); -#endif -} - -__device__ __forceinline__ void cp_async_commit_group() { -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800 - asm volatile("cp.async.commit_group;\n"); -#endif -} - -template -__device__ __forceinline__ void cp_asyc_wait_group() { -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800 - asm volatile("cp.async.wait_group %0;\n" : : "n"(N)); -#endif -} - -template -__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128(const uint32_t& idata, - T* fdata); - -template <> -// fast conversion: 4xuint8 to 4xhalf, subtracting bias = 128 -__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128<__half2>( - const uint32_t& idata, __half2* fdata) { - uint32_t i10, i32; - asm volatile( - "prmt.b32 %0, %2, 0x64, 0x4140;" - "prmt.b32 %1, %2, 0x64, 0x4342;" - : "=r"(i10), "=r"(i32) - : "r"(idata)); - - static constexpr uint32_t MAGIC_NUM = 0x64806480; - fdata[0] = __hsub2(reinterpret_cast(i10), - reinterpret_cast(MAGIC_NUM)); - fdata[1] = __hsub2(reinterpret_cast(i32), - reinterpret_cast(MAGIC_NUM)); -} - -template <> -// fast conversion: 4xuint8 to 4xbfloat16, subtracting bias = 128 -// reference from marlin fast implementation -__device__ __forceinline__ void cvt_8bx4_to_16bx4_bias128<__nv_bfloat162>( - const uint32_t& idata, __nv_bfloat162* fdata) { - float fp32_imd[4]; - uint32_t* fp32_imd_casted = reinterpret_cast(fp32_imd); - asm volatile( - "prmt.b32 %0, %4, 0x4B000000, 0x7650;" - "prmt.b32 %1, %4, 0x4B000000, 0x7651;" - "prmt.b32 %2, %4, 0x4B000000, 0x7652;" - "prmt.b32 %3, %4, 0x4B000000, 0x7653;" - : "=r"(fp32_imd_casted[0]), "=r"(fp32_imd_casted[1]), - "=r"(fp32_imd_casted[2]), "=r"(fp32_imd_casted[3]) - : "r"(idata)); - - fp32_imd[0] -= 8388736.f; - fp32_imd[1] -= 8388736.f; - fp32_imd[2] -= 8388736.f; - fp32_imd[3] -= 8388736.f; - - uint32_t* bf16_res = reinterpret_cast(fdata); - asm volatile( - "prmt.b32 %0, %2, %3, 0x7632;" - "prmt.b32 %1, %4, %5, 0x7632;" - : "=r"(bf16_res[0]), "=r"(bf16_res[1]) - : "r"(fp32_imd_casted[0]), "r"(fp32_imd_casted[1]), - "r"(fp32_imd_casted[2]), "r"(fp32_imd_casted[3])); -} - -static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 - assert(false); -#else - return __bfloat162bfloat162(x); -#endif - __builtin_unreachable(); // Suppress missing return statement warning -} - -static __device__ half2 inline num2num2(const half x) { - return __half2half2(x); -} - -} // namespace allspark diff --git a/csrc/libtorch_stable/quantization/hadamard/hadacore/hadamard_transform_cuda.cu b/csrc/libtorch_stable/quantization/hadamard/hadacore/hadamard_transform_cuda.cu index 665585caa4..a5d199db27 100644 --- a/csrc/libtorch_stable/quantization/hadamard/hadacore/hadamard_transform_cuda.cu +++ b/csrc/libtorch_stable/quantization/hadamard/hadacore/hadamard_transform_cuda.cu @@ -796,7 +796,7 @@ torch::stable::Tensor hadacore_transform(torch::stable::Tensor& x, bool inplace) VLLM_STABLE_DISPATCH_HALF_TYPES(x.scalar_type(), "hadacore_transform_runfht", [&] { auto constexpr SCALAR_TYPE = torch::headeronly::CppTypeToScalarType::value; - hadacore::run_fht(x.data_ptr(), x.data_ptr(), x.numel(), had_size, stream); + hadacore::run_fht(x.data_ptr(), out.data_ptr(), x.numel(), had_size, stream); }); if (numel % 256 != 0) { diff --git a/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu b/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu index 86696e2f7a..abebd6f02f 100644 --- a/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu +++ b/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu @@ -1,11 +1,60 @@ #include "../../../../quantization/w8a8/fp8/common.cuh" +#include "../../../../cuda_compat.h" #include "../../../dispatch_utils.h" #include "../../../cub_helpers.h" #include "../../vectorization_utils.cuh" #include "../../../torch_utils.h" #include + +#include + namespace vllm { +namespace { + +// 16 fp8 outputs form one 16-byte store; the matching 16-element input load +// is 32 B for 16-bit types and 64 B for fp32. +constexpr int kFp8QuantVecSize = 16; +constexpr int kFp8QuantBlockSize = 256; + +template +using fp8_quant_vec_t = vec_n_t; + +template +bool ptr_vec_aligned(const T* ptr) { + return reinterpret_cast(ptr) % alignof(fp8_quant_vec_t) == 0; +} + +__device__ __forceinline__ float warp_reduce_max(float v) { +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { + v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset)); + } + return v; +} + +// Block-wide max, returned to every thread. Warp partials go through one +// shared-memory exchange, so the whole reduction costs a single +// __syncthreads. blockDim.x must be a multiple of WARP_SIZE and at most +// kFp8QuantBlockSize. +__device__ __forceinline__ float block_reduce_max(float v, float* warp_max) { + const int lane = threadIdx.x % WARP_SIZE; + const int warp = threadIdx.x / WARP_SIZE; + const int num_warps = blockDim.x / WARP_SIZE; + v = warp_reduce_max(v); + if (lane == 0) { + warp_max[warp] = v; + } + __syncthreads(); + v = warp_max[0]; + for (int w = 1; w < num_warps; ++w) { + v = fmaxf(v, warp_max[w]); + } + return v; +} + +} // namespace + // STRIDE_I_ZERO: true if scale_stride_i == 0 (per-tensor or per-channel) // STRIDE_J_ZERO: true if scale_stride_j == 0 (per-tensor or per-token) template -__global__ void segmented_max_reduction_strided( - float* __restrict__ scale, const scalar_t* __restrict__ input, - int hidden_size, int64_t in_row_stride, int64_t num_tokens) { - __shared__ float cache[256]; - const int tid = threadIdx.x; - int64_t token_idx = blockIdx.x; +__global__ void __launch_bounds__(kFp8QuantBlockSize) + absmax_reduction_flat(float* __restrict__ scale, + const scalar_t* __restrict__ input, + int64_t num_vecs) { + using in_vec_t = fp8_quant_vec_t; + const auto* v_in = reinterpret_cast(input); + const int64_t stride = static_cast(gridDim.x) * blockDim.x; - // one block per token. Guard in case gridDim.x > num_tokens. + float thread_max = 0.0f; +#pragma unroll 4 + for (int64_t v = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + v < num_vecs; v += stride) { + const in_vec_t vec = v_in[v]; +#pragma unroll + for (int i = 0; i < kFp8QuantVecSize; ++i) { + thread_max = fmaxf(thread_max, fabsf(static_cast(vec.val[i]))); + } + } + + __shared__ float warp_max[kFp8QuantBlockSize / 32]; + const float block_max = block_reduce_max(thread_max, warp_max); + if (threadIdx.x == 0) { + atomicMaxFloat(scale, block_max / quant_type_max_v); + } +} + +// Per-tensor abs-max for strided or unaligned rows: one block per token. +template +__global__ void __launch_bounds__(kFp8QuantBlockSize) + segmented_max_reduction_strided(float* __restrict__ scale, + const scalar_t* __restrict__ input, + int hidden_size, int64_t in_row_stride, + int64_t num_tokens) { + const int64_t token_idx = blockIdx.x; if (token_idx >= num_tokens) { return; } - const scalar_t* row_ptr = input + token_idx * in_row_stride; - // each thread scans elements of the row in a strided fashion. float thread_max = 0.0f; - for (int e = tid; e < hidden_size; e += blockDim.x) { - float v = fabsf(static_cast(row_ptr[e])); - thread_max = fmaxf(thread_max, v); - } + vectorize_read_with_alignment( + row_ptr, hidden_size, threadIdx.x, blockDim.x, + [&] __device__(const scalar_t& v) { + thread_max = fmaxf(thread_max, fabsf(static_cast(v))); + }); - cache[tid] = thread_max; - __syncthreads(); - - // parallel reduction to find row max. - for (int offset = blockDim.x / 2; offset > 0; offset >>= 1) { - if (tid < offset) { - cache[tid] = fmaxf(cache[tid], cache[tid + offset]); - } - __syncthreads(); - } - - // thread 0 updates global scale (per-tensor) atomically. - if (tid == 0) { - atomicMaxFloat(scale, cache[0] / quant_type_max_v); + __shared__ float warp_max[kFp8QuantBlockSize / 32]; + const float block_max = block_reduce_max(thread_max, warp_max); + if (threadIdx.x == 0) { + atomicMaxFloat(scale, block_max / quant_type_max_v); } } @@ -345,7 +411,7 @@ void dynamic_scaled_fp8_quant(torch::stable::Tensor& out, // [..., d] const int hidden_size = input.size(-1); const int num_tokens = input.numel() / hidden_size; - const int block_size = 256; + const int block_size = vllm::kFp8QuantBlockSize; dim3 grid(num_tokens); dim3 block(block_size); @@ -364,11 +430,29 @@ void dynamic_scaled_fp8_quant(torch::stable::Tensor& out, // [..., d] input.scalar_type(), "scaled_fp8_quant_kernel_scalar_type", [&] { VLLM_STABLE_DISPATCH_FP8_TYPES( out.scalar_type(), "scaled_fp8_quant_kernel_fp8_type", [&] { - vllm::segmented_max_reduction_strided - <<>>( - scale.mutable_data_ptr(), - input.const_data_ptr(), hidden_size, - in_row_stride, static_cast(num_tokens)); + const scalar_t* in_ptr = input.const_data_ptr(); + const bool flat = in_row_stride == hidden_size && + hidden_size % vllm::kFp8QuantVecSize == 0 && + vllm::ptr_vec_aligned(in_ptr); + if (flat) { + const int64_t num_vecs = input.numel() / vllm::kFp8QuantVecSize; + const int64_t max_blocks = + static_cast( + get_device_prop()->multiProcessorCount) * + 8; + const int64_t num_blocks = std::max( + 1, + std::min( + max_blocks, (num_vecs + block_size - 1) / block_size)); + vllm::absmax_reduction_flat + <<>>( + scale.mutable_data_ptr(), in_ptr, num_vecs); + } else { + vllm::segmented_max_reduction_strided + <<>>( + scale.mutable_data_ptr(), in_ptr, hidden_size, + in_row_stride, static_cast(num_tokens)); + } vllm::scaled_fp8_quant_kernel_strided_dynamic <<>>(out.mutable_data_ptr(), diff --git a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu index cfd1fab1ad..a0aee0cd92 100644 --- a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu +++ b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu @@ -169,6 +169,156 @@ __global__ void per_token_group_quant_8bit_kernel( #endif } +// Register-resident variant for the unpacked float32 scale layout, mirroring +// per_token_group_quant_8bit_packed_register_kernel below, which float32-scale +// callers cannot reach because it is hardwired to UE8M0 exponents packed +// 4-per-int32 with TMA-aligned strides. +// +// A group_size-128 group of 16-bit input is 256 B, so it fits in registers and +// the smem staging plus the __syncthreads() that the kernel above needs between +// its absmax and quantize passes are both avoidable. +// +// Requires GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE, i.e. group_size 128 with +// 16-bit input; CanUseRegisterPath enforces that. +template +__global__ void per_token_group_quant_8bit_register_kernel( + const T* __restrict__ input, void* __restrict__ output_q, + float* __restrict__ output_s, const int64_t num_groups, const float eps, + const float min_8bit, const float max_8bit, const int scale_num_rows, + const int scale_stride) { + constexpr int THREADS_PER_GROUP = 8; + constexpr int VEC_SIZE = 32 / sizeof(T); + static_assert(32 % THREADS_PER_GROUP == 0, + "THREADS_PER_GROUP must divide the warp size so the shuffle " + "mask stays inside one group"); + static_assert(sizeof(DST_DTYPE) == 1, + "the uint4 store packs one byte per element"); + + // CanUseRegisterPath only routes 16-bit inputs here, but + // VLLM_STABLE_DISPATCH_FLOATING_TYPES instantiates every floating type, so + // gate the body on the shape this schedule can actually serve instead of a + // static_assert. + if constexpr (GROUP_SIZE != THREADS_PER_GROUP * VEC_SIZE) { + return; + } else { + const int64_t group_id = + (static_cast(blockIdx.x) * blockDim.x + threadIdx.x) / + THREADS_PER_GROUP; + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); +#endif + + if (group_id >= num_groups) { +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaTriggerProgrammaticLaunchCompletion(); +#endif + return; + } + + const int lane_id = threadIdx.x % THREADS_PER_GROUP; + + // alignas(16) is required for the uint4 reinterpret_cast: the default + // alignof for 16-bit T is 2. + alignas(16) T regs[VEC_SIZE]; + { + const T* group_input = input + group_id * GROUP_SIZE + lane_id * VEC_SIZE; + uint4* dst = reinterpret_cast(®s[0]); + const uint4* src = reinterpret_cast(group_input); + dst[0] = src[0]; + dst[1] = src[1]; + } + + float local_absmax = eps; +#pragma unroll + for (int i = 0; i < VEC_SIZE; ++i) { + local_absmax = fmaxf(local_absmax, fabsf(static_cast(regs[i]))); + } + + // Reduce within the 8-lane subgroup that shares this group. +#ifdef USE_ROCM + const int lane_in_wave = threadIdx.x % warpSize; + const unsigned long long mask = 0xFFull << (lane_in_wave & ~7); + local_absmax = + fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4, 8)); + local_absmax = + fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2, 8)); + local_absmax = + fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1, 8)); +#else + const unsigned mask = 0xffu << (threadIdx.x & 24u); + local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4)); + local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2)); + local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1)); +#endif + + float y_s = local_absmax / max_8bit; + if constexpr (SCALE_UE8M0) { + // Bit-exact with exp2f(ceilf(log2f(fmaxf(fabsf(y_s), 1e-10f)))): bump the + // exponent whenever the mantissa is non-zero, then clear the mantissa. + y_s = fmaxf(fabsf(y_s), 1e-10f); + const uint32_t bits = __float_as_uint(y_s); + const uint32_t exp_bits = (bits >> 23) & 0xffu; + const uint32_t mant_bits = bits & 0x7fffffu; + y_s = __uint_as_float((exp_bits + (mant_bits != 0u ? 1u : 0u)) << 23); + } + + if (lane_id == 0) { + if constexpr (IS_COLUMN_MAJOR) { + const int64_t row_idx = group_id / scale_num_rows; + const int64_t col_idx = group_id % scale_num_rows; + output_s[col_idx * scale_stride + row_idx] = y_s; + } else { + output_s[group_id] = y_s; + } + } + + // A UE8M0 scale is an exact power of two, so multiplying by the reciprocal + // matches dividing. Otherwise the reciprocal costs an ulp and can move + // values that land exactly on an 8-bit boundary, so divide instead to stay + // bit-exact with the smem kernel. + const float inv_y = 1.0f / y_s; + uint32_t packed[VEC_SIZE / 4] = {}; +#pragma unroll + for (int i = 0; i < VEC_SIZE; ++i) { + const float v = static_cast(regs[i]); + const float scaled = SCALE_UE8M0 ? (v * inv_y) : (v / y_s); + const float q = fminf(fmaxf(scaled, min_8bit), max_8bit); + DST_DTYPE qb = DST_DTYPE(q); + const uint8_t byte = *reinterpret_cast(&qb); + packed[i >> 2] |= static_cast(byte) << ((i & 3) * 8); + } + + DST_DTYPE* group_output = static_cast(output_q) + + group_id * GROUP_SIZE + lane_id * VEC_SIZE; + *reinterpret_cast(group_output) = + make_uint4(packed[0], packed[1], packed[2], packed[3]); + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaTriggerProgrammaticLaunchCompletion(); +#endif + } +} + +// The register path needs group_size 128 with 16-bit input so that one group is +// exactly THREADS_PER_GROUP(8) x VEC_SIZE(16) elements, and it loads/stores +// through uint4, so both buffers must be 16 B aligned. +inline bool CanUseRegisterPath(const torch::stable::Tensor& input, + const torch::stable::Tensor& output_q, + int64_t group_size) { + const auto dtype = input.scalar_type(); + if (group_size != 128) return false; + if (dtype != torch::headeronly::ScalarType::Half && + dtype != torch::headeronly::ScalarType::BFloat16) { + return false; + } + if (!input.is_contiguous() || !output_q.is_contiguous()) return false; + const auto in_addr = reinterpret_cast(input.data_ptr()); + const auto out_addr = reinterpret_cast(output_q.data_ptr()); + return (in_addr % 16 == 0) && (out_addr % 16 == 0); +} + inline int GetGroupsPerBlock(int64_t num_groups) { if (num_groups % 16 == 0) { return 16; @@ -235,6 +385,15 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input, const int scale_num_rows = output_s.size(1); const int scale_stride = output_s.stride(1); + const bool use_register_path = + CanUseRegisterPath(input, output_q, group_size); + constexpr int REG_THREADS_PER_GROUP = 8; + constexpr int REG_BLOCK_THREADS = 256; + const int64_t reg_groups_per_block = + REG_BLOCK_THREADS / REG_THREADS_PER_GROUP; + const int reg_num_blocks = static_cast( + (num_groups + reg_groups_per_block - 1) / reg_groups_per_block); + #ifndef USE_ROCM #define LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, CHECK_BOUNDS, \ SMEM_BYTES) \ @@ -281,21 +440,72 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input, } \ } while (0) +#ifndef USE_ROCM + #define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \ + do { \ + cudaLaunchConfig_t config = {}; \ + config.gridDim = dim3(reg_num_blocks); \ + config.blockDim = dim3(REG_BLOCK_THREADS); \ + config.dynamicSmemBytes = 0; \ + config.stream = stream; \ + cudaLaunchAttribute attrs[1]; \ + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \ + attrs[0].val.programmaticStreamSerializationAllowed = 1; \ + config.numAttrs = 1; \ + config.attrs = attrs; \ + cudaLaunchKernelEx( \ + &config, \ + per_token_group_quant_8bit_register_kernel, \ + static_cast(input.data_ptr()), output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + static_cast(num_groups), (float)eps, (float)min_8bit, \ + (float)max_8bit, scale_num_rows, scale_stride); \ + } while (0) +#else + #define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \ + do { \ + per_token_group_quant_8bit_register_kernel \ + <<>>( \ + static_cast(input.data_ptr()), output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + static_cast(num_groups), (float)eps, (float)min_8bit, \ + (float)max_8bit, scale_num_rows, scale_stride); \ + } while (0) +#endif + #define LAUNCH_KERNEL(T, DST_DTYPE) \ do { \ size_t smem_bytes = \ static_cast(groups_per_block) * group_size * sizeof(T); \ if (is_column_major) { \ if (scale_ue8m0) { \ - LAUNCH_KERNEL_INST(T, DST_DTYPE, true, true, smem_bytes); \ + if (use_register_path) { \ + LAUNCH_REGISTER_INST(T, DST_DTYPE, true, true); \ + } else { \ + LAUNCH_KERNEL_INST(T, DST_DTYPE, true, true, smem_bytes); \ + } \ } else { \ - LAUNCH_KERNEL_INST(T, DST_DTYPE, true, false, smem_bytes); \ + if (use_register_path) { \ + LAUNCH_REGISTER_INST(T, DST_DTYPE, true, false); \ + } else { \ + LAUNCH_KERNEL_INST(T, DST_DTYPE, true, false, smem_bytes); \ + } \ } \ } else { \ if (scale_ue8m0) { \ - LAUNCH_KERNEL_INST(T, DST_DTYPE, false, true, smem_bytes); \ + if (use_register_path) { \ + LAUNCH_REGISTER_INST(T, DST_DTYPE, false, true); \ + } else { \ + LAUNCH_KERNEL_INST(T, DST_DTYPE, false, true, smem_bytes); \ + } \ } else { \ - LAUNCH_KERNEL_INST(T, DST_DTYPE, false, false, smem_bytes); \ + if (use_register_path) { \ + LAUNCH_REGISTER_INST(T, DST_DTYPE, false, false); \ + } else { \ + LAUNCH_KERNEL_INST(T, DST_DTYPE, false, false, smem_bytes); \ + } \ } \ } \ } while (0) @@ -314,6 +524,7 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input, #undef LAUNCH_KERNEL #undef LAUNCH_KERNEL_INST #undef LAUNCH_KERNEL_IMPL +#undef LAUNCH_REGISTER_INST } // Register-resident fast path for group_size==128. diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index a76852df89..3833c95fde 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -327,21 +327,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // BF16/FP32 x FP32 -> FP32 router GEMM for H=3072, E=256, M<=32 (SM90+). // conditionally compiled so impl registration is in source file ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); - - // reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel - ops.def( - "rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, " - "Tensor? b_zeros, " - "bool has_zp, Tensor! b_qweight_reorder, Tensor! b_scales_reorder, " - "Tensor!? b_zeros_reorder, " - "int K, int N, int N_32align) -> ()"); - - // AllSpark quantization ops - ops.def( - "allspark_w8a16_gemm(Tensor a, Tensor b_qweight, Tensor b_scales, " - "Tensor? b_qzeros, " - "SymInt n, SymInt group_size, SymInt sm_count, SymInt sm_version, SymInt " - "CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) -> Tensor"); #endif // Merge attn states @@ -794,9 +779,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { // DSV3 fused A GEMM: conditionally compiled so impl registration is in // source file (dsv3_fused_a_gemm.cu) - - // AllSpark ops: conditionally compiled so impl registrations are in source - // files (allspark_repack.cu and allspark_qgemm_w8a16.cu) #endif ops.impl("merge_attn_states", TORCH_BOX(&merge_attn_states)); diff --git a/docker/Dockerfile b/docker/Dockerfile index 895968601f..366a9a8752 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -207,7 +207,7 @@ ARG PYTORCH_NIGHTLY # Check whether to install torch nightly instead of release for this build COPY requirements/common.txt requirements/common.txt COPY requirements/cuda.txt requirements/cuda.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY pyproject.toml pyproject.toml RUN --mount=type=cache,target=/opt/uv/cache \ --mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \ @@ -229,7 +229,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ fi \ && if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ echo "Installing Rubin prerelease requirements..." \ - && /opt/venv/bin/python3 use_existing_torch.py --prefix \ + && /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 \ --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ @@ -243,7 +243,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ && uv pip install --python /opt/venv/bin/python3 torch torchaudio torchvision --pre \ --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ && echo "Installing other requirements..." \ - && /opt/venv/bin/python3 use_existing_torch.py --prefix \ + && /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ else \ @@ -316,12 +316,12 @@ COPY requirements/build/rust.txt requirements/build/rust.txt RUN --mount=type=cache,target=/opt/uv/cache \ uv pip install --python /opt/venv/bin/python3 -r requirements/build/rust.txt -# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed +# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed # by the wheel build stage. COPY rust rust COPY rust-toolchain.toml rust-toolchain.toml COPY tools/build_rust.py tools/build_rust.py -COPY build_rust.sh build_rust.sh +COPY tools/build_rust.sh tools/build_rust.sh # Cap cargo parallelism to avoid exhausting the CI host's open-file limit # (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). @@ -341,7 +341,7 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ export RUSTC_WRAPPER=sccache; \ sccache --show-stats; \ fi \ - && SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh \ + && SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh \ && if [ "$USE_SCCACHE" = "1" ]; then \ sccache --show-stats; \ fi @@ -352,7 +352,7 @@ FROM rust-build-cache AS rust-build RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,target=/root/.cargo/git,sharing=locked \ --mount=type=bind,source=.git,target=.git \ - SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh + SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh #################### RUST BUILD IMAGE #################### #################### CSRC BUILD IMAGE #################### @@ -368,7 +368,7 @@ ARG PYTORCH_NIGHTLY # Install build dependencies COPY requirements/build/cuda.txt requirements/build/cuda.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out @@ -381,11 +381,11 @@ ENV UV_LINK_MODE=copy RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ echo "Installing build requirements with Rubin prerelease PyTorch..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \ elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing build requirements without torch..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ && echo "Installing torch nightly..." \ && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \ @@ -427,7 +427,7 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build" RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${PYTORCH_NIGHTLY}" = "1" ] || \ [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ - python3 use_existing_torch.py --prefix; \ + python3 tools/use_existing_torch.py --prefix; \ fi # Provision one bare Python per `requires-python` entry; cmake reads @@ -605,7 +605,7 @@ ARG PYTORCH_NIGHTLY # Install build dependencies COPY requirements/build/cuda.txt requirements/build/cuda.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out @@ -618,11 +618,11 @@ ENV UV_LINK_MODE=copy RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ echo "Installing build requirements with Rubin prerelease PyTorch..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \ elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing build requirements without torch..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ && echo "Installing torch nightly..." \ && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \ @@ -658,7 +658,7 @@ ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1 RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${PYTORCH_NIGHTLY}" = "1" ] || \ [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ - python3 use_existing_torch.py --prefix; \ + python3 tools/use_existing_torch.py --prefix; \ fi \ && if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ sed -Ei '/^[[:space:]]*(flashinfer-python|nvidia-cutlass-dsl\[cu13\]|quack-kernels)[[:space:]]*==/Id' \ @@ -729,12 +729,12 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing dev requirements plus torch nightly..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && cat torch_lib_versions.txt >> requirements/test/cuda.in \ && uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ @@ -999,7 +999,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.6.18.post1 +ARG FLASHINFER_VERSION=0.7.0 RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ @@ -1143,14 +1143,14 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ CUDA_MAJOR="${CUDA_VERSION%%.*}"; \ if [ "$CUDA_MAJOR" -ge 12 ]; then \ if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing dev requirements plus torch nightly..." \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && cat torch_lib_versions.txt >> requirements/test/cuda.in \ && uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index d61e186dac..36792698a2 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -107,61 +107,6 @@ ENV UV_LINK_MODE="copy" ENV TARGETARCH=${TARGETARCH} -######################### TRITON-CPU BUILD IMAGE ######################### -# Branches off base-common directly (before requirements/cpu.txt is installed -# below) so a torch/etc. version bump doesn't invalidate this stage's cache -- -# it builds triton-cpu in an isolated `uv build` environment and never touches -# these packages. -FROM base-common AS vllm-triton-cpu-build - -# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ... -# Re-declared here because this stage is `FROM base-common` (not `vllm-build`), -# so it does not inherit the ARG/ENV defined there. Without it, the guard -# below would see an empty value and build triton-cpu on non-x86 targets -# (e.g. arm64). -ARG VLLM_CPU_X86=0 - -ARG USE_SCCACHE -ARG SCCACHE_ENDPOINT -ARG SCCACHE_LOCAL_ONLY=0 - -WORKDIR /vllm-workspace - -RUN mkdir dist - -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=cache,target=/root/.cache/ccache \ - --mount=type=cache,target=/root/.cache/sccache-local,sharing=locked \ - --mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \ - --mount=type=cache,target=/root/.triton \ - --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ - set -ex; \ - if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \ - git clone --recurse-submodules --shallow-submodules --filter=blob:none "https://github.com/triton-lang/triton-cpu.git"; \ - cd triton-cpu; \ - git checkout "270e696d"; \ - if [ ! -f third_party/sleef/CMakeLists.txt ]; then \ - echo "sleef submodule empty after shallow fetch; retrying unshallow"; \ - git submodule foreach --recursive 'git fetch --unshallow origin || true'; \ - git submodule update --init --recursive --force; \ - fi; \ - test -f third_party/sleef/CMakeLists.txt || { echo "FATAL: sleef submodule unavailable"; exit 1; }; \ - if [ "$USE_SCCACHE" = "1" ]; then \ - if [ "$SCCACHE_LOCAL_ONLY" = "1" ]; then \ - unset SCCACHE_BUCKET SCCACHE_REGION; \ - export SCCACHE_DIR=/root/.cache/sccache-local SCCACHE_CACHE_SIZE=20G; \ - elif [ -n "${SCCACHE_ENDPOINT}" ]; then \ - export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; \ - fi; \ - export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \ - sccache --show-stats; \ - fi; \ - uv build --wheel --out-dir=../dist; \ - if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \ - cd ..; \ - rm -rf triton-cpu; \ - fi - ######################### COMMON BASE IMAGE + PYTHON DEPS ######################### FROM base-common AS base-common-pydeps @@ -174,10 +119,10 @@ ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl # Copy requirements files for installation COPY requirements/common.txt requirements/common.txt COPY requirements/cpu.txt requirements/cpu.txt -COPY use_existing_torch.py use_existing_torch.py +COPY tools/use_existing_torch.py tools/use_existing_torch.py # For nightly: install torch nightly (CPU) first, strip torch pins from the -# requirements via use_existing_torch.py so they can't pull it back to a release +# requirements via tools/use_existing_torch.py so they can't pull it back to a release # build, then install the rest from the nightly CPU index (no --torch-backend, # which would force the stable channel). Release path is unchanged. RUN --mount=type=cache,target=/root/.cache/uv \ @@ -186,7 +131,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ echo "Installing torch nightly (CPU)..." \ && uv pip install torch torchaudio torchvision torchcodec --pre \ --index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu \ - && python3 use_existing_torch.py --prefix \ + && python3 tools/use_existing_torch.py --prefix \ && uv pip install -r requirements/cpu.txt \ --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ else \ @@ -238,12 +183,12 @@ ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} # Avoid port collision with vllm-build's own sccache daemon. ENV SCCACHE_SERVER_PORT=4227 -# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed +# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed # by the wheel build stage. COPY rust rust COPY rust-toolchain.toml rust-toolchain.toml COPY tools/build_rust.py tools/build_rust.py -COPY build_rust.sh build_rust.sh +COPY tools/build_rust.sh tools/build_rust.sh # Cap cargo parallelism to avoid exhausting the CI host's open-file limit # (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). @@ -265,10 +210,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ fi; \ sccache --show-stats; \ fi && \ - SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh && \ + SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh && \ if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi -# build_rust.sh installed rustup here via rustup.rs; put it on PATH so the +# tools/build_rust.sh installed rustup here via rustup.rs; put it on PATH so the # child stage below finds it on disk instead of re-downloading it. ENV PATH="/root/.cargo/bin:${PATH}" @@ -278,7 +223,7 @@ FROM rust-build-cache AS rust-build RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,target=/root/.cargo/git,sharing=locked \ --mount=type=bind,source=.git,target=.git \ - SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh + SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh ######################### SOURCE PREP IMAGE ######################### # Shared prep (source, rust artifacts, build deps) for both vllm-build and @@ -317,7 +262,7 @@ COPY requirements/build/cpu.txt requirements/build/cpu.txt RUN --mount=type=cache,target=/root/.cache/uv \ if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ - python3 /workspace/use_existing_torch.py --prefix \ + python3 /workspace/tools/use_existing_torch.py --prefix \ && uv pip install -r requirements/build/cpu.txt \ --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ else \ @@ -342,7 +287,7 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi # now-unpinned dependency. Done at the end of vllm-src so vllm-build and # vllm-dev, which both derive from it, inherit the unpinned tree. RUN if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ - python3 /workspace/use_existing_torch.py --prefix; \ + python3 /workspace/tools/use_existing_torch.py --prefix; \ fi ######################### BUILD IMAGE ######################### @@ -385,9 +330,12 @@ WORKDIR /vllm-workspace COPY requirements/test/cpu.txt requirements/test/cpu.txt # cpu.txt is compiled for x86_64, so platform markers are resolved away. Drop -# packages unavailable on aarch64 (decord, terratorch) for arm builds. +# packages unavailable on aarch64 (decord, terratorch) for arm builds. Also +# drop the generic PyPI triton pulled in transitively (via xgrammar) on x86 +# builds, so it doesn't clobber the triton-cpu wheel from requirements/cpu.txt. RUN case "$(uname -m)" in \ aarch64|arm64) sed -i '/^decord==/d; /^terratorch==/d' requirements/test/cpu.txt ;; \ + x86_64) sed -i '/^triton==/d' requirements/test/cpu.txt ;; \ esac # pytrec-eval-terrier bundles pre-C23 trec_eval sources. GCC 15 defaults to C23; @@ -395,7 +343,7 @@ RUN case "$(uname -m)" in \ RUN --mount=type=cache,target=/root/.cache/uv \ export CFLAGS="${CFLAGS:+${CFLAGS} }-std=gnu17" && \ if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ - python3 /workspace/use_existing_torch.py --prefix \ + python3 /workspace/tools/use_existing_torch.py --prefix \ && uv pip install -r requirements/test/cpu.txt \ --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ else \ @@ -440,14 +388,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ fi && \ pre-commit install --hook-type pre-commit --hook-type commit-msg -# Installed after requirements/test/cpu.txt (which pins triton==3.6.0) so the -# triton-cpu wheel isn't overwritten by that pin. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \ - if ls dist/*.whl >/dev/null 2>&1; then \ - uv pip install dist/*.whl; \ - fi - ENTRYPOINT ["bash"] ######################### TEST IMAGE ######################### @@ -459,12 +399,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \ uv pip install dist/*.whl -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \ - if ls dist/*.whl >/dev/null 2>&1; then \ - uv pip install dist/*.whl; \ - fi - ADD ./tests/ ./tests/ ADD ./examples/ ./examples/ ADD ./benchmarks/ ./benchmarks/ @@ -494,13 +428,7 @@ WORKDIR /vllm-workspace RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/root/.cache/ccache \ --mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \ - uv pip install "$(realpath dist/*.whl)[audio]" - -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \ - if ls dist/*.whl >/dev/null 2>&1; then \ - uv pip install dist/*.whl; \ - fi + uv pip install "$(realpath dist/*.whl)[audio,bench]" # Add labels to document build configuration LABEL org.opencontainers.image.title="vLLM CPU" diff --git a/docker/Dockerfile.ppc64le b/docker/Dockerfile.ppc64le index f0363d43be..1bb08591ca 100644 --- a/docker/Dockerfile.ppc64le +++ b/docker/Dockerfile.ppc64le @@ -36,10 +36,10 @@ COPY pyproject.toml ./ # The script is expected to install whatever python dependencies are missing # as well as whatever system libraries need to be installed from source -COPY build_vllm_*.sh ./ +COPY docker/build_vllm_*.sh docker/ RUN --mount=type=cache,target=/root/.cache/uv \ - sh ./build_vllm_$(uname -m).sh + sh ./docker/build_vllm_$(uname -m).sh # copy vllm source code to build cache COPY . . diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rock index 16e01ad723..babc4db100 100644 --- a/docker/Dockerfile.rock +++ b/docker/Dockerfile.rock @@ -135,7 +135,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ cd ${COMMON_WORKDIR}/vllm \ && uv pip install --system -r requirements/build/rust.txt \ && python3 -c \ - "from setuptools_scm import get_version; print(get_version())" \ + "from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \ > /vllm-version.txt # ----------------------- @@ -171,7 +171,7 @@ COPY rust/proto /rust-input/vllm/rust/proto COPY rust/src /rust-input/vllm/rust/src COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py -COPY build_rust.sh /rust-input/vllm/build_rust.sh +COPY tools/build_rust.sh /rust-input/vllm/tools/build_rust.sh FROM base AS rust_input_1 ARG VLLM_REPO @@ -188,7 +188,7 @@ RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \ rust/src/** \ rust-toolchain.toml \ tools/build_rust.py \ - build_rust.sh \ + tools/build_rust.sh \ && git checkout FETCH_HEAD FROM rust_input_${REMOTE_VLLM} AS rust-input @@ -244,7 +244,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ COPY --from=rust-input /rust-input/vllm/rust rust COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py -COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh +COPY --from=rust-input /rust-input/vllm/tools/build_rust.sh tools/build_rust.sh COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt # Build the release binary. Cargo's registry/git caches can be written by @@ -258,7 +258,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry, && sccache --show-stats; \ fi \ && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ - && bash build_rust.sh \ + && bash tools/build_rust.sh \ && test -x vllm/vllm-rs \ && if [ "$USE_SCCACHE" = "1" ]; then \ sccache --show-stats; \ @@ -368,7 +368,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu @@ -729,7 +729,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu @@ -776,13 +776,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ uv pip install --system /nixl_install/*.whl /deep_install/*.whl # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ util-linux \ unzip \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index adde16af3c..79896933e7 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -3,7 +3,9 @@ ARG PYTHON_VERSION=3.12 ARG TORCH_VERSION=2.12.0+rocm10.0.0 ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0 ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 -ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0 +ARG TRITON_BRANCH="c0a142ffcbb42f9a67a56bf791cad472be1d6425" # release/internal/3.8.x as of 9/21/2026 +ARG TRITON_VERSION=3.8.0+gitc0a142ff +ARG TRITON_REPO="https://github.com/ROCm/triton.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" ARG AITER_BRANCH="v0.1.22.post1" @@ -90,19 +92,15 @@ RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ && rocm-sdk init # Install PyTorch wheels (nightly for latest torch, release as fallback) -# Triton is pinned here rather than left to torch's Requires-Dist, so a wheel bump that -# moves the triton torch wants fails resolution instead of quietly changing compilers. ARG TORCH_VERSION ARG TORCHAUDIO_VERSION ARG TORCHVISION_VERSION -ARG TRITON_VERSION RUN pip install \ --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ "torch==${TORCH_VERSION}" \ "torchaudio==${TORCHAUDIO_VERSION}" \ - "torchvision==${TORCHVISION_VERSION}" \ - "triton==${TRITON_VERSION}" + "torchvision==${TORCHVISION_VERSION}" # Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH: @@ -257,6 +255,20 @@ RUN pip install pyyaml && cd aiter \ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install +### +### Triton Build +### +FROM base AS build_triton +ARG TRITON_BRANCH +ARG TRITON_REPO +RUN git clone ${TRITON_REPO} +RUN cd triton \ + && git checkout ${TRITON_BRANCH} \ + && if [ ! -f setup.py ]; then cd python; fi \ + && python3 setup.py bdist_wheel --dist-dir=dist \ + && mkdir -p /app/install && cp dist/*.whl /app/install + + ### ### ROCr + CLR build ### @@ -320,6 +332,8 @@ RUN mkdir -p /staging/lib \ # only includes dependencies used by wheel release pipeline FROM base AS debs_wheel_release RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ @@ -327,6 +341,8 @@ RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ FROM base AS debs RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ @@ -335,6 +351,7 @@ RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ cp /install/*.whl /app/debs FROM base AS final +ARG TRITON_VERSION # torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA # teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix. @@ -359,7 +376,15 @@ RUN set -eux; \ RUN ldconfig -ARG TRITON_VERSION +# This is to prevent resolvers from checking METADATA and RECORD and clobbering +# the Triton source build we installed. Constraints files are unable to help +# with this. However, these entries are optional and can be blanked which +# solves the problem. +RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \ + && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \ + && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \ + && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD" + RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ pip install /install/*.whl \ && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \ @@ -383,6 +408,8 @@ ARG TORCHAUDIO_VERSION ARG TORCHVISION_VERSION ARG PYTORCH_ROCM_ARCH ARG AITER_ROCM_ARCH +ARG TRITON_BRANCH +ARG TRITON_REPO ARG FA_BRANCH ARG FA_REPO ARG AITER_BRANCH @@ -391,6 +418,7 @@ ARG MORI_BRANCH ARG MORI_REPO ARG ROCM_SYSTEMS_REPO ARG ROCM_RUNTIME_COMMIT +ARG TRITON_VERSION # Record hashes for Triton and pybind11 so we can verify later. RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \ @@ -400,6 +428,8 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \ && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \ && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \ + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 0bcc8a6ca6..ecc2600a71 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -5,7 +5,7 @@ ARG VLLM_BRANCH="main" ARG COMMON_WORKDIR=/app ARG BASE_IMAGE=rocm/vllm-dev:base ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base -ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7 +ARG ROCM_TRITON_KERNELS_COMMIT=669b31acc1dd1b3fd93286afd5db67f65d9f7557 ARG INSTALL_LMCACHE=false # NIC backend for MoRI RDMA support. # By default (all), drivers and userspace libraries for all supported NIC types @@ -135,7 +135,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ cd ${COMMON_WORKDIR}/vllm \ && uv pip install --system -r requirements/build/rust.txt \ && python3 -c \ - "from setuptools_scm import get_version; print(get_version())" \ + "from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \ > /vllm-version.txt # ----------------------- @@ -171,7 +171,7 @@ COPY rust/proto /rust-input/vllm/rust/proto COPY rust/src /rust-input/vllm/rust/src COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py -COPY build_rust.sh /rust-input/vllm/build_rust.sh +COPY tools/build_rust.sh /rust-input/vllm/tools/build_rust.sh FROM base AS rust_input_1 ARG VLLM_REPO @@ -188,7 +188,7 @@ RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \ rust/src/** \ rust-toolchain.toml \ tools/build_rust.py \ - build_rust.sh \ + tools/build_rust.sh \ && git checkout FETCH_HEAD FROM rust_input_${REMOTE_VLLM} AS rust-input @@ -244,7 +244,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ COPY --from=rust-input /rust-input/vllm/rust rust COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py -COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh +COPY --from=rust-input /rust-input/vllm/tools/build_rust.sh tools/build_rust.sh COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt # Build the release binary. Cargo's registry/git caches can be written by @@ -258,7 +258,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry, && sccache --show-stats; \ fi \ && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ - && bash build_rust.sh \ + && bash tools/build_rust.sh \ && test -x vllm/vllm-rs \ && if [ "$USE_SCCACHE" = "1" ]; then \ sccache --show-stats; \ @@ -366,8 +366,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ @@ -749,8 +748,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ @@ -799,13 +797,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ util-linux \ unzip \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index eb4bb5cc97..750d4ce4d9 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -1,9 +1,9 @@ ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete -ARG TRITON_BRANCH="f0b55c0" # release/internal/3.7.x as of 08/18 +ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17 ARG TRITON_REPO="https://github.com/ROCm/triton.git" -ARG PYTORCH_BRANCH="6bbd260" # release/2.12 as of 08/02 +ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17 ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git" -ARG PYTORCH_VISION_BRANCH="v0.27.1" +ARG PYTORCH_VISION_BRANCH="v0.28.0" ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" ARG PYTORCH_AUDIO_BRANCH="v2.11.0" ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git" diff --git a/docker/Dockerfile.rocm_gfx1250 b/docker/Dockerfile.rocm_gfx1250 index ecb618d990..ac6dcdff0a 100644 --- a/docker/Dockerfile.rocm_gfx1250 +++ b/docker/Dockerfile.rocm_gfx1250 @@ -136,7 +136,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ cd ${COMMON_WORKDIR}/vllm \ && uv pip install --system -r requirements/build/rust.txt \ && python3 -c \ - "from setuptools_scm import get_version; print(get_version())" \ + "from setuptools_scm import get_version; print(get_version(git_describe_command=['git', 'describe', '--dirty', '--tags', '--long', '--abbrev=40', '--match', 'v[0-9]*']))" \ > /vllm-version.txt # ----------------------- @@ -166,7 +166,7 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry, --mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \ cd ${COMMON_WORKDIR}/vllm \ && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ - && bash build_rust.sh \ + && bash tools/build_rust.sh \ && test -x vllm/vllm-rs # ----------------------- @@ -560,11 +560,19 @@ RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \ COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index ee4c2a0d3b..9a33f6ba38 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -14,12 +14,12 @@ WORKDIR /workspace COPY requirements/build/rust.txt requirements/build/rust.txt RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt -# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed +# Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed # by the wheel build stage. COPY rust rust COPY rust-toolchain.toml rust-toolchain.toml COPY tools/build_rust.py tools/build_rust.py -COPY build_rust.sh build_rust.sh +COPY tools/build_rust.sh tools/build_rust.sh # Cap cargo parallelism to avoid exhausting the CI host's open-file limit # (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). @@ -29,7 +29,7 @@ ENV CARGO_BUILD_JOBS=4 # inputs. The child stage below relinks the crates that embed the exact version. RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,target=/root/.cargo/git,sharing=locked \ - SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh + SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh # Relink with the exact Git-derived package version. FROM rust-build-cache AS rust-build @@ -37,7 +37,7 @@ FROM rust-build-cache AS rust-build RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,target=/root/.cargo/git,sharing=locked \ --mount=type=bind,source=.git,target=.git \ - SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash build_rust.sh + SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh FROM ubuntu:24.04 AS vllm-base @@ -87,11 +87,14 @@ RUN mkdir neo && \ # Install XPU-Manager payloads without registering conflicting libze package # dependencies in dpkg. +# Note that ocl-icd-opencl-dev is required by DLE upgrade but xpu-smi, should +# can be removed in future. RUN apt-get update -y && \ apt-get install -y --no-install-recommends software-properties-common && \ add-apt-repository -y ppa:kobuk-team/intel-graphics && \ apt-get update -y && \ apt-get install -y --no-install-recommends libhwloc15 libpciaccess0 libigsc1 intel-metrics-library && \ + apt-get install -y ocl-icd-opencl-dev && \ mkdir xpu-m && \ cd xpu-m && \ wget https://github.com/intel/xpumanager/releases/download/v2.1.0/xpu-smi_2.1.0+26.33.6468cec-1.24.04_amd64.deb && \ @@ -264,4 +267,19 @@ RUN --mount=type=cache,target=/root/.cache/uv \ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install -e tests/vllm_test_utils +# Replace 2026.1.2 DLE packages to 2026.1.3 +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=.git,target=.git \ + uv pip install \ + "intel-cmplr-lib-rt==2026.1.1" \ + "intel-cmplr-lib-ur==2026.1.1" \ + "intel-cmplr-lic-rt==2026.1.1" \ + "intel-sycl-rt==2026.1.1" \ + "oneccl-devel==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "oneccl==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dpcpp-cpp-rt==2026.1.1" \ + "intel-opencl-rt==2026.1.1" \ + "intel-openmp==2026.1.1" \ + "intel-pti==1.1.0" + ENTRYPOINT ["vllm", "serve"] diff --git a/build_vllm_ppc64le.sh b/docker/build_vllm_ppc64le.sh similarity index 100% rename from build_vllm_ppc64le.sh rename to docker/build_vllm_ppc64le.sh diff --git a/docker/versions.json b/docker/versions.json index 7a5b9f44ab..ec23b95186 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -71,7 +71,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.6.18.post1" + "default": "0.7.0" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/docs/cli/README.md b/docs/cli/README.md index 274dac2d8f..27425cd92c 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -9,7 +9,7 @@ vllm --help Available Commands: ```bash -vllm {chat,complete,serve,launch,bench,collect-env,run-batch} +vllm {chat,complete,serve,launch,bench,collect-env,run-batch,preload} ``` ## serve @@ -212,6 +212,33 @@ vllm run-batch \ See [vllm run-batch](./run-batch.md) for the full reference of all available arguments. +## preload + +Launch weight cache daemons (one per GPU) that hold the post-quantized, +TP-sharded weights in GPU memory and serve CUDA IPC handles to vLLM engines +over a Unix domain socket. Restarting engines then map the weights via +zero-copy IPC instead of reloading from disk, enabling fast engine restarts. + +```bash +# Launch one daemon per GPU +vllm preload --model meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4 + +# Engines then load from the daemons +vllm serve meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4 \ + --load-format ipc_cache +``` + +The daemon accepts the standard engine arguments (model, dtype, quantization, +tensor-parallel-size, ...) plus `--weight-cache-socket-dir` to override the +directory holding the per-GPU Unix sockets, and `--weight-cache-master-port` / +`--weight-cache-draft-master-port` to pin the daemon rendezvous ports for +multi-node and speculative-decoding setups. Tensor, expert and data +parallelism are supported; pipeline parallelism is rejected at launch. + +See [Preload](../features/preload.md) for how it works, cache modes, and +limitations, and [vllm preload](./preload.md) for the full reference of all +available arguments. + ## More Help For detailed options of any subcommand, use: diff --git a/docs/contributing/README.md b/docs/contributing/README.md index e4b86f99ae..d0c2e25315 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -46,8 +46,8 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e . To rebuild only the Rust frontend binary: ```bash -./build_rust.sh # release build -./build_rust.sh --debug # faster build for development +./tools/build_rust.sh # release build +./tools/build_rust.sh --debug # faster build for development ``` If you are developing vLLM's Python and CUDA/C++ code, install PyTorch first: @@ -89,6 +89,11 @@ pre-commit install vLLM's `pre-commit` hooks will now run automatically every time you commit. +When running ShellCheck or markdownlint directly, or configuring an editor +integration, use the configuration files in `tools/pre_commit/`: +`--rcfile=tools/pre_commit/.shellcheckrc` for ShellCheck and +`--config tools/pre_commit/.markdownlint.yaml` for markdownlint-cli2. + !!! tip "Tips" You can manually run the `pre-commit` hooks using: diff --git a/RELEASE.md b/docs/contributing/release_process.md similarity index 100% rename from RELEASE.md rename to docs/contributing/release_process.md diff --git a/docs/design/cuda_graphs.md b/docs/design/cuda_graphs.md index e274b68c70..abebdd54ae 100644 --- a/docs/design/cuda_graphs.md +++ b/docs/design/cuda_graphs.md @@ -172,6 +172,8 @@ class AttentionCGSupport(enum.Enum): Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation mode. For the complete fallback policy, please see the code for [this][vllm.v1.worker.gpu_model_runner.GPUModelRunner._check_and_update_cudagraph_mode]. +Variable-length decode batches, where each request carries a different number of query tokens read from the device `query_start_loc` (as in adaptive verification), are declared separately and not ordered against the enum: `AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len()` returns the largest per-request query length a FULL decode graph can replay. It returns `None` for builders reporting `ALWAYS`, which replay any batch, and for builders that cannot replay variable-length batches; FlashInfer returns `1 + num_speculative_tokens` when its TRTLLM-GEN varlen decode path is active. Batches with a prefill never replay these graphs. + The following table lists backends that support full CUDA Graphs at the time of writing. | Attention Backend | cudagraph_support | Comments | diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 202d8d04c1..8e49a4099c 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -42,34 +42,39 @@ When new data is passed in, we first check which items are in the cache, and whi ### Fused Normalisation on the Device -To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the heavy numerical preprocessing from the CPU to the GPU and optimise data movement. +To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the **normalisation and rescaling** steps from the CPU to the device and optimises data movement. #### Fusing Normalisation and Rescaling on the GPU -Traditionally, the CPU would divide pixel values by 255, then subtract the mean and divide by the standard deviation. We fuse these steps into one operation and run it entirely on the GPU. +Traditionally, the CPU would divide pixel values by 255, then subtracts the mean and divides by the standard deviation. We fuse these steps into a single per-channel affine transform on the device. -**How it works**: We use a dedicated `FusedInputNorm` module that bakes the rescale factor (1/255) directly into the layer's `weight` and `bias` parameters. Instead of performing three separate steps (scale, subtract, divide), the module does everything in a single affine transformation: `y = x * weight + bias`. +`FusedMMInputNorm` dispatches to `fused_mm_input_norm_triton` on CUDA, which applies: -The parameters are set as follows: + y = x * weight[c] + bias[c] + = (x * rescale_factor - image_mean[c]) / image_std[c] -- `weight` controls both the standard deviation and the rescale factor -- `bias` centers the data using the mean and the same rescale factor +with: -This means the module takes raw pixel values (0–255) and outputs properly normalised values without ever explicitly dividing by 255 as a separate step. + weight[c] = rescale_factor / image_std[c] + bias[c] = -image_mean[c] / image_std[c] -#### Optimized Data Path for Fused Normalisation +The kernel accepts `uint8`, `float16`, `bfloat16` and `float32` inputs. The `uint8` path is the primary fast path: raw bytes travel to the device unprocessed, and the rescale factor is folded into `weight` — no separate divide-by-255 step. Compute is always done in fp32 inside the kernel. -Performing fused normalisation directly on the device allows us to keep the entire transfer path—from **Entrypoint** through **Engine Core** to **GPU memory**—in **`uint8`**. This halves PCIe bandwidth and reduces CPU memory footprint. +#### Optimized Data Path -Only after data reaches GPU memory do we cast to `fp32` for the `FusedInputNorm` layer (to ensure numerical accuracy), then cast to `bf16` for subsequent layers—all within the GPU, avoiding any host‑side conversions. + Before: CPU decode → CPU resize → CPU rescale (÷255) → CPU normalize + → cast to bf16 → PCIe (2 B/elem) → GPU -Overall path: **`Entrypoint (uint8) → Engine Core (uint8) → GPU Memory (uint8)`** → GPU‑local `fp32` `FusedInputNorm` → `bf16` output. + After: CPU decode → CPU resize → uint8 pixel_values + → PCIe (1 B/elem) → GPU fused affine (fp32 compute) → bf16 + +The transfer path **Entrypoint (API server / offline LLM) → Engine Core(scheduler + workers) → Device Memory** stays in `uint8`. On the device,`fused_mm_input_norm_triton` computes in fp32 internally and writes the requested `visual_dtype` (a per-call argument to `forward_*`, commonly `bf16`) directly, without materialising a global fp32 intermediate. #### Toggle: `mm_device_do_normalize` -This GPU‑side fusion is controlled by a config flag called **`mm_device_do_normalize`**. +GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: -- When `True`, normalisation and rescaling are done on the GPU using the `FusedInputNorm` layer; when `False`, we fall back to the old CPU‑side path. +- When `True`, normalisation and rescaling are done on the GPU using the `FusedMMInputNorm` layer; when `False`, we fall back to the old CPU‑side path. - The flag is **enabled by default** for all models that support it. - Currently, it’s on by default for these architectures: @@ -78,8 +83,9 @@ This GPU‑side fusion is controlled by a config flag called **`mm_device_do_nor | `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | | `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | -#### What We Gain Overall +#### Key Properties and Gains -- **CPU offload**: The arithmetic for normalisation and rescaling is completely gone from the CPU. -- **PCIe savings**: Sending `uint8` (1 byte) instead of `bf16` (2 bytes) slashes data transfer volume by **50%** . -- **GPU overhead**: The fused kernel is very lightweight and can often be merged with subsequent CUDA operations, so it hardly adds any extra cost. +- **CPU offload & 50% PCIe savings** — normalisation/rescaling leaves the CPU entirely; sending `uint8` (1 byte) instead of `bf16` (2 bytes) halves transfer volume. +- **Single-pass fusion** — `y = x * weight[c] + bias[c]` in one kernel launch via `fused_mm_input_norm_triton`; input is read in its native dtype (`uint8`). +- **float32 compute for free** — arithmetic runs in fp32 inside the kernel regardless of I/O dtypes. Accuracy matches fp32, with no extra bandwidth: intermediates stay in registers, and no global fp32 tensor is materialised. +- **Preallocated output with batch-dim padding** — callers can pass a larger buffer; only the leading `N` rows are written, so buffers are reusable across calls. diff --git a/docs/design/torch_compile.md b/docs/design/torch_compile.md index 4dc0da0c7d..1ffc0ac53a 100644 --- a/docs/design/torch_compile.md +++ b/docs/design/torch_compile.md @@ -27,6 +27,12 @@ With all these factors taken into consideration, usually we can guarantee that t A unique aspect of vLLM's `torch.compile` integration, is that we guarantee all the compilation finishes before we serve any requests. No requests will trigger new compilations. Otherwise, the engine would be blocked on that request, and the response time will have unexpected spikes. +An explicit `TRITON_CACHE_DIR` is preserved, allowing processes and model +configurations to reuse Triton artifacts. The directory can start empty: the +first compilation populates it and later matching specializations reuse it. +When copying compilation caches, also copy this directory if set. Cold-cache +tests should set it to a fresh directory. + By default, the cache saves compiled artifacts as binary files. If you would like to interact with the generated code for debugging purposes, set the field `compile_cache_save_format=unpacked` in the compilation config, or omit this and set the env variable `VLLM_COMPILE_CACHE_SAVE_FORMAT=unpacked`. ## Dynamic shapes and vllm guard dropping diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 789878f48a..0fad4682fc 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -144,7 +144,10 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode) +3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) +4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), + runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out + (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). !!! note Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility. diff --git a/docs/features/disagg_encoder.md b/docs/features/disagg_encoder.md index 383e7d45ff..ac304345fd 100644 --- a/docs/features/disagg_encoder.md +++ b/docs/features/disagg_encoder.md @@ -61,6 +61,19 @@ and compares the disaggregated outputs against it. Set `MODEL` to use another mo or `MOONCAKE_EC_PROTOCOL=rdma` to use RDMA on supported hardware. See the script for additional configuration options. +### Audio inputs + +The Python EPD proxy also rewrites audio in `/v1/chat/completions` into +metadata-only references for Qwen2-Audio, AudioFlamingo3, Ultravox, +Qwen2.5-Omni, and Qwen3-Omni. The encoder publishes `audio_num_tokens` +(the placeholder token count of each audio) together with any feature +lengths the model needs; the consumer uses them to reconstruct the audio +placeholders and receives the embeddings through its EC connector without +repeating audio preprocessing. + +This covers pure audio inputs, not video with an embedded audio track. It does +not add `/v1/audio/transcriptions` or realtime routes to the example proxy. + --- ## 3 Test Script diff --git a/docs/features/preload.md b/docs/features/preload.md new file mode 100644 index 0000000000..74839bf35b --- /dev/null +++ b/docs/features/preload.md @@ -0,0 +1,203 @@ +# Preload + +vLLM's preload feature keeps model artifacts resident in GPU memory across +engine restarts, so a restarting engine reuses them instead of rebuilding them +from scratch. The design is extensible to different kinds of artifacts; it +currently supports **model weights** via the weight cache daemon. + +With weight preloading, a daemon process per GPU holds its rank's +post-quantized, TP-sharded weights and serves CUDA IPC handles to vLLM engines +over a Unix domain socket, so a restarting engine maps the weights via +zero-copy IPC instead of reloading them from disk. + +Key benefits: + +- **Fast engine restarts**: weight loading reduces to mapping existing GPU + memory, cutting restart time from minutes (disk load + quantization + processing) to seconds. +- **Zero-copy sharing**: in the default `zero_copy` mode the engine directly + shares the daemon's GPU memory, so a restart adds no extra GPU memory + footprint. +- **Safe fallback**: if the daemon is unavailable or its cached weights don't + match the engine's configuration, the engine falls back to loading from + disk. + +!!! note + Weight preloading is only supported on CUDA and ROCm platforms. Tensor, + expert and data parallelism are supported, including across nodes; + pipeline parallelism is rejected when launching the daemon. + +## Quick start + +Launch one weight cache daemon per GPU with a single command: + +```bash +vllm preload --model meta-llama/Llama-3.1-8B-Instruct --tensor-parallel-size 4 +``` + +The daemon process loads and post-processes the weights once, then waits for +engines. Each rank binds its Unix socket only after its shard is fully cached, +so engines connecting before that simply fall back to disk loading. + +Then start (or restart) engines with the `ipc_cache` load format: + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct \ + --tensor-parallel-size 4 \ + --load-format ipc_cache +``` + +The daemon itself must load from disk; passing `--load-format ipc_cache` to +`vllm preload` is an error. + +## How it works + +1. `vllm preload` spawns one daemon process per GPU. Each daemon loads its + shard from disk using the configured loader, runs quantization + post-processing, and exports every parameter/buffer as a CUDA IPC handle. +2. An engine started with `--load-format ipc_cache` builds its model on the + meta device (no storage) and requests the tensors from the daemon on its + GPU. +3. Before serving anything, the engine and daemon compare a fingerprint of the + cached weights: checkpoint content (hashed from safetensors metadata, so + identical weights in different directories still match), model + architecture, TP/DP size/rank, dtype, quantization method and config, model + revision, and vLLM version. On any mismatch the engine falls back to disk + loading (unless `fallback` is disabled). +4. The engine also verifies the daemon's GPU UUID matches its own device, so a + stale socket cannot serve weights for the wrong GPU. + +Tied weights (e.g. `lm_head.weight` sharing storage with +`embed_tokens.weight`) are exported once and re-established as aliases in the +engine, preserving parameter identity. + +## Multi-node and data parallelism + +CUDA IPC handles are node-local, so each node serves only its local GPUs' +shards. For multi-node tensor parallelism, run one `vllm preload` launcher per +node with a shared rendezvous: reuse the `--nnodes` / `--node-rank` / +`--master-addr` flags you pass the engine, plus a `--weight-cache-master-port` +distinct from the engine's `--master-port`: + +```bash +# node 0 (8 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 16 \ + --nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \ + --weight-cache-master-port 29600 +# node 1 (8 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 16 \ + --nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \ + --weight-cache-master-port 29600 +``` + +For data parallelism (e.g. a TP1 x DP16 x EP decode fleet), run one launcher +per node with the engine's DP placement flags. Local GPU `i` serves DP rank +`start_rank + i // tp_size` and TP rank `i % tp_size`, and all +`dp_size * tp_size` daemons form one world group on `--data-parallel-address` +/ `--weight-cache-master-port` so the expert shards are laid out exactly as in +the engine: + +```bash +# node r (4 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 1 \ + --enable-expert-parallel \ + --data-parallel-size 16 --data-parallel-size-local 4 \ + --data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \ + --weight-cache-master-port 29600 +``` + +Data parallelism also combines with multi-node tensor parallelism: pass both +flag sets. Each node then serves a contiguous block of the +`dp_size * tp_size` global ranks. + +## Speculative decoding + +With MTP, EAGLE or EAGLE3 speculative decoding, `vllm preload` additionally +starts a draft daemon group that caches the draft model. It uses its own cache +key, Unix sockets (`*_draft.sock`) and rendezvous port +(`--weight-cache-draft-master-port`, default `--weight-cache-master-port + 1`), +so each daemon process serves exactly one model role. Other draft types are +not cached and keep loading from disk in the engine. + +## Cache modes + +The loader supports two modes, selected via `--model-loader-extra-config`: + +- `zero_copy` (default): the engine maps the daemon's CUDA allocations + directly. The daemon must stay alive for the engine's lifetime, and each + GPU carries only one copy of the weights. +- `copy`: the engine clones every tensor into its own GPU memory and then + asks the daemon to release its cache. Use this when the daemon should free + GPU memory after handing off, at the cost of a full copy per restart. + +!!! warning + In `zero_copy` mode the weights live in the daemon's CUDA IPC allocations, + so [sleep mode](sleep_mode.md) (CuMemAllocator weight offloading) must not + be used with this loader. + +## Loader configuration + +The `ipc_cache` loader accepts extra keys via `--model-loader-extra-config`: + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct \ + --load-format ipc_cache \ + --model-loader-extra-config '{"mode": "copy", "fallback": false}' +``` + +| Key | Default | Description | +| --- | ------- | ----------- | +| `socket_path` | per-GPU path derived from the GPU UUID | Explicit daemon socket path. | +| `socket_dir` | per-user private dir under the temp dir | Directory containing the daemon sockets. | +| `mode` | `zero_copy` | `zero_copy` or `copy` (see above). | +| `fallback` | `true` | Fall back to disk loading when the daemon is unavailable or the fingerprints mismatch. | +| `connect_timeout_s` | `5.0` | Socket connect timeout in seconds. | +| `state_timeout_s` | `300.0` | Timeout for the weight-transfer request in seconds. | + +The daemon's `--weight-cache-socket-dir` and the loader's `socket_dir` / +`socket_path` must agree when the default per-user directory is not used. +Socket paths are derived from the GPU UUID, so they are stable regardless of +`CUDA_VISIBLE_DEVICES` index remapping. + +## Limitations + +- **Platform**: CUDA and ROCm only; other platforms raise + `UnsupportedPlatformForIPCError` even when `fallback` is enabled, since it + is a permanent misconfiguration rather than a transient daemon outage. +- **Parallelism**: tensor, expert and data parallelism are supported; + launching the daemon with pipeline parallelism is rejected. +- **Quantization**: every quantization method in the model must declare + support for pre-processed weights (the daemon transfers weights *after* + quantization post-processing). Unsupported methods raise + `UnsupportedQuantForIPCError`. +- **Consistency**: the daemon and engine must run the same vLLM version and + agree on model, dtype, quantization, and TP/DP layout, otherwise the + fingerprint mismatch triggers the disk fallback. +- **Localhost only**: the daemon serves over a Unix domain socket; both the + daemon and the engines must run on the same node as the same user. + +## Security + +The socket protocol uses pickle and is intended only for trusted local +processes owned by the same user: + +- Daemon sockets live in a per-user private directory (mode `0700`) and the + socket files are restricted to the owner (`0600`). +- Both sides reject symlinked or non-owned socket paths; the auto-derived + directory is additionally rejected if it is group/world accessible. +- On Linux the daemon verifies the connecting peer's UID via `SO_PEERCRED`. + +See the [security documentation](../usage/security.md) for vLLM's general +threat model. + +## Daemon lifecycle + +- Each GPU's socket path is guarded by an exclusive lock file, so a second + daemon for the same GPU fails fast instead of clobbering the live socket. +- `vllm preload` reports readiness once every rank is serving; if any rank + dies during startup, the remaining ranks are terminated and the command + exits with that rank's exit code. +- `SIGINT`/`SIGTERM` to the `vllm preload` process terminates all daemon + ranks, which unlinks their sockets and releases the GPU memory. + +See [vllm preload](../cli/preload.md) for the full CLI reference. diff --git a/docs/features/quantization/llm_compressor/fp8.md b/docs/features/quantization/llm_compressor/fp8.md index f823c089f4..f0d1b88264 100644 --- a/docs/features/quantization/llm_compressor/fp8.md +++ b/docs/features/quantization/llm_compressor/fp8.md @@ -143,14 +143,14 @@ If you encounter any issues or have feature requests, please open an issue on th ## Online Dynamic Quantization -Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor. +Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8_per_tensor"` in the command line or setting `quantization="fp8_per_tensor"` in the LLM constructor. In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode. ```python from vllm import LLM -llm = LLM("facebook/opt-125m", quantization="fp8") +llm = LLM("facebook/opt-125m", quantization="fp8_per_tensor") # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB result = llm.generate("Hello, my name is") print(result[0].outputs[0].text) diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index a58dc6033e..16f2a675f8 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -25,12 +25,14 @@ vLLM currently supports the following reasoning models: | [Holo2 series](https://huggingface.co/collections/Hcompany/holo2) | `holo2` | `json`, `regex` | ✅ | | [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ | | [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ | +| [IBM Granite 4.2 language models](https://huggingface.co/ibm-granite/granite-4.2-30b) | `granite_thinking_parser` | `json`, `regex` | ✅ | | [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ | | [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ | | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ | !!! note IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`. + IBM Granite 4.2 reasoning is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`. The reasoning feature for the Qwen3 series is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`. Gemma 4 reasoning is disabled by default; to enable it, pass `enable_thinking=True` in your `chat_template_kwargs` or set `reasoning_effort` (which enables it automatically). DeepSeek-V3.1 tool calling is supported in non-thinking mode. diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index f4417c7fde..7e2c36d293 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -9,6 +9,7 @@ To train your own draft models for optimized speculative decoding, see [vllm-pro vLLM supports a variety of methods of speculative decoding. Model-based methods such as EAGLE, MTP, draft models, PARD and MLP provide the best latency reduction, while simpler methods such as n-gram and suffix decoding provide modest speedups without increasing workload during peak traffic. - [EAGLE](eagle.md) +- [LiLiCorr](lilicorr.md) - [Multi-Token Prediction (MTP)](mtp.md) - [Draft Model](draft_model.md) - [Parallel Draft Model (PARD)](parallel_draft_model.md) diff --git a/docs/features/speculative_decoding/lilicorr.md b/docs/features/speculative_decoding/lilicorr.md new file mode 100644 index 0000000000..5d45d3c51f --- /dev/null +++ b/docs/features/speculative_decoding/lilicorr.md @@ -0,0 +1,77 @@ +# LiLiCorr + +LiLiCorr uses a DFlash backbone to produce per-position candidates, then scores +candidate combinations with a learned correlator. It requires a compatible +trained checkpoint and Model Runner V2, which is selected automatically. +Both plain and grouped-convolution checkpoints are supported. + +## Usage + +For a checkpoint trained with block size 16: + +```bash +vllm serve /path/to/target --dtype bfloat16 \ + --speculative-config '{"method":"dflash","model":"/path/to/lilicorr","num_speculative_tokens":15,"draft_sample_method":"probabilistic"}' +``` + +We strongly recommend explicitly setting `num_speculative_tokens` to the +checkpoint's trained `block_size - 1` (for example, `15` for `block_size=16`). +The trained block includes one anchor position. Other supported lengths are +available, but may degrade acceptance length unpredictably; benchmark them with +your checkpoint and workload before using them. + +Shorter drafts, down to one token, use the corresponding prefix of the learned +slot embeddings and attention bias without changing checkpoint parameter shapes. +Lengths above `block_size - 1` are unsupported because the checkpoint has no +learned slot embeddings for those positions. + +Fixed-length drafting with shorter verification is a separate option provided by +adaptive verification (`enable_adaptive_verification`). LiLiCorr shares the +candidate-sampling and acceptance-estimation path, but this combination has not +been validated end-to-end. Adaptive verification also requires compatible target +attention backends and variable-length CUDA graph support; do not assume it works +with Mamba2 targets. + +## Proposal sampling + +- `"draft_sample_method":"greedy"` selects the highest-scoring candidate at each + step of the correlator's conditional walk. +- `"draft_sample_method":"probabilistic"` samples from those conditional scores + using the request's temperature and supplies the realized proposal + distributions to the rejection sampler. + +These settings control proposals; they do not replace the target model's +sampling settings. With request temperature zero, probabilistic proposals also +use argmax. For nonzero temperatures, benchmark both modes with your checkpoint +and workload: compare acceptance length and end-to-end throughput or latency. +Neither mode is universally faster. + +The default rejection method is `"rejection_sample_method":"standard"`. +The shared Model Runner V2 rejection sampler also exposes `"block"`, but +LiLiCorr-specific GPU correctness and performance validation for that combination +is still pending. Do not assume that changing the rejection method improves +throughput; evaluate it separately from the proposal sampling mode. + +## Checkpoint requirements + +The checkpoint must declare `LiLiCorrDraftModel` and include all `lilicorr_*` +geometry fields and trained head weights. Geometry is read from `dflash_config`. +Convolution tensors must match the configured `conv_kernel_size` and +`conv_group_size`. Target input embeddings must be available on the draft rank. +By default, candidates use the target LM head, which must also be available there. +A checkpoint with top-level `"has_own_lm_head": true` instead uses its own +`lm_head` with the draft quantization configuration and exclusions. Its weights +must be present; for ModelOpt NVFP4 this includes `lm_head.weight`, +`lm_head.weight_scale`, and `lm_head.weight_scale_2`. The owned head is preserved +and is not replaced by the target head. Candidate token embeddings still come +from the target. + +Correlator linear layers and convolution kernel projections use the draft +quantization configuration and its module exclusions. The following parameters +must remain in floating-point model dtype: + +- Lattice QKV `in_proj_weight` and `in_proj_bias`, which retain the exported + parameter layout. +- `lilicorr.factor_input_proj`, `lilicorr.out_head`, and `lilicorr.in_head`, which + are used by the split factor and fused edge projections. Exclude these modules + when exporting a quantized checkpoint. diff --git a/docs/getting_started/installation/gpu.cuda.inc.md b/docs/getting_started/installation/gpu.cuda.inc.md index d110e377da..a0b406e6a4 100644 --- a/docs/getting_started/installation/gpu.cuda.inc.md +++ b/docs/getting_started/installation/gpu.cuda.inc.md @@ -111,8 +111,8 @@ This command will do the following: If you need to recompile the `vllm-rs` Rust frontend binary, you can rebuild and install it without re-running the full pip install: ```bash - ./build_rust.sh # release build - ./build_rust.sh --debug # faster build for development + ./tools/build_rust.sh # release build + ./tools/build_rust.sh --debug # faster build for development ``` This will install the required Rust toolchain if needed, build the binary, and place it in `vllm/vllm-rs`. @@ -183,7 +183,7 @@ To build vLLM using an existing PyTorch installation: # install PyTorch first, either from PyPI or from source git clone https://github.com/vllm-project/vllm.git cd vllm -python use_existing_torch.py +python tools/use_existing_torch.py uv pip install -r requirements/build/cuda.txt uv pip install --no-build-isolation -e . ``` diff --git a/docs/getting_started/installation/gpu.rocm.inc.md b/docs/getting_started/installation/gpu.rocm.inc.md index 0169d01546..28e74b670c 100644 --- a/docs/getting_started/installation/gpu.rocm.inc.md +++ b/docs/getting_started/installation/gpu.rocm.inc.md @@ -187,8 +187,8 @@ If you need to recompile the `vllm-rs` Rust frontend binary, you can rebuild and install it without re-running the full installation: ```bash - ./build_rust.sh # release build - ./build_rust.sh --debug # faster build for development + ./tools/build_rust.sh # release build + ./tools/build_rust.sh --debug # faster build for development ``` This will install the required Rust toolchain if needed, build the binary, diff --git a/docs/getting_started/installation/gpu.xpu.inc.md b/docs/getting_started/installation/gpu.xpu.inc.md index 84dc69ca84..893150148a 100644 --- a/docs/getting_started/installation/gpu.xpu.inc.md +++ b/docs/getting_started/installation/gpu.xpu.inc.md @@ -32,6 +32,22 @@ To install the wheel built from the latest main branch: uv pip install vllm --extra-index-url https://wheels.vllm.ai/nightly/xpu --extra-index-url https://download.pytorch.org/whl/xpu --index-strategy unsafe-best-match ``` +Note that after upgrading to PyTorch 2.14, xpu graph support requires specific oneAPI packages. + +```bash +uv pip install \ + "intel-cmplr-lib-rt==2026.1.1" \ + "intel-cmplr-lib-ur==2026.1.1" \ + "intel-cmplr-lic-rt==2026.1.1" \ + "intel-sycl-rt==2026.1.1" \ + "oneccl-devel==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "oneccl==2022.1.2; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dpcpp-cpp-rt==2026.1.1" \ + "intel-opencl-rt==2026.1.1" \ + "intel-openmp==2026.1.1" \ + "intel-pti==1.1.0" +``` + #### Install specific revisions If you want to access the wheels for previous commits (e.g. to bisect the behavior change, performance regression), you can specify the commit hash in the URL: diff --git a/docs/mkdocs/gen_files/generate_argparse.py b/docs/mkdocs/gen_files/generate_argparse.py index ecdce7761f..a45de12f2e 100644 --- a/docs/mkdocs/gen_files/generate_argparse.py +++ b/docs/mkdocs/gen_files/generate_argparse.py @@ -177,6 +177,7 @@ BenchmarkMMProcessorSubcommand = auto_mock( LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase") launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION") RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand") +PreloadSubcommand = auto_mock("vllm.entrypoints.cli.preload", "PreloadSubcommand") sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS") openai_cli_args = auto_mock("vllm.entrypoints.launchers", "cli_args") run_batch = auto_mock("vllm.entrypoints.launchers", "run_batch") @@ -330,6 +331,7 @@ pages = { "cli/chat.md": (create_parser(ChatCommand.add_cli_args), False), "cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False), "cli/run-batch.md": (create_parser(run_batch.make_arg_parser), True), + "cli/preload.md": (create_parser(PreloadSubcommand.add_cli_args), True), "cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True), "cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True), # URL kept as `mm_processor` for back-compat; command name is `mm-processor` diff --git a/docs/mkdocs/gen_files/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py index b952f53977..4d4fa93cc5 100644 --- a/docs/mkdocs/gen_files/generate_metrics.py +++ b/docs/mkdocs/gen_files/generate_metrics.py @@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [ "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", "key": "metrics-nixl", }, + { + "path": "vllm/v1/simple_kv_offload/metrics.py", + "key": "metrics-simple-kv-offload", + }, {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, ] diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index c6e1786ee0..df44a3e8b0 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -164,9 +164,9 @@ For your model to be compatible with vLLM's tensor parallel and/or pipeline para ```python -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class MyConfig(PretrainedConfig): +class MyConfig(PreTrainedConfig): base_model_tp_plan = { "layers.*.self_attn.k_proj": "colwise", "layers.*.self_attn.v_proj": "colwise", @@ -549,6 +549,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ | | `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I+ | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ | | `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I+ | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ | +| `DiffusionGemmaForBlockDiffusion` | DiffusionGemma | T + I+ + V+ | `google/diffusiongemma-26B-A4B-it`, etc. | | | | `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + IE+ | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ | | `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I+/ V+ | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ | | `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + IE+ | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ | diff --git a/docs/serving/expert_parallel_deployment.md b/docs/serving/expert_parallel_deployment.md index d1dff32fc9..ea60233645 100644 --- a/docs/serving/expert_parallel_deployment.md +++ b/docs/serving/expert_parallel_deployment.md @@ -315,6 +315,6 @@ except Exception as e: ### Benchmarking -- To simulate the decode deployment of disaggregated serving, pass `--kv-transfer-config '{"kv_connector":"DecodeBenchConnector","kv_role":"kv_both"}'` to the `vllm serve` invocation. The connector populates KV cache with random values so decode can be profiled in isolation. +- To simulate the decode deployment of disaggregated serving, pass `--kv-transfer-config '{"kv_connector":"DecodeBenchConnector","kv_role":"kv_both"}'` to the `vllm serve` invocation. The connector populates KV cache with random values so decode can be profiled in isolation. Add `"kv_connector_extra_config":{"startup_fill":true}` to fill the whole KV cache once at startup instead of in the decode steps that admit new requests (see the connector docstring for its limitations). - **CUDAGraph capture**: Use `--compilation_config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'` to enable CUDA graph capture for decode only and save KV cache. diff --git a/docs/training/weight_transfer/README.md b/docs/training/weight_transfer/README.md index a389dc7165..77d2728345 100644 --- a/docs/training/weight_transfer/README.md +++ b/docs/training/weight_transfer/README.md @@ -35,6 +35,7 @@ engine drives on your behalf: | [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU | | [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Checkpoint-coordinate sparse weight patches | | [sharded_rdt](sharded_rdt.md) | NIXL / Ray Direct Transport (pull-based) | Very large models where each worker needs only its own slice (MoE with expert parallelism) | +| [nccl_m2n](m2n.md) | NCCL M2N reshard | Trainer and inference use different sharding layouts | ## Quickstart @@ -49,7 +50,8 @@ from vllm.config import WeightTransferConfig llm = LLM( model="my-model", - weight_transfer_config=WeightTransferConfig(backend="nccl"), # or "ipc", "sparse_nccl", "sharded_rdt" + # Other backends: "ipc", "sparse_nccl", "sharded_rdt", "nccl_m2n". + weight_transfer_config=WeightTransferConfig(backend="nccl"), ) ``` diff --git a/docs/training/weight_transfer/m2n.md b/docs/training/weight_transfer/m2n.md new file mode 100644 index 0000000000..61510978c8 --- /dev/null +++ b/docs/training/weight_transfer/m2n.md @@ -0,0 +1,101 @@ +# NCCL M2N Engine + +The `nccl_m2n` weight transfer engine uses [NCCL M2N](https://github.com/NVIDIA/nccl-extensions) to **reshard** weights between two disjoint meshes of GPUs — the trainer and the inference workers — in a single collective per parameter. Unlike the broadcast-based [NCCL engine](nccl.md), the trainer sends its local shard as-is: no full tensor is ever materialized on the training side, whatever its parallelism layout. + +## When to Use NCCL M2N + +- The trainer and the inference engine use **different parallelism layouts** (e.g. trainer FSDP/EP, inference TP) and gathering full tensors on the trainer is expensive +- Training and inference are on **separate GPUs**, possibly across nodes +- You want weight sync off the critical path of an async RL loop + +## Requirements + +This backend depends on the `nccl-extensions` Python package (and its `nccl4py` dependency), which vLLM does not install. It needs NCCL 2.30.5 or newer, and `libnccl_m2n.so` is linked against a specific NCCL build, so vLLM must load the same library: + +```bash +export VLLM_NCCL_SO_PATH=/path/to/the/same/libnccl.so +``` + +The backend imports the runtime lazily, so vLLM is unaffected unless you select it. + +## How It Works + +1. The trainer ranks and all inference workers join **one** NCCL communicator via `StatelessProcessGroup`. Trainer ranks occupy `[0, T)` and workers `[T, T + N)` — m2n meshes are contiguous rank intervals, which is exactly what vLLM's existing `rank_offset` convention produces. +2. At the init handshake the trainer ships the full transfer plan: both meshes — one per side, shared by every parameter — then each parameter's name, dtype, shape and placement. The workers resolve the matching **destination placements** and publish them back over the shared group; the meshes themselves are declared, not negotiated. +3. Each round, both sides issue one `reshard` per parameter in the same order. Every rank in the communicator participates in every reshard, so — as with the NCCL engine — the worker must be inside `update_weights` while the trainer sends. The trainer engine handles that concurrency itself. + +### Destination resolution + +A parameter is resharded **directly into the live vLLM parameter** when its checkpoint name maps 1:1 onto a model parameter whose shape is either identical to the checkpoint shape (replicated) or differs on exactly one dim by the tensor-parallel factor. That rank then only ever receives its own shard. + +Anything else falls back to receiving the full tensor and letting `load_weights` do the sharding, exactly as the NCCL engine does for every parameter. The fallback covers: + +- fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`), whose checkpoint names do not name a model parameter +- quantized models, where the weight loader does more than a sharded copy +- pipeline-parallel deployments + +Correctness never depends on a parameter resolving. The engine logs how many parameters took each path at init. + +## Configuration + +```python +from vllm import LLM +from vllm.config import WeightTransferConfig + +llm = LLM( + model="my-model", + weight_transfer_config=WeightTransferConfig(backend="nccl_m2n"), +) +``` + +```bash +vllm serve my-model --weight-transfer-config '{"backend": "nccl_m2n"}' +``` + +## Trainer Side + +The trainer uses the stateful trainer engine. m2n needs each parameter's source layout, which the base `WeightSource` does not carry, so it takes an `M2NWeightSource`. `DTensorModuleSource` covers the common case: it reads each parameter's device mesh and placements and yields **local shards** — it never calls `full_tensor()`. + +```python +from vllm.distributed.weight_transfer import ( + RayVLLMWeightSyncClient, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.m2n_source import DTensorModuleSource +from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo + +# Called on every trainer rank; rank 0 is the sender. +engine = WeightTransferTrainerFactory.trainer_init( + init_info=M2NTrainerInitInfo( + master_address=master_address, + master_port=master_port, + world_size=num_trainer_ranks + num_inference_workers, + num_trainer_ranks=num_trainer_ranks, + rank=my_rank, + ), + client=RayVLLMWeightSyncClient(llm_handle), + source=DTensorModuleSource(model, num_trainer_ranks), +) + +# Each round, on every trainer rank: +engine.send_weights() +``` + +`send_weights()` drives the whole round — `start_weight_update`, `update_weights`, `finish_weight_update`. Every rank in the communicator must be inside the same reshard, so `update_weights` has to be in flight while the trainer sends; the engine runs it on a helper thread so the trainer loop does not have to open-code that. + +Trainers with a custom producer (a Megatron export, MoE re-fusing) subclass `M2NWeightSource` and supply layouts explicitly. + +See [`examples/rl/rlhf_m2n.py`](../../../examples/rl/rlhf_m2n.py) for a runnable FSDP → TP example. + +## Limitations + +- Tensor rank 1..3; 4-D parameters are rejected. +- dtypes: int8/uint8, fp8 e4m3/e5m2, fp16, bf16, int32/uint32, fp32, int64/uint64, fp64. No fp4. +- The trainer's device mesh must cover a contiguous rank interval starting at 0, and be 1-D or 2-D. +- `Partial` placements are not supported. +- Models with known model-level checkpoint preprocessing use the full-tensor + fallback for the affected module. +- A reshard cannot be issued from inside a CUDA graph capture (weight updates run outside capture, so this only matters for custom callers). +- The reshard plan is capped per *shard*, not per mesh: at most 16 source shards may feed one destination shard, and one source shard may feed at most 64 destination shards (`MAX_SOURCES` / `MAX_TARGETS` in the m2n build; raising either needs a rebuild). +- A directly-resharded parameter spreads over `tp_size` destination shards, so each one is fed by fewer source shards. A parameter on the fallback path is replicated, i.e. a single destination shard fed by every source shard, so a trainer that **sharded** its parameters is still limited to 16 ranks whenever any parameter falls back — which fused parameters always do. A replicated trainer is unaffected. +- Each communicator holds a cached staging pool (~256 MiB by default: 4 channels x 64 MiB). diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 305c21478f..8254a7f420 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -45,6 +45,30 @@ The following metrics are exposed: --8<-- "gen:metrics-nixl" +## Simple CPU Offload Connector Metrics + +These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector +is configured (e.g. `--kv-transfer-config='{"kv_connector": +"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config": +{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are +updated once per engine step. + +Caveats to keep in mind when interpreting them: + +- A "completed" store means the write syscalls returned; it is **not** + fsync-durable, and the disk backend's file is process-lifetime scratch, + unlinked at startup and shutdown. +- `save_outcomes_total` classifies eager-mode boundary hand-off stores; + lazy-mode stores are not classified. +- `used_blocks` counts blocks pinned by in-flight transfers or cache hits; + warm cached blocks that are evictable are not counted. Use the + `capacity_blocks` label of `simple_kv_offload_info` as the denominator. +- Counters and gauges are quantized to engine steps. They are reported by the + scheduler process and reflect engine-wide logical block counts, not values + pooled from individual tensor-parallel workers. + +--8<-- "gen:metrics-simple-kv-offload" + ## Model Flops Utilization (MFU) Performance Metrics These metrics are available via `--enable-mfu-metrics`: diff --git a/docs/usage/security.md b/docs/usage/security.md index 5d4d429c6f..a6c9867755 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -289,6 +289,19 @@ To mitigate this, vLLM enforces a configurable upper bound on the `n` parameter - **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads. - **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse. +### Per-request multimodal arguments + +API server endpoints reject non-empty per-request `mm_processor_kwargs` and +`media_io_kwargs` by default. These arguments can change image, video, or audio +loading, sizing, sampling, and preprocessing behavior, causing excessive CPU, +GPU, or memory use when controlled by an untrusted client. Server-level +`--mm-processor-kwargs` and `--media-io-kwargs` remain available for deployment +configuration. + +Only deployments whose API clients are trusted should start the server with +`--trust-request-mm-kwargs` to restore per-request overrides. Do not enable +this option on an endpoint exposed to untrusted clients. + ## Tool Server and MCP Security vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures. diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index abae7eb65d..4715a15f86 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -480,6 +480,17 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: the grid its processor actually produced), so the grid is never re-derived here -- a second derivation could disagree with the encoder's. """ + mm_items = extract_mm_items(req_data) + # Audio processors may consume placeholders from other modalities too. + # Keep the whole request raw if any item cannot be rewritten safely. + control_fields = {"mm_hash", "ec_mm_hash", "transfer_id"} + raw_audio_request = any( + item["type"] in {"audio_url", "input_audio"} for item in mm_items + ) and any( + not item_meta.get(i, {}).get("mm_hash") + or not (item_meta.get(i, {}).keys() - control_fields) + for i in range(len(mm_items)) + ) rewritten = 0 transfer_items = [] idx = 0 @@ -499,6 +510,18 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: item_uuid = meta.pop("mm_hash", None) ec_mm_hash = meta.pop("ec_mm_hash", None) or item_uuid transfer_id = meta.pop("transfer_id", None) + if raw_audio_request: + new_content.append( + { + **item, + "uuid": item_uuid or item.get("uuid") or content_uuid(item), + } + ) + if transfer_id is not None: + transfer_items.append( + {"mm_hash": ec_mm_hash, "transfer_id": transfer_id} + ) + continue # Whatever keys the encoder reported are the metadata its model # declared as needed to size the placeholder range; the proxy does # not need to know their names. @@ -532,9 +555,10 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: rewritten += 1 new_messages.append({**msg, "content": new_content}) - if not rewritten: + if not rewritten and not raw_audio_request: return req_data - logger.info("Rewrote %d media item(s) as metadata references", rewritten) + if rewritten: + logger.info("Rewrote %d media item(s) as metadata references", rewritten) rewritten_request = {**req_data, "messages": new_messages} if transfer_items: ec_transfer_params = dict(req_data.get("ec_transfer_params") or {}) @@ -632,7 +656,7 @@ async def fanout_encoder_primer( content = [] for idx in indices: item = mm_items[idx] - item_uuid = None if NO_REWRITE else content_uuid(item) + item_uuid = None if NO_REWRITE else (item.get("uuid") or content_uuid(item)) if item_uuid is not None: item_uuids[idx] = item_uuid item_transfer_ids[idx] = uuid.uuid4().hex @@ -756,14 +780,13 @@ async def fanout_encoder_primer( raise HTTPException(502, "Encoder metadata cannot be matched") ec_mm_hash, reported = matched metadata = reported.get("metadata") or {} - if metadata: - item_meta[idx] = { - **metadata, - "mm_hash": item_uuids.get(idx, ec_mm_hash), - "ec_mm_hash": ec_mm_hash, - } - if idx in item_transfer_ids: - item_meta[idx]["transfer_id"] = item_transfer_ids[idx] + item_meta[idx] = { + **metadata, + "mm_hash": item_uuids.get(idx, ec_mm_hash), + "ec_mm_hash": ec_mm_hash, + } + if idx in item_transfer_ids: + item_meta[idx]["transfer_id"] = item_transfer_ids[idx] # Whatever the encoder reported alongside `metadata` is the # connector's own handle on the published embedding (for NIXL, # peer_host/peer_port/size_bytes). The decoder's connector diff --git a/examples/features/logging_configuration.md b/examples/features/logging_configuration.md index dcc11c9add..f2e1e5b8a0 100644 --- a/examples/features/logging_configuration.md +++ b/examples/features/logging_configuration.md @@ -3,46 +3,60 @@ vLLM leverages Python's `logging.config.dictConfig` functionality to enable robust and flexible configuration of the various loggers used by vLLM. -vLLM offers two environment variables that can be used to accommodate a range -of logging configurations that range from simple-and-inflexible to -more-complex-and-more-flexible. +For `vllm serve`, configure logging with CLI arguments: -- No vLLM logging (simple and inflexible) - - Set `VLLM_CONFIGURE_LOGGING=0` (leaving `VLLM_LOGGING_CONFIG_PATH` unset) -- vLLM's default logging configuration (simple and inflexible) - - Leave `VLLM_CONFIGURE_LOGGING` unset or set `VLLM_CONFIGURE_LOGGING=1` -- Fine-grained custom logging configuration (more complex, more flexible) - - Leave `VLLM_CONFIGURE_LOGGING` unset or set `VLLM_CONFIGURE_LOGGING=1` and - set `VLLM_LOGGING_CONFIG_PATH=` +- Use the built-in configuration, optionally with `--log-level`. +- Use a custom Python logging configuration file with + `--logging-config.pylogging_config_file`. +- Disable vLLM logging configuration with + `--logging-config.configure_logging false`. -## Logging Configuration Environment Variables +## CLI logging configuration -### `VLLM_CONFIGURE_LOGGING` +`--logging-config` accepts a JSON object. Its fields are `log_level`, +`configure_logging`, and `pylogging_config_file`. For example: -`VLLM_CONFIGURE_LOGGING` controls whether or not vLLM takes any action to -configure the loggers used by vLLM. This functionality is enabled by default, -but can be disabled by setting `VLLM_CONFIGURE_LOGGING=0` when running vLLM. +```bash +vllm serve mistralai/Mistral-7B-v0.1 \ + --logging-config '{"log_level":"DEBUG","configure_logging":true}' +``` -If `VLLM_CONFIGURE_LOGGING` is enabled and no value is given for -`VLLM_LOGGING_CONFIG_PATH`, vLLM will use built-in default configuration to -configure the root vLLM logger. By default, no other vLLM loggers are -configured and, as such, all vLLM loggers defer to the root vLLM logger to make -all logging decisions. +Fields can also be set individually with dotted arguments: -If `VLLM_CONFIGURE_LOGGING` is disabled and a value is given for -`VLLM_LOGGING_CONFIG_PATH`, an error will occur while starting vLLM. +```bash +vllm serve mistralai/Mistral-7B-v0.1 \ + --logging-config.log_level DEBUG +``` -### `VLLM_LOGGING_CONFIG_PATH` +`--log-level` is a shortcut for `--logging-config.log_level` and takes +precedence if both are supplied. It sets the level of vLLM's built-in logging +configuration. `--log-config-file` is deprecated and will be removed in +v0.33.0; use `--logging-config.pylogging_config_file` in new commands. -`VLLM_LOGGING_CONFIG_PATH` allows users to specify a path to a JSON file of -alternative, custom logging configuration that will be used instead of vLLM's -built-in default logging configuration. The logging configuration should be -provided in JSON format following the schema specified by Python's [logging +If `configure_logging` is `false`, vLLM does not apply a logging +configuration. It cannot be combined with `pylogging_config_file`. This has +the same effect as the legacy `VLLM_CONFIGURE_LOGGING=0` setting. + +The custom logging configuration file must be JSON following Python's [logging configuration dictionary schema](https://docs.python.org/3/library/logging.config.html#dictionary-schema-details). -If `VLLM_LOGGING_CONFIG_PATH` is specified, but `VLLM_CONFIGURE_LOGGING` is -disabled, an error will occur while starting vLLM. +!!! note "Custom configurations override `--log-level`" + When `pylogging_config_file` is set, vLLM loads that JSON file and replaces + its built-in `dictConfig`; it does not merge the two. Therefore, + `--log-level` applies only when no custom configuration file is provided. + Set logger and handler levels in the custom file itself. vLLM applies the + resolved configuration in its child processes as well. + +## Environment variables + +The CLI configuration is recommended for `vllm serve`. The legacy +`VLLM_CONFIGURE_LOGGING`, `VLLM_LOGGING_LEVEL`, and +`VLLM_LOGGING_CONFIG_PATH` variables remain supported as defaults. The default +handler's stream, prefix, and color are currently controlled only through +environment variables. Values from a YAML `--config` file or the command line +override those defaults. See [Environment Variables](https://docs.vllm.ai/en/latest/configuration/env_vars/) +for those settings. ## Examples @@ -61,7 +75,8 @@ To begin, first, create an appropriate JSON logging configuration file: { "formatters": { "json": { - "class": "pythonjsonlogger.jsonlogger.JsonFormatter" + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": "%(asctime)s %(levelname)s %(name)s %(vllm_process_name)s %(process)d %(message)s" } }, "handlers": { @@ -83,14 +98,29 @@ To begin, first, create an appropriate JSON logging configuration file: } ``` -Finally, run vLLM with the `VLLM_LOGGING_CONFIG_PATH` environment variable set -to the path of the custom logging configuration JSON file: +Finally, run vLLM with the custom logging configuration JSON file: ```bash -VLLM_LOGGING_CONFIG_PATH=/path/to/logging_config.json \ - vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 +vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 \ + --logging-config.pylogging_config_file /path/to/logging_config.json ``` +Each vLLM log record is one JSON object and includes `vllm_process_name`, which +identifies vLLM's logical process (including worker ranks where applicable). +The standard `process` record attribute contains the operating-system PID. +vLLM avoids altering `stdout` or `stderr`, so no text is prepended to JSON +log records. + +This applies to records emitted through the configured Python loggers. A JSON +formatter cannot convert unrelated output into JSON, such as a third-party +library writing directly to `stdout` or `stderr`; configure or route such +output separately when a consumer requires every collected line to be JSON. + +When serving, `VLLM_LOGGING_CONFIG_PATH` is also used as Uvicorn's logging +configuration. This example configures only `vllm`; configure `uvicorn`, +`uvicorn.error`, and `uvicorn.access` with a JSON handler when those server +logs must also be structured. + ### Example 2: Silence a particular vLLM logger To silence a particular vLLM logger, it is necessary to provide custom logging @@ -137,26 +167,27 @@ configuration for the root vLLM logger and for the logger you wish to silence: } ``` -Finally, run vLLM with the `VLLM_LOGGING_CONFIG_PATH` environment variable set -to the path of the custom logging configuration JSON file: +Finally, run vLLM with the custom logging configuration JSON file: ```bash -VLLM_LOGGING_CONFIG_PATH=/path/to/logging_config.json \ - vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 +vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 \ + --logging-config.pylogging_config_file /path/to/logging_config.json ``` ### Example 3: Disable vLLM default logging configuration -To disable vLLM's default logging configuration and silence all vLLM loggers, -simple set `VLLM_CONFIGURE_LOGGING=0` when running vLLM. This will prevent vLLM -for configuring the root vLLM logger, which in turn, silences all other vLLM -loggers. +To disable vLLM's default logging configuration and silence vLLM log output, +set `--logging-config.configure_logging false` when running vLLM. This prevents +vLLM from configuring the root vLLM logger, which in turn silences other vLLM +loggers unless the application or Python root logger configures a handler. ```bash -VLLM_CONFIGURE_LOGGING=0 \ - vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 +vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 \ + --logging-config.configure_logging false ``` +For legacy launch scripts, `VLLM_CONFIGURE_LOGGING=0` has the same effect. + ### Example 4: Disable access logs for health check endpoints In production environments, health check endpoints like `/health`, `/metrics`, diff --git a/examples/features/structured_diffusion/README.md b/examples/features/structured_diffusion/README.md new file mode 100644 index 0000000000..37e58dbcdd --- /dev/null +++ b/examples/features/structured_diffusion/README.md @@ -0,0 +1,76 @@ +# Structured reads on DiffusionGemma + +A discrete diffusion model denoises a whole canvas per forward pass. If the +canvas is seeded with the answer's fixed text and only the answer slots are +left as noise, one denoise step gives a distribution over each slot. These +`extra_args` fields (`vllm_xargs` on the OpenAI server) expose that: + +| field | type | meaning | +| --- | --- | --- | +| `diffusion_seed_canvas` | `list[int]`, exactly `canvas_length` ids | replaces the random initial canvas after prefill | +| `diffusion_pinned` | `list[int]` of canvas positions | held at their seed value on every denoise step, so a read past one step keeps its template | +| `diffusion_max_steps` | `int` | denoise steps before the canvas is emitted | +| `diffusion_read_only` | `bool` | emit the argmax canvas as soon as the cap is reached, end the request there, and return temperature-1 logprobs at every position | +| `diffusion_constrained` | `bool` | run the unembedding, sampler and self-conditioning over the request's `logprob_token_ids` only. Logprobs are normalized over that set. A step uses this only when every read in it has the same set | + +`structured_server.py` turns a question schema into those fields, with +`diffusion_constrained` on for every read (`--no-constrained` turns it off). It serves +`/v1/chat/completions`: the system message is the schema, the user message +is the state JSON, and the reply content is one distribution per question +with a standard error over a few noise draws. + +```bash +vllm serve google/diffusiongemma-26B-A4B-it \ + --diffusion-config '{"canvas_length": 64}' --max-logprobs 32 --enable-prefix-caching +python examples/features/structured_diffusion/structured_server.py \ + --upstream http://127.0.0.1:8000 --tokenizer google/diffusiongemma-26B-A4B-it --canvas 64 +curl -s localhost:8011/v1/chat/completions -H 'content-type: application/json' -d '{ + "messages": [ + {"role": "system", "content": "{\"questions\": [{\"id\": \"urgent\", \"type\": \"noul\", \"instructions\": \"Does the customer need a reply within the hour?\"}]}"}, + {"role": "user", "content": "{\"ticket\": \"Everything is down and we have a demo at noon.\"}"} + ]}' +``` + +The attention backend is picked as for Gemma 4: FlashAttention 4 on every +layer when available, otherwise Triton. FlashInfer cannot serve this model (a +batch mixes causal prefill with bidirectional denoising), and +`--attention-backend FLASHINFER` is rejected. + +Per-request canvas widths smaller than the served canvas require async +scheduling. The diffusion async scheduler is selected automatically; no +`--scheduler-cls` argument is needed. Synchronous execution supports full-width +canvases only. + +Question types: `noul` (yes/no), `choice` with `options`, `score` with +ordered `levels`. Each label must be a single token in the answer template, +which the server checks with the tokenizer when a request uses the schema. + +`POST /v1/systemone` implements the Jev decision API. The body holds +`state`, `questions` (a map of id to `type`, `instructions` and `criteria`) +and `model`. Answers come back in that API's shapes: a `noul` probability, +a `choice` with `probabilities` and `confidence`, or a `score` with a +0-indexed `legend`. The schema options above go in the same body as +extensions. + +A question may declare `depends_on` (read in a later stage with those +answers in its prompt), `ask_if` (asked only when a named question's answer +is among the listed ones, otherwise null) and `alone` (a read of its own). + +Images attach as `multipart/form-data`, with the JSON in a part named +`request` and each image as a file part, or as an `images` array of data +URLs. + +```bash +curl -s localhost:8011/v1/systemone -H 'content-type: application/json' -d '{ + "model": "jev-latest", + "state": {"ticket": "Everything is down and we have a demo at noon."}, + "questions": {"urgent": {"type": "noul", "instructions": "Does the customer need a reply within the hour?"}}}' +``` + +`"think": N` in the schema lets the model write up to N tokens in its +thought channel before the read. The thought is an ordinary generation with +the chat template's thinking marker on, and the read then runs with the +thought in its prompt, so the answer slots condition on it. The noise draws +of a decision share one thought. `diagnostics.thought` returns the text, its +length in tokens, whether the model closed the channel itself and the +generation time. diff --git a/examples/features/structured_diffusion/structured_server.py b/examples/features/structured_diffusion/structured_server.py new file mode 100644 index 0000000000..5cb98e4117 --- /dev/null +++ b/examples/features/structured_diffusion/structured_server.py @@ -0,0 +1,1544 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Structured decisions in front of a vLLM DiffusionGemma server. + +POST /v1/systemone takes Jev's request body: {"model", "state", "questions"}. +"questions" maps an id to {"type", "instructions", "criteria"}, where "type" +is "noul", "choice" or "score" and the criteria shape follows the type: + noul: optional {"true": ..., "false": ...} descriptions + choice: option name -> description or null + score: ordered list of levels +Answers take Jev's shapes, with this server's diagnostics alongside: + noul: {"noul": p} + choice: {"choice", "probabilities", "confidence"} + score: {"score", "legend", "probabilities", "confidence"} +The request body may also carry the schema keys "instructions", "samples", +"auto_max", "auto_threshold", "steps", "think", "ask", "chunk_rows", +"chunk_prompt" and "sequential" as extensions. Images go ahead of the +state, either as multipart/form-data with the JSON body in a part named +"request" and each image as a file part, or as an "images" array of data +URLs in the JSON body. + +POST /v1/chat/completions makes the same decision from an OpenAI-shaped +call. The system message is the schema JSON below and the user message is +the state. The reply's `content` is the JSON answer set. + +POST /v1/raw/chat/completions passes the body to vLLM's chat completions +unchanged, for plain generation through this port. + +With API_KEY set in the environment, every POST needs "Authorization: +Bearer ". --tls-port adds an HTTPS listener with a self-signed +certificate kept in --cert-dir, for clients that need a secure origin. + +Each answer is one distribution per question, from one denoise step over a +seeded canvas, averaged over a few noise draws. This server handles the +canvas, tokenizer, slot resolution, noise draws and averaging. + +Schema (system message): + {"questions": [ + {"id": "urgent", "type": "noul", "instructions": "..."}, + {"id": "bucket", "type": "choice", "instructions": "...", + "options": [{"name": "billing", "description": "..."}, ...]}, + {"id": "tone", "type": "score", "instructions": "...", + "levels": ["calm", "annoyed", "furious"]}], + "instructions": "optional context", + "samples": "auto" | N, "auto_threshold": 0.1, "auto_max": 4, + "steps": 1, "think": 0} + +A question may also declare: + "depends_on": [ids] answered after those, with their answers in + its prompt + "ask_if": {id: [answers]} asked only when that question's answer is + among them (a skipped answer is null) + "alone": true a read of its own +Questions run in stages by these dependencies. Each stage is one joint +read. Later stages continue the earlier answers, prefilled for a text +state and restated for an image. + +Up to ten questions answer as "id: label" lines. Past that the id runs +straight into the label, space separated, one row fewer per question. The +server splits a schema whose answer template does not fit the canvas into +chunks that run together, each with its own question list. "chunk_rows" +sets the rows per chunk, "ask" picks a subset of question ids for one +read, "sequential": true runs the chunks in order with the earlier answers +prefilled, and "chunk_prompt": "shared" lists every question in each +chunk's prompt. "think": N lets the model write up to N tokens in its +thought channel, as an ordinary generation, and the read then runs with +that thought in its prompt. With images the model writes the thought with +the image in view and the server seeds it into the canvas ahead of the +answer, so the canvas bounds it. The noise draws of a decision share one +thought. + +Serve the model with a canvas that holds the answer template, for example: + vllm serve google/diffusiongemma-26B-A4B-it \ + --diffusion-config '{"canvas_length": 64}' --max-logprobs 32 \ + --enable-prefix-caching +then run this in front of it: + python structured_server.py --upstream http://127.0.0.1:8000 \ + --tokenizer google/diffusiongemma-26B-A4B-it --canvas 64 --port 8011 +""" + +import argparse +import json +import math +import os +import random +import ssl +import subprocess +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pybase64 as base64 +from transformers import AutoTokenizer + +ARGS = None +TOK = None +API_KEY = os.environ.get( + "API_KEY", "" +) # when set, POST routes need "Authorization: Bearer " +CANVAS_LEN = 64 # the served canvas length. A request may be narrower. +CANVAS_STEP = 16 # request widths are multiples of this +VOCAB = 262144 +TURN_CLOSE = 106 +PAD = 0 +TOPK = 20 +MAX_QUESTIONS = 64 # per request +MAX_SAMPLES = 32 # reads per question, fixed or auto +MAX_PARALLEL = 16 # question groups read at once +# the empty thought block the chat template leaves to the model +SCAFFOLD_TEXT = "<|channel>thought\n" +SCAFFOLD = None +THOUGHT_OPEN = None +THOUGHT_CLOSE = None + + +# ---------------------------------------------------------------------------- +# Schema +# ---------------------------------------------------------------------------- + + +class SchemaError(ValueError): + pass + + +def parse_schema(value): + if ( + not isinstance(value, dict) + or not isinstance(value.get("questions"), list) + or not value["questions"] + ): + raise SchemaError("schema: needs a non-empty questions array") + if len(value["questions"]) > MAX_QUESTIONS: + raise SchemaError(f"schema: at most {MAX_QUESTIONS} questions") + qs = [] + seen = set() + for q in value["questions"]: + qid = str(q.get("id", "")).strip() + if not qid or ":" in qid or "\n" in qid: + raise SchemaError( + f"question id {qid!r} must be non-empty, no ':' or newline" + ) + if qid in seen: + raise SchemaError(f"duplicate question id {qid!r}") + seen.add(qid) + kind = q.get("type") + if kind in ("noul", "bool", "boolean"): + kind = "noul" + crit = q.get("criteria") or {} + choices = [("yes", crit.get("true")), ("no", crit.get("false"))] + labels = ["yes", "no"] + elif kind == "choice": + opts = q.get("options") or [] + choices = [ + (o["name"], o.get("description")) + if isinstance(o, dict) + else (str(o), None) + for o in opts + ] + labels = [chr(ord("A") + i) for i in range(len(choices))] + elif kind == "score": + choices = [(str(level), None) for level in (q.get("levels") or [])] + labels = ( + [str(i + 1) for i in range(len(choices))] + if len(choices) <= 9 + else [chr(ord("A") + i) for i in range(len(choices))] + ) + else: + raise SchemaError(f"question {qid!r}: unknown type {kind!r}") + if len(choices) < 2: + raise SchemaError(f"question {qid!r}: needs at least two alternatives") + if len(choices) > 26: + raise SchemaError(f"question {qid!r}: at most 26 alternatives") + deps = q.get("depends_on") or [] + ask_if = q.get("ask_if") or {} + if not isinstance(deps, list) or not all(isinstance(d, str) for d in deps): + raise SchemaError( + f"question {qid!r}: depends_on must be a list of question ids" + ) + if not isinstance(ask_if, dict) or not all( + isinstance(v, list) and v for v in ask_if.values() + ): + raise SchemaError( + f"question {qid!r}: ask_if must map a question id to a " + "non-empty list of its answers" + ) + qs.append( + { + "id": qid, + "type": kind, + "instructions": str(q.get("instructions", "")), + "choices": choices, + "labels": labels, + "depends_on": list(dict.fromkeys(list(deps) + list(ask_if))), + "ask_if": ask_if, + "alone": bool(q.get("alone", False)), + } + ) + by_id = {q["id"]: q for q in qs} + for q in qs: + for dep in q["depends_on"]: + if dep not in by_id or dep == q["id"]: + raise SchemaError( + f"question {q['id']!r}: depends on unknown question {dep!r}" + ) + for dep, vals in q["ask_if"].items(): + names = [c[0] for c in by_id[dep]["choices"]] + if any(v not in names for v in vals): + raise SchemaError( + f"question {q['id']!r}: ask_if values for {dep!r} " + f"must be among {names}" + ) + schedule(qs) # refuses a cycle + samples = value.get("samples", "auto") + if samples == "auto": + policy = { + "mode": "auto", + "max": max(1, min(int(value.get("auto_max", 4)), MAX_SAMPLES)), + "threshold": float(value.get("auto_threshold", 0.1)), + } + elif isinstance(samples, int) and samples >= 1: + policy = {"mode": "fixed", "n": min(samples, MAX_SAMPLES)} + else: + raise SchemaError('schema: samples must be a positive count or "auto"') + ask = value.get("ask") + if ask is not None: + if not isinstance(ask, list) or not ask or any(a not in seen for a in ask): + raise SchemaError("schema: ask must list question ids from this schema") + for q in qs: + if q["id"] in ask and any(d not in ask for d in q["depends_on"]): + raise SchemaError( + f"schema: ask names {q['id']!r} but not everything it depends on" + ) + chunk_rows = value.get("chunk_rows") + if chunk_rows is not None and (not isinstance(chunk_rows, int) or chunk_rows < 8): + raise SchemaError("schema: chunk_rows must be an integer of at least 8") + chunk_prompt = value.get("chunk_prompt", "own") + if chunk_prompt not in ("shared", "own"): + raise SchemaError('schema: chunk_prompt must be "shared" or "own"') + sequential = bool(value.get("sequential", False)) + think = value.get("think", 0) + if isinstance(think, bool) or not isinstance(think, int) or not 0 <= think <= 4096: + raise SchemaError("schema: think must be a thought budget in tokens, 0 to 4096") + return { + "questions": qs, + "instructions": value.get("instructions"), + "policy": policy, + "steps": max(1, min(int(value.get("steps", 1)), 8)), + "think": think, + "ask": ask, + "chunk_rows": chunk_rows, + "chunk_prompt": chunk_prompt, + "sequential": sequential, + "format": "lines" if len(qs) <= 10 else "indexed", + } + + +# Answer template shape: (join between questions, what precedes the label, +# reply instruction). A small schema gets "lines", which is readable. +# "indexed" ("0yes 1no") costs three tokens a question against four or five +# and agreed with "lines" on every set tried: 42 booleans, ten 26-way +# choices, twenty 5-level scores. Past ten questions the saved rows keep a +# schema in one read. Two tokens a question, or no id at all, loses +# alignment beyond about twenty questions, because the id ties a label to +# its question. +FORMATS = { + "lines": ( + "\n", + "{id}: ", + 'Reply with one line per question, in this order, formatted as "id: label".', + ), + "indexed": ( + " ", + "{id}", + "Reply on one line with each question's id immediately followed by its " + "label, separated by single spaces.", + ), +} + + +def system_text(schema, chunked=False): + s = ( + "Answer a fixed set of questions about the state the user provides. " + "Each question lists its allowed answers; reply with exactly one label " + "per question.\n" + ) + if schema.get("instructions"): + s += "\n" + str(schema["instructions"]).strip() + "\n" + for q in schema["questions"]: + s += f"\nQuestion {q['id']}: {q['instructions'].strip()}\n" + for (name, desc), label in zip(q["choices"], q["labels"]): + if q["type"] == "noul": + s += f" {label}: {str(desc).strip()}\n" if desc else f" {label}\n" + elif desc: + s += f" {label}: {name} ({str(desc).strip()})\n" + else: + s += f" {label}: {name}\n" + s += "\n" + FORMATS[schema.get("format", "lines")][2] + if chunked: + s += ( + " A reply may cover only some of the questions; answer every line " + "that is present." + ) + return s + + +def answer_text(qs, labels, fmt="lines"): + join, lead, _ = FORMATS[fmt] + return join.join( + lead.format(id=q["id"]) + q["labels"][i] for q, i in zip(qs, labels) + ) + + +def enc(text): + return TOK.encode(text, add_special_tokens=False) + + +def init_tokenizer(tok): + global TOK, SCAFFOLD, THOUGHT_OPEN, THOUGHT_CLOSE + TOK = tok + THOUGHT_OPEN = enc("<|channel>thought\n") + THOUGHT_CLOSE = enc("") + SCAFFOLD = enc(SCAFFOLD_TEXT) + assert THOUGHT_OPEN + THOUGHT_CLOSE == SCAFFOLD, ( + "the thought tags must tokenize apart" + ) + + +def resolve_template(qs, head, lead, fmt): + """Tokenize the answer template and find each question's slot. Every label + must change exactly one token, at the same position for all of a question's + labels, or this raises SchemaError. ``head`` is the token run the canvas + starts with: the empty thought block for a plain read, and empty when the + prompt already ends the thought channel. ``lead`` is the text before the + first answer: the join when earlier answers are in the prompt, so the + tokens match one joint template.""" + base_labels = [0] * len(qs) + base = head + enc(lead + answer_text(qs, base_labels, fmt)) + if len(base) + 1 > CANVAS_LEN: + raise SchemaError( + f"answer template is {len(base)} tokens; the canvas holds {CANVAS_LEN - 1}" + ) + if len(qs) == 1 and len(base) + 1 > CANVAS_LEN: + raise SchemaError( + f"question {qs[0]['id']!r} alone needs {len(base) + 1} canvas rows" + ) + slots = [] + for qi, q in enumerate(qs): + pos = None + ids = [0] * len(q["labels"]) + for li in range(1, len(q["labels"])): + labels = list(base_labels) + labels[qi] = li + e = head + enc(lead + answer_text(qs, labels, fmt)) + if len(e) != len(base): + raise SchemaError( + f"question {q['id']!r}: label {q['labels'][li]!r} is not a " + "single token" + ) + diffs = [i for i in range(len(e)) if e[i] != base[i]] + if len(diffs) != 1 or (pos is not None and diffs[0] != pos): + raise SchemaError( + f"question {q['id']!r}: labels do not share one template slot" + ) + pos = diffs[0] + ids[li] = e[pos] + ids[0] = base[pos] + if len(set(ids)) != len(ids): + raise SchemaError( + f"question {q['id']!r}: two labels tokenize to the same id" + ) + slots.append({"pos": pos, "label_ids": ids}) + return base, slots + + +_template_cache = {} + + +def template_for(schema, head, lead): + fmt = schema.get("format", "lines") + key = json.dumps( + [head, lead, fmt] + [(q["id"], q["labels"]) for q in schema["questions"]] + ) + if key not in _template_cache: + _template_cache[key] = resolve_template(schema["questions"], head, lead, fmt) + return _template_cache[key] + + +# ---------------------------------------------------------------------------- +# Reads +# ---------------------------------------------------------------------------- + + +def canvas_width(template): + """Smallest multiple of CANVAS_STEP that holds the template and the turn close.""" + need = len(template) + 1 + return min(CANVAS_LEN, -(-need // CANVAS_STEP) * CANVAS_STEP) + + +def constrained_xargs(): + """Read over the labels only. The engine runs the unembedding, sampler and + self-conditioning over the request's logprob_token_ids instead of the whole + vocabulary. Same argmax, about a quarter less GPU time per read. Logprobs + come back normalized over the labels.""" + return {"diffusion_constrained": True} if ARGS.constrained else {} + + +def pin_xargs(template, slots, steps): + """Past one denoise step the template must be held, or accept/renoise + rewrites it: pin every canvas position that is not an answer slot.""" + if steps <= 1: + return {} + free = {s["pos"] for s in slots} + return { + "diffusion_pinned": [p for p in range(canvas_width(template)) if p not in free] + } + + +def build_canvas(template, slots, seed): + rng = random.Random(seed) + canvas = list(template) + [TURN_CLOSE] + canvas += [PAD] * (canvas_width(template) - len(canvas)) + for s in slots: + canvas[s["pos"]] = rng.randrange(VOCAB) + return canvas + + +def label_id_union(slots): + ids = sorted({i for s in slots for i in s["label_ids"]}) + return ids[:128] # vLLM's cap per request. A schema needs far fewer. + + +def upstream_chat(body, timeout=600): + req = urllib.request.Request( + ARGS.upstream.rstrip("/") + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + return json.load(urllib.request.urlopen(req, timeout=timeout)) + + +def upstream_completions(body, timeout=600): + req = urllib.request.Request( + ARGS.upstream.rstrip("/") + "/v1/completions", + data=json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + return json.load(urllib.request.urlopen(req, timeout=timeout)) + + +def chat_prompt_ids(sys_text, state_text, thinking=False): + """The prompt the chat endpoint would build, as token ids, ending after + the model turn marker. Text states only. ``thinking`` turns the chat + template's thinking marker on.""" + messages = [ + {"role": "system", "content": sys_text}, + {"role": "user", "content": state_text}, + ] + out = TOK.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, enable_thinking=thinking + ) + ids = ( + out["input_ids"] if hasattr(out, "keys") else out + ) # newer transformers return a dict + return [int(t) for t in ids] + + +def think(sys_text, state_text, budget): + """A read prefix that ends with a thought the model wrote: the chat prompt + with thinking on, the open tag, up to ``budget`` generated tokens, the + close tag. Returns the prefix and a diagnostics dict for the thought.""" + prompt = chat_prompt_ids(sys_text, state_text, thinking=True) + THOUGHT_OPEN + started = time.time() + d = upstream_completions( + { + "model": ARGS.model, + "prompt": prompt, + "max_tokens": budget, + "logprobs": 0, + "return_tokens_as_token_ids": True, + "stop_token_ids": THOUGHT_CLOSE, + } + ) + ids = [int(t.split(":")[1]) for t in d["choices"][0]["logprobs"]["tokens"]] + closed = THOUGHT_CLOSE[0] in ids + if closed: + ids = ids[: ids.index(THOUGHT_CLOSE[0])] + info = { + "tokens": len(ids), + "closed": closed, + "ms": (time.time() - started) * 1e3, + "text": TOK.decode(ids), + } + return prompt + ids + THOUGHT_CLOSE, info + + +def think_chat(sys_text, state_content, budget): + """A thought written with the image in view: the chat endpoint with + thinking on, capped at ``budget`` tokens and cut at the close tag. + Returns the thought's token ids and a diagnostics dict.""" + body = { + "model": ARGS.model, + "messages": [ + {"role": "system", "content": sys_text}, + {"role": "user", "content": state_content}, + ], + "max_tokens": budget, + "logprobs": True, + "top_logprobs": 0, + "return_tokens_as_token_ids": True, + "stop_token_ids": THOUGHT_CLOSE, + "chat_template_kwargs": {"enable_thinking": True}, + } + started = time.time() + d = upstream_chat(body) + choice = d["choices"][0] + ids = [ + int(t["token"].split(":")[1]) + for t in (choice.get("logprobs") or {}).get("content") or [] + ] + if ids[: len(THOUGHT_OPEN)] == THOUGHT_OPEN: # the model opens the channel itself + ids = ids[len(THOUGHT_OPEN) :] + closed = THOUGHT_CLOSE[0] in ids + if closed: + ids = ids[: ids.index(THOUGHT_CLOSE[0])] + return ids, { + "tokens": len(ids), + "closed": closed, + "ms": (time.time() - started) * 1e3, + "text": TOK.decode(ids), + } + + +def one_read( + schema, template, slots, sys_text, state_content, seed, prefix=None, thinking=False +): + if prefix is not None: + return one_read_continuation(schema, template, slots, prefix, seed) + messages = [ + {"role": "system", "content": sys_text}, + {"role": "user", "content": state_content}, + ] + body = { + "model": ARGS.model, + "messages": messages, + "max_tokens": len(template) + 1, + "logprobs": True, + "top_logprobs": TOPK, + # Exact logprobs for every label at every position. With a long + # option list most labels never rank in the top-k, and the model's + # mass sits on tokens that spell the option name instead. + "logprob_token_ids": label_id_union(slots), + "return_tokens_as_token_ids": True, + "chat_template_kwargs": {"enable_thinking": thinking}, + "vllm_xargs": { + "diffusion_seed_canvas": build_canvas(template, slots, seed), + "diffusion_canvas_length": canvas_width(template), + "diffusion_max_steps": schema["steps"], + "diffusion_read_only": True, + **pin_xargs(template, slots, schema["steps"]), + **constrained_xargs(), + }, + } + d = upstream_chat(body) + content = d["choices"][0]["logprobs"]["content"] + out = [] + for q, s in zip(schema["questions"], slots): + top = { + int(t["token"].split(":")[1]): t["logprob"] + for t in content[s["pos"]]["top_logprobs"] + } + out.append(slot_distribution(top, s["label_ids"])) + return out, d.get("usage", {}) + + +def slot_distribution(top, label_ids): + """Label probabilities at one slot from the returned logprobs: every + label's own value plus the argmax token. Read-only logprobs are at + temperature 1, so the label softmax uses them directly. The entropy is + over that returned set.""" + floor = min(top.values()) - 5.0 + lp_t = [top.get(i, floor) for i in label_ids] + mx = max(lp_t) + ex = [math.exp(x - mx) for x in lp_t] + probs = [e / sum(ex) for e in ex] + top_p = [math.exp(v) for v in top.values()] + return { + "probs": probs, + "label_mass": sum(math.exp(x) for x in lp_t), + "entropy": -sum(p * math.log(p) for p in top_p if p > 0), + "argmax_is_label": max(top, key=top.get) in label_ids, + } + + +def one_read_continuation(schema, template, slots, prompt_ids, seed): + """A read whose prompt already holds the thought scaffold and earlier + answer lines, sent as token ids so the chat template cannot alter it.""" + body = { + "model": ARGS.model, + "prompt": prompt_ids, + "max_tokens": len(template) + 1, + "logprobs": TOPK, + "logprob_token_ids": label_id_union(slots), + "return_tokens_as_token_ids": True, + "vllm_xargs": { + "diffusion_seed_canvas": build_canvas(template, slots, seed), + "diffusion_canvas_length": canvas_width(template), + "diffusion_max_steps": schema["steps"], + "diffusion_read_only": True, + **pin_xargs(template, slots, schema["steps"]), + **constrained_xargs(), + }, + } + d = upstream_completions(body) + rows = d["choices"][0]["logprobs"]["top_logprobs"] + out = [] + for q, sl in zip(schema["questions"], slots): + top = {int(k.split(":")[1]): v for k, v in rows[sl["pos"]].items()} + out.append(slot_distribution(top, sl["label_ids"])) + return out, d.get("usage", {}) + + +def read_many( + schema, + template, + slots, + sys_text, + state_content, + seed, + n, + prefix=None, + thinking=False, +): + results = [None] * n + errors = [None] * n + usages = [None] * n + + def run(k): + try: + results[k], usages[k] = one_read( + schema, + template, + slots, + sys_text, + state_content, + seed + k * 7919, + prefix, + thinking, + ) + except Exception as e: # raised again below + errors[k] = e + + threads = [threading.Thread(target=run, args=(k,)) for k in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + for e in errors: + if e is not None: + raise e + return results, usages + + +def schedule(qs): + """Questions in stages: a question's stage comes after the stages of + everything it depends on. Declaration order is kept within a stage.""" + ids = {q["id"] for q in qs} + pending = list(qs) + done: set = set() + levels = [] + while pending: + level = [ + q + for q in pending + if all(d in done or d not in ids for d in q["depends_on"]) + ] + if not level: + raise SchemaError( + "schema: dependency cycle among " + ", ".join(q["id"] for q in pending) + ) + levels.append(level) + done |= {q["id"] for q in level} + pending = [q for q in pending if q["id"] not in done] + return levels + + +def chunk_groups(schema, qs): + """``qs`` split, in order, into the fewest groups whose answer templates + fit ``chunk_rows`` (the canvas by default). A question marked alone gets + its own group.""" + limit = schema.get("chunk_rows") or CANVAS_LEN + groups, group = [], [] + for q in qs: + if q["alone"]: + if group: + groups.append(group) + group = [] + groups.append([q]) + continue + trial = group + [q] + rows = ( + len(SCAFFOLD) + + len( + enc(answer_text(trial, [0] * len(trial), schema.get("format", "lines"))) + ) + + 1 + ) + if rows > limit and group: + groups.append(group) + group = [q] + else: + group = trial + if group: + groups.append(group) + return groups + + +def answer_name(q, a): + """The answer as the name ask_if compares against: yes or no, an option + name, or a level name.""" + if a is None: + return None + return a["label"] if q["type"] == "noul" else a.get("choice", a.get("level")) + + +def decide(schema, state_content, seed): + """One decision. Questions run in stages by their dependencies. A stage + is one joint read, chunked by the canvas, with a question marked alone + in its own read. Later stages condition on every earlier answer: a + prefilled continuation for a text state, or the answers restated in the + state for an image. A question whose ask_if condition failed is skipped + and its answer is null.""" + started = time.time() + qs = [ + q + for q in schema["questions"] + if not schema.get("ask") or q["id"] in schema["ask"] + ] + levels = schedule(qs) + text_state = isinstance(state_content, str) + fmt = schema["format"] + join = FORMATS[fmt][0] + # More than one read in sequence needs the full question list in every + # prompt, so that later reads continue one answer. + chained = len(levels) > 1 or schema["sequential"] + sys_full = system_text(schema, chunked=not text_state and chained) + base_ids, thought = None, None + if chained and text_state: + if schema["think"]: + base_ids, thought = think(sys_full, state_content, schema["think"]) + else: + base_ids = chat_prompt_ids(sys_full, state_content) + SCAFFOLD + shared = schema["chunk_prompt"] == "shared" + answered, lines, earlier = {}, [], [] + parts, stages, chunks, skipped = [], [], [], {} + by_id = {q["id"]: q for q in schema["questions"]} + + def run(group, k, conditioned): + sub = dict(schema, questions=group) + # The thought is written once: in the prefix of a chained text decision, + # or in the first read of anything else. + sub["think"] = 0 if (chained and text_state) or conditioned else schema["think"] + if text_state: + if conditioned: + prefix, lead, sys_text = ( + base_ids + enc(join.join(lines)), + join, + sys_full, + ) + elif chained: + prefix, lead, sys_text = (base_ids if thought else None), "", sys_full + else: + prefix, lead, sys_text = ( + None, + "", + (system_text(schema, chunked=True) if shared else system_text(sub)), + ) + return decide_group( + sub, sys_text, state_content, seed + 104729 * k, prefix, lead + ) + state = state_content + if conditioned: + text = next( + (p["text"] for p in state_content if p.get("type") == "text"), "" + ) + text += "\n\nAnswers so far:\n" + "\n".join(earlier) + state = [p for p in state_content if p.get("type") != "text"] + [ + {"type": "text", "text": text} + ] + sys_text = sys_full + else: + sys_text = sys_full if (chained or shared) else system_text(sub) + return decide_group(sub, sys_text, state, seed + 104729 * k) + + def absorb(group, body, rows): + parts.append((body, rows)) + chunks.append([q["id"] for q in group]) + answered.update(body["answers"]) + lines.append( + answer_text( + group, + [q["labels"].index(body["answers"][q["id"]]["label"]) for q in group], + fmt, + ) + ) + earlier.extend( + f"{q['id']}: {answer_name(q, body['answers'][q['id']])}" for q in group + ) + + k = 0 + for level in levels: + asked = [] + for q in level: + failed = next( + ( + (dep, vals) + for dep, vals in q["ask_if"].items() + if answer_name(by_id[dep], answered.get(dep)) not in vals + ), + None, + ) + if failed: + answered[q["id"]] = None + skipped[q["id"]] = { + "because": failed[0], + "was": answer_name(by_id[failed[0]], answered.get(failed[0])), + "wanted": failed[1], + } + continue + asked.append(q) + if not asked: + continue + stages.append([q["id"] for q in asked]) + groups = chunk_groups(schema, asked) + conditioned = bool(lines) + if schema["sequential"] or len(groups) == 1: + for group in groups: + body, rows = run( + group, k, conditioned or (schema["sequential"] and bool(lines)) + ) + absorb(group, body, rows) + k += 1 + else: + with ThreadPoolExecutor(max_workers=min(len(groups), MAX_PARALLEL)) as ex: + results = list( + ex.map( + lambda gk, conditioned=conditioned: run( + gk[1], gk[0], conditioned + ), + [(k + i, g) for i, g in enumerate(groups)], + ) + ) + for group, (body, rows) in zip(groups, results): + absorb(group, body, rows) + k += len(groups) + + answers = {q["id"]: answered.get(q["id"]) for q in qs} + diag_q = {} + for body, _ in parts: + diag_q.update(body["diagnostics"]["questions"]) + # A thought written here (a chained text decision) is outside every + # group's row count. One written inside a group is already counted there. + extra_rows = thought["tokens"] if thought else 0 + if thought is None: + thoughts = [b["diagnostics"].get("thought") for b, _ in parts] + thought = ( + thoughts[0] if len(parts) == 1 else ([t for t in thoughts if t] or None) + ) + one = len(parts) == 1 and not skipped + diagnostics = { + "steps": schema["steps"], + "stages": stages, + "skipped": skipped, + "chunks": chunks, + "chunk_prompt": "full" if chained else schema["chunk_prompt"], + "sequential": schema["sequential"], + "conditioning": ( + None + if len(stages) <= 1 and not schema["sequential"] + else ("prefill" if text_state else "restated") + ), + "thought": thought, + "samples": ( + parts[0][0]["diagnostics"]["samples"] + if one + else { + "n": [b["diagnostics"]["samples"]["n"] for b, _ in parts], + "tops": [b["diagnostics"]["samples"]["tops"] for b, _ in parts], + "policy": [b["diagnostics"]["samples"]["policy"] for b, _ in parts], + } + ), + "timing": { + "total_ms": (time.time() - started) * 1e3, + "reads": sum(b["diagnostics"]["timing"]["reads"] for b, _ in parts), + }, + "prompt_tokens": max( + (b["diagnostics"].get("prompt_tokens") or 0) for b, _ in parts + ) + or None, + "questions": diag_q, + "engine": "vllm", + } + return {"answers": answers, "diagnostics": diagnostics}, sum( + rows for _, rows in parts + ) + extra_rows + + +def decide_group(schema, sys_text, state_content, seed, prefix=None, lead=""): + started = time.time() + thought = None + head = SCAFFOLD if prefix is None else [] + thinking = False + if prefix is None and schema["think"]: + if isinstance(state_content, str): + prefix, thought = think(sys_text, state_content, schema["think"]) + head = [] + else: + # With images the model writes the thought with the image in view, and + # the server seeds it into the canvas ahead of the answer, so the read + # keeps the image. The canvas bounds the thought. + answer = enc( + answer_text( + schema["questions"], + [0] * len(schema["questions"]), + schema.get("format", "lines"), + ) + ) + fits = CANVAS_LEN - 1 - len(THOUGHT_OPEN) - len(THOUGHT_CLOSE) - len(answer) + if fits < 8: + raise SchemaError( + f"think: the canvas leaves {fits} rows for a thought " + "beside this template" + ) + ids, thought = think_chat( + sys_text, state_content, min(schema["think"], fits) + ) + thought["budget"] = min(schema["think"], fits) + head = THOUGHT_OPEN + ids + THOUGHT_CLOSE + thinking = True + template, slots = template_for(schema, head, lead) + policy = schema["policy"] + if policy["mode"] == "fixed": + reads, usages = read_many( + schema, + template, + slots, + sys_text, + state_content, + seed, + policy["n"], + prefix, + thinking, + ) + extended = None + first_entropy = None + else: + reads, usages = read_many( + schema, template, slots, sys_text, state_content, seed, 1, prefix, thinking + ) + first_entropy = { + q["id"]: r["entropy"] for q, r in zip(schema["questions"], reads[0]) + } + extended = ( + max(first_entropy.values()) > policy["threshold"] and policy["max"] > 1 + ) + if extended: + more, more_usages = read_many( + schema, + template, + slots, + sys_text, + state_content, + seed + 1, + policy["max"] - 1, + prefix, + thinking, + ) + reads += more + usages += more_usages + prompt_tokens = next( + (u["prompt_tokens"] for u in usages if u and u.get("prompt_tokens")), None + ) + elapsed_ms = (time.time() - started) * 1e3 + + answers = {} + diag_q = {} + n = len(reads) + for qi, q in enumerate(schema["questions"]): + per = [r[qi]["probs"] for r in reads] + mean = [sum(p[i] for p in per) / n for i in range(len(q["labels"]))] + top = max(range(len(mean)), key=lambda i: mean[i]) + a = { + "type": q["type"], + "label": q["labels"][top], + "confidence": mean[top], + "probabilities": {c[0]: m for c, m in zip(q["choices"], mean)}, + } + if q["type"] == "noul": + a["noul"] = mean[0] + elif q["type"] == "choice": + a["choice"] = q["choices"][top][0] + else: + a["score"] = sum((i + 1) * m for i, m in enumerate(mean)) + a["level"] = q["choices"][top][0] + if n > 1: + var = sum((p[top] - mean[top]) ** 2 for p in per) / (n - 1) + a["stderr"] = (var / n) ** 0.5 + a["agreement"] = ( + sum(1 for p in per if max(range(len(p)), key=lambda i: p[i]) == top) / n + ) + answers[q["id"]] = a + diag_q[q["id"]] = { + "pos": slots[qi]["pos"], + "entropy": [r[qi]["entropy"] for r in reads], + "label_mass": reads[0][qi]["label_mass"], + "argmax_is_label": reads[0][qi]["argmax_is_label"], + } + tops = [ + { + q["id"]: [ + q["labels"][ + max(range(len(r[qi]["probs"])), key=lambda i: r[qi]["probs"][i]) + ], + max(r[qi]["probs"]), + r[qi]["entropy"], + ] + for qi, q in enumerate(schema["questions"]) + } + for r in reads + ] + return { + "answers": answers, + "diagnostics": { + "steps": schema["steps"], + "samples": { + "n": n, + "tops": tops, + "policy": dict( + policy, extended=extended, first_read_entropy=first_entropy + ), + }, + "timing": {"total_ms": elapsed_ms, "reads": n}, + "thought": thought, + "prompt_tokens": prompt_tokens, + "questions": diag_q, + "engine": "vllm", + }, + }, len(template) + 1 + (thought["tokens"] if thought else 0) + + +# ---------------------------------------------------------------------------- +# Jev's contract +# ---------------------------------------------------------------------------- + +JEV_EXTENSIONS = ( + "instructions", + "samples", + "auto_max", + "auto_threshold", + "steps", + "think", + "ask", + "chunk_rows", + "chunk_prompt", + "sequential", +) + + +def jev_schema(body): + """This server's schema from a Jev request body.""" + qs = body.get("questions") + if not isinstance(qs, dict) or not qs: + raise SchemaError("questions: needs a non-empty map of id -> question") + out = [] + for qid, q in qs.items(): + if not isinstance(q, dict): + raise SchemaError(f"question {qid!r}: must be an object") + kind, crit, ins = q.get("type"), q.get("criteria"), q.get("instructions", "") + item = { + "id": qid, + "type": kind, + "instructions": ins if isinstance(ins, str) else json.dumps(ins), + } + if kind == "noul": + if crit is not None and not isinstance(crit, dict): + raise SchemaError( + f"question {qid!r}: noul criteria must be an object with " + "true and false" + ) + item["criteria"] = crit + elif kind == "choice": + if not isinstance(crit, dict) or not crit: + raise SchemaError( + f"question {qid!r}: choice criteria must map option names " + "to descriptions" + ) + item["options"] = [ + {"name": str(n), "description": d} for n, d in crit.items() + ] + elif kind == "score": + if not isinstance(crit, list): + raise SchemaError( + f"question {qid!r}: score criteria must be an ordered list " + "of levels" + ) + item["levels"] = crit + else: + raise SchemaError(f"question {qid!r}: unknown type {kind!r}") + for key in ("depends_on", "ask_if", "alone"): + if key in q: + item[key] = q[key] + out.append(item) + schema = {k: body[k] for k in JEV_EXTENSIONS if k in body} + schema["questions"] = out + return parse_schema(schema) + + +def image_part(content_type, data): + return { + "type": "image_url", + "image_url": { + "url": f"data:{content_type};base64," + base64.b64encode(data).decode() + }, + } + + +def jev_images(value): + """Image parts from the body's "images": data URLs, or objects with + content_type and base64.""" + parts = [] + for i, im in enumerate(value or []): + if isinstance(im, str) and im.startswith("data:image/"): + parts.append({"type": "image_url", "image_url": {"url": im}}) + elif ( + isinstance(im, dict) + and str(im.get("content_type", "")).startswith("image/") + and isinstance(im.get("base64"), str) + ): + parts.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{im['content_type']};base64,{im['base64']}" + }, + } + ) + else: + raise SchemaError( + f"images[{i}]: a data:image/... URL or an object with " + "content_type and base64" + ) + return parts + + +def jev_state(body, image_parts=()): + """The user message: the state as text (as given, or as JSON), with any + images ahead of it.""" + state = body.get("state") + if state is None: + raise SchemaError("state: required") + text = state if isinstance(state, str) else json.dumps(state) + if not image_parts: + return text + return list(image_parts) + [{"type": "text", "text": text}] + + +def jev_answer(q, a): + if a is None: + return None + if q["type"] == "noul": + return {"type": "noul", "noul": a["noul"]} + if q["type"] == "choice": + return { + "type": "choice", + "choice": a["choice"], + "probabilities": a["probabilities"], + "confidence": a["confidence"], + } + names = [c[0] for c in q["choices"]] + probs = {str(i): a["probabilities"][n] for i, n in enumerate(names)} + return { + "type": "score", + "score": sum(i * p for i, p in enumerate(probs.values())), + "legend": {str(i): n for i, n in enumerate(names)}, + "probabilities": probs, + "confidence": a["confidence"], + } + + +# ---------------------------------------------------------------------------- +# HTTP +# ---------------------------------------------------------------------------- + + +def message_text(m): + c = m.get("content", "") + if isinstance(c, list): + return "".join(p.get("text", "") for p in c if isinstance(p, dict)) + return c if isinstance(c, str) else "" + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + pass + + def _json(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path == "/health": + return self._json(200, {"status": "ok"}) + return self._json(404, {"error": {"message": "unknown route"}}) + + def _read_request(self): + """-> (body, image parts): a JSON body, or multipart/form-data with the + JSON in a part named request and each image as a file part, in order.""" + raw = self.rfile.read(int(self.headers.get("content-length", "0"))) + ctype = self.headers.get("content-type", "") + if not ctype.lower().startswith("multipart/form-data"): + return json.loads(raw), [] + from email.parser import BytesParser + from email.policy import HTTP + + msg = BytesParser(policy=HTTP).parsebytes( + b"Content-Type: " + ctype.encode() + b"\r\n\r\n" + raw + ) + body, images = None, [] + for part in msg.iter_parts(): + name = part.get_param("name", header="content-disposition") + data = part.get_payload(decode=True) + if name == "request": + body = json.loads(data) + elif part.get_content_type().startswith("image/"): + images.append(image_part(part.get_content_type(), data)) + else: + raise ValueError( + f"part {name!r}: neither the request JSON nor an image" + ) + if body is None: + raise ValueError( + "multipart needs a part named request holding the JSON body" + ) + return body, images + + def do_POST(self): + if API_KEY and self.headers.get("authorization", "") != f"Bearer {API_KEY}": + return self._json( + 401, + { + "error": { + "message": "missing or wrong bearer token", + "type": "authentication_error", + } + }, + ) + if self.path == "/v1/raw/chat/completions": + return self._raw_chat() + try: + req, images = self._read_request() + except Exception as e: + return self._json( + 400, + { + "error": { + "message": f"invalid body: {e}", + "type": "invalid_request_error", + } + }, + ) + if self.path == "/v1/systemone": + return self._systemone(req, images) + if self.path == "/v1/chat/completions": + return self._chat(req) + return self._json(404, {"error": {"message": "unknown route"}}) + + def _raw_chat(self): + """Pass the body and status through to vLLM's chat completions.""" + raw = self.rfile.read(int(self.headers.get("content-length", "0"))) + req = urllib.request.Request( + ARGS.upstream.rstrip("/") + "/v1/chat/completions", + data=raw, + headers={ + "content-type": self.headers.get("content-type", "application/json") + }, + ) + try: + with urllib.request.urlopen(req, timeout=600) as r: + code, body = r.status, r.read() + except urllib.error.HTTPError as e: + code, body = e.code, e.read() + self.send_response(code) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _decide(self, schema, state, seed): + """-> (status, body) with the error body already shaped.""" + try: + return 200, decide(schema, state, seed) + except SchemaError as e: + return 422, {"error": {"message": str(e), "type": "validation_error"}} + except urllib.error.HTTPError as e: + return 502, { + "error": { + "message": ( + f"upstream {e.code}: {e.read()[:300].decode(errors='replace')}" + ), + "type": "server_error", + } + } + except Exception as e: + return 500, {"error": {"message": repr(e), "type": "server_error"}} + + def _systemone(self, req, images): + try: + schema = jev_schema(req) + state = jev_state(req, images + jev_images(req.get("images"))) + except SchemaError as e: + return self._json( + 422, {"error": {"message": str(e), "type": "validation_error"}} + ) + code, result = self._decide(schema, state, int(req.get("seed", 42))) + if code != 200: + return self._json(code, result) + body, completion_tokens = result + answers = { + q["id"]: jev_answer(q, body["answers"][q["id"]]) + for q in schema["questions"] + } + labels = " ".join( + f"{k}={v['label'] if v else 'skipped'}" for k, v in body["answers"].items() + ) + print( + f"systemone: {labels} " + f"reads={body['diagnostics']['timing']['reads']} " + f"{body['diagnostics']['timing']['total_ms']:.0f}ms", + flush=True, + ) + self._json( + 200, + { + "model": ARGS.model, + "answers": answers, + # vLLM's prompt count when the reads reported one, since it covers + # images. Otherwise the tokenizer's count of the text prompt. + "usage": { + "input_tokens": body["diagnostics"].get("prompt_tokens") + or ( + len(chat_prompt_ids(system_text(schema), state)) + if isinstance(state, str) + else 0 + ), + "output_tokens": completion_tokens, + }, + "diagnostics": body["diagnostics"], + }, + ) + + def _chat(self, req): + msgs = req.get("messages") or [] + if ( + len(msgs) != 2 + or msgs[0].get("role") not in ("system", "developer") + or msgs[1].get("role") != "user" + ): + return self._json( + 400, + { + "error": { + "message": ( + "a structured request is exactly two messages: the " + "schema (system) and the state JSON (user)" + ), + "type": "invalid_request_error", + } + }, + ) + try: + schema_value = json.loads(message_text(msgs[0])) + schema = parse_schema(schema_value) + content = msgs[1].get("content", "") + has_image = isinstance(content, list) and any( + isinstance(p, dict) and p.get("type") in ("image_url", "image") + for p in content + ) + if has_image: + # image parts pass through to vLLM unchanged, with text parts as context + state = content + else: + state = message_text(msgs[1]).strip() + json.loads(state) + except SchemaError as e: + return self._json( + 400, {"error": {"message": str(e), "type": "invalid_request_error"}} + ) + except Exception as e: + return self._json( + 400, + { + "error": { + "message": ( + "system must be a JSON question schema and user " + f"must be JSON state or image parts: {e}" + ), + "type": "invalid_request_error", + } + }, + ) + code, result = self._decide(schema, state, int(req.get("seed", 42))) + if code != 200: + if code == 422: + result["error"]["type"] = "invalid_request_error" + code = 400 + return self._json(code, result) + body, completion_tokens = result + content = json.dumps(body, indent=2) + labels = " ".join( + f"{k}={v['label'] if v else 'skipped'}" for k, v in body["answers"].items() + ) + print( + f"structured: {labels} " + f"reads={body['diagnostics']['timing']['reads']} " + f"{body['diagnostics']['timing']['total_ms']:.0f}ms", + flush=True, + ) + self._json( + 200, + { + "id": f"chatcmpl-{int(time.time() * 1000)}", + "object": "chat.completion", + "created": int(time.time()), + "model": req.get("model", "dgemma-structured"), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 0, + "completion_tokens": completion_tokens, + "total_tokens": completion_tokens, + }, + }, + ) + + +def self_signed(cert_dir): + """Paths of a self-signed certificate and key in cert_dir, made with + openssl on first use.""" + os.makedirs(cert_dir, exist_ok=True) + cert, key = os.path.join(cert_dir, "djev.crt"), os.path.join(cert_dir, "djev.key") + if not (os.path.exists(cert) and os.path.exists(key)): + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "3650", + "-subj", + "/CN=djev", + "-keyout", + key, + "-out", + cert, + ], + check=True, + capture_output=True, + ) + return cert, key + + +def serve_tls(host, port, cert_dir): + cert, key = self_signed(cert_dir) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert, key) + srv = ThreadingHTTPServer((host, port), Handler) + srv.socket = ctx.wrap_socket(srv.socket, server_side=True) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def main(): + global ARGS, CANVAS_LEN, CANVAS_STEP + p = argparse.ArgumentParser() + p.add_argument("--upstream", default="http://127.0.0.1:8010") + p.add_argument("--model", default="dgemma") + p.add_argument("--tokenizer", default="/models/dgemma", help="HF id or local path") + p.add_argument("--canvas", type=int, default=64, help="the served canvas length") + p.add_argument( + "--canvas-step", + type=int, + default=16, + help="request widths round up to a multiple of this", + ) + p.add_argument( + "--no-constrained", + dest="constrained", + action="store_false", + help="read over the whole vocabulary instead of the labels", + ) + p.add_argument("--host", default="0.0.0.0") + p.add_argument("--port", type=int, default=8011) + p.add_argument( + "--tls-port", type=int, default=0, help="also listen with HTTPS here (0 = off)" + ) + p.add_argument( + "--cert-dir", + default=os.path.expanduser("~/.cache/djev"), + help="directory for the self-signed certificate", + ) + ARGS = p.parse_args() + CANVAS_LEN = ARGS.canvas + CANVAS_STEP = ARGS.canvas_step + init_tokenizer(AutoTokenizer.from_pretrained(ARGS.tokenizer)) + if ARGS.tls_port: + serve_tls(ARGS.host, ARGS.tls_port, ARGS.cert_dir) + print( + f"structured server https on {ARGS.host}:{ARGS.tls_port} (self-signed)", + flush=True, + ) + print( + f"structured server on {ARGS.host}:{ARGS.port} -> {ARGS.upstream} " + f"(canvas {CANVAS_LEN})", + flush=True, + ) + ThreadingHTTPServer((ARGS.host, ARGS.port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index cccb75d838..7565f1e42b 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -55,7 +55,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData: model_name = "rhymes-ai/Aria" engine_args = EngineArgs( model=model_name, - tokenizer_mode="slow", trust_remote_code=True, dtype="bfloat16", limit_mm_per_prompt={"image": len(image_urls)}, diff --git a/examples/rl/rlhf_m2n.py b/examples/rl/rlhf_m2n.py new file mode 100644 index 0000000000..e02229ed09 --- /dev/null +++ b/examples/rl/rlhf_m2n.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""RLHF with FSDP2 training and vLLM tensor-parallel inference using **NCCL M2N** +sharding-aware weight transfer. + +Layout (4 GPUs, no colocation): + * GPUs 0-1: two FSDP2 training workers, one per GPU. + * GPUs 2-3: one vLLM ``LLM`` actor with ``tensor_parallel_size=2``. + +The trainer and the inference workers share one NCCL communicator of 4 ranks: +trainer ranks ``[0, 2)`` and inference ranks ``[2, 4)``. Every parameter moves +with a single ``reshard`` that redistributes it from the FSDP layout to the +inference layout — the trainer sends its local shards and never all-gathers a +full tensor, which broadcast-based weight sync would force it to do. + +Every FSDP rank builds an ``M2NTrainerWeightTransferEngine`` and calls +``send_weights()``; all ranks run every reshard, and only rank 0 drives the +inference side through its ``RayVLLMWeightSyncClient``. + +Requires the ``nccl-extensions`` package (NCCL 2.30.5+) and a +``VLLM_NCCL_SO_PATH`` pointing at the same ``libnccl.so`` that +``libnccl_m2n.so`` was linked against. + +This example was written for 4xH100. +""" + +from __future__ import annotations + +import os + +import pybase64 as base64 +import ray +import torch +import torch.distributed as dist +from huggingface_hub import snapshot_download +from torch.distributed.fsdp import fully_shard +from transformers import AutoModelForCausalLM + +from vllm import LLM, SamplingParams +from vllm.config import WeightTransferConfig +from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary +from vllm.distributed.weight_transfer import ( + RayVLLMWeightSyncClient, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.m2n_source import DTensorModuleSource +from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo +from vllm.utils.network_utils import get_ip, get_open_port + +MODEL_NAME = "Qwen/Qwen3-0.6B" + +FSDP_WORLD_SIZE = 2 +INFERENCE_TP_SIZE = 2 + + +class MyLLM(LLM): + """LLM subclass that keeps Ray from pinning it to a single device.""" + + def __init__(self, *args, **kwargs): + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + super().__init__(*args, **kwargs) + + def ready(self): + return True + + +@ray.remote(num_cpus=1, num_gpus=1) +class FSDPTrainWorker: + """One FSDP2 worker per GPU. Rank 0 is the weight-transfer sender.""" + + def __init__( + self, + model_name: str, + rank: int, + fsdp_master_addr: str, + fsdp_master_port: int, + ): + self.rank = rank + + os.environ["MASTER_ADDR"] = fsdp_master_addr + os.environ["MASTER_PORT"] = str(fsdp_master_port) + + dist.init_process_group(backend="nccl", rank=rank, world_size=FSDP_WORLD_SIZE) + torch.accelerator.set_device_index(0) + + model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=torch.bfloat16 + ).cuda() + for layer in model.model.layers: + fully_shard(layer) + fully_shard(model) + self.model = model + + def ready(self): + return True + + def setup_engine(self, llm_handle, nccl_unique_id_b64, world_size, num_workers): + """Build the trainer engine on every FSDP rank. + + `DTensorModuleSource` reads each parameter's FSDP device mesh and + placements, so the engine knows the source layout without gathering + anything. Rank 0 additionally drives the inference-side handshake. + """ + self.engine = WeightTransferTrainerFactory.trainer_init( + init_info=M2NTrainerInitInfo( + nccl_unique_id_b64=nccl_unique_id_b64, + world_size=world_size, + num_trainer_ranks=FSDP_WORLD_SIZE, + # One DP group of TP=2 workers; declared so both sides + # describe the destination the same way. + dst_mesh_dims=(num_workers // INFERENCE_TP_SIZE, INFERENCE_TP_SIZE), + rank=self.rank, # FSDP rank; sender is 0 + ), + client=RayVLLMWeightSyncClient(llm_handle), + source=DTensorModuleSource(self.model, FSDP_WORLD_SIZE), + ) + + def send_weights(self): + """Called on all ranks concurrently; every rank runs every reshard.""" + self.engine.send_weights() + + +def main(): + ray.init() + + local_model_path = snapshot_download(MODEL_NAME) + print(f"[init] Model downloaded to {local_model_path}") + + fsdp_master_addr = get_ip() + fsdp_master_port = get_open_port() + + fsdp_workers = [ + FSDPTrainWorker.remote( + local_model_path, rank, fsdp_master_addr, fsdp_master_port + ) + for rank in range(FSDP_WORLD_SIZE) + ] + ray.get([w.ready.remote() for w in fsdp_workers]) + print(f"[init] {FSDP_WORLD_SIZE} FSDP workers ready.") + + llm = ray.remote(num_cpus=0, num_gpus=0)(MyLLM).remote( + model=local_model_path, + enforce_eager=True, + tensor_parallel_size=INFERENCE_TP_SIZE, + distributed_executor_backend="ray", + weight_transfer_config=WeightTransferConfig(backend="nccl_m2n"), + load_format="dummy", + ) + ray.get(llm.ready.remote()) + num_workers = ray.get(llm.get_world_size.remote()) + print(f"[init] vLLM ready with {num_workers} inference workers.") + + prompts = [ + "Hello, my name is", + "The capital of France is", + ] + sampling_params = SamplingParams(temperature=0) + + outputs = ray.get(llm.generate.remote(prompts, sampling_params)) + print("-" * 60) + print("BEFORE weight sync (dummy weights):") + for output in outputs: + print(f"Prompt: {output.prompt!r}") + print(f"Generated: {output.outputs[0].text!r}") + print("-" * 60) + + # Trainer ranks [0, FSDP_WORLD_SIZE) and inference ranks after them share + # one communicator, so the two meshes are contiguous rank intervals. + nccl = NCCLLibrary() + nccl_unique_id_b64 = base64.b64encode( + bytes(nccl.ncclGetUniqueId().internal) + ).decode() + world_size = FSDP_WORLD_SIZE + num_workers + + print("[transfer] Initializing nccl_m2n weight transfer (all FSDP ranks)...") + ray.get( + [ + w.setup_engine.remote(llm, nccl_unique_id_b64, world_size, num_workers) + for w in fsdp_workers + ] + ) + + print("[sync] Resharding FSDP -> vLLM...") + ray.get([w.send_weights.remote() for w in fsdp_workers]) + print("[sync] Weight transfer complete.") + + outputs_updated = ray.get(llm.generate.remote(prompts, sampling_params)) + print("-" * 60) + print("AFTER weight sync (real weights):") + for output in outputs_updated: + print(f"Prompt: {output.prompt!r}") + print(f"Generated: {output.outputs[0].text!r}") + print("-" * 60) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index b5c03b22b7..4a7c018929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,54 @@ markers = [ "optional: optional tests that are automatically skipped, include --optional to run them", ] +[tool.coverage.run] +# Track the installed vllm package (this is what actually gets imported during tests) +# Use wildcard pattern to match the installed location +source = ["vllm", "*/dist-packages/vllm", "*/site-packages/vllm"] +omit = [ + "*/tests/*", + "*/test_*", + "*/__pycache__/*", + "*/build/*", + "*/dist/*", + "*/vllm.egg-info/*", + "*/third_party/*", + "*/examples/*", + "*/benchmarks/*", + "*/docs/*", +] + +[tool.coverage.paths] +# Map all possible vllm locations to a canonical "vllm" path +# This ensures coverage.combine properly merges data from different test runs +source = [ + "vllm", + "/vllm-workspace/src/vllm", + "/vllm-workspace/vllm", + "*/site-packages/vllm", + "*/dist-packages/vllm", +] + +[tool.coverage.report] +exclude_lines = [ + 'pragma: no cover', + 'def __repr__', + 'if self.debug:', + 'if settings.DEBUG', + 'raise AssertionError', + 'raise NotImplementedError', + 'if 0:', + 'if __name__ == .__main__.:', + 'class .*\bProtocol\):', + '@(abc\.)?abstractmethod', +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.coverage.xml] +output = "coverage.xml" + [tool.ty.src] respect-ignore-files = true diff --git a/requirements/cpu.txt b/requirements/cpu.txt index 30e47a8cab..06bddd4a67 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -21,5 +21,10 @@ torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64 # Intel Extension for PyTorch, only for x86_64 CPUs intel-openmp==2024.2.1; platform_machine == "x86_64" +# triton-cpu, only for x86_64 CPUs +triton @ https://github.com/triton-lang/triton-cpu/releases/download/v3.8.0/triton-3.8.0+cpu-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_machine == "x86_64" and python_version == "3.10" +triton @ https://github.com/triton-lang/triton-cpu/releases/download/v3.8.0/triton-3.8.0+cpu-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_machine == "x86_64" and python_version == "3.11" +triton @ https://github.com/triton-lang/triton-cpu/releases/download/v3.8.0/triton-3.8.0+cpu-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; platform_machine == "x86_64" and python_version >= "3.12" + # Use this to gather CPU info and optimize based on ARM Neoverse cores py-cpuinfo; platform_machine == "aarch64" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 72609d8983..2bbf1e244d 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4 # flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from # install_requires so the published wheel does not carry an unresolvable pin --extra-index-url https://flashinfer.ai/whl/ -flashinfer-python==0.6.18.post1 -flashinfer-cubin==0.6.18.post1 +flashinfer-python==0.7.0 +flashinfer-cubin==0.7.0 apache-tvm-ffi==0.1.11 tilelang==0.1.12 nvidia-cudnn-frontend>=1.19.1 @@ -33,4 +33,4 @@ quack-kernels==0.6.5 # Required by MSA tokenspeed-mla==0.1.8; platform_system == "Linux" # Humming kernels for quantization gemm -humming-kernels[cu13]==0.1.15 +humming-kernels[cu13]==0.1.16 diff --git a/requirements/rubin-prerelease.txt b/requirements/rubin-prerelease.txt index 8b9f7d3155..d02fb38884 100644 --- a/requirements/rubin-prerelease.txt +++ b/requirements/rubin-prerelease.txt @@ -16,11 +16,11 @@ torchaudio==2.11.0.dev20260911+cu134 # INSTALL_RUBIN_PRERELEASE=true, so they must be pinned here instead to # still get installed. nvidia-cutlass-dsl[cu13]==4.8.0.dev0 -flashinfer-python==0.6.18.post1 +flashinfer-python==0.7.0 quack-kernels==0.6.5 cuda-python==13.4.1 cuda-bindings==13.4.1 -flashinfer-jit-cache==0.6.18.post1+cu134 -flashinfer-cubin==0.6.18.post1 +flashinfer-jit-cache==0.7.0+cu134 +flashinfer-cubin==0.7.0 nixl==1.4.1 diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index 796e76eb44..d2ad5a0cfd 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -38,6 +38,7 @@ jiwer bm25s pystemmer mteb[bm25s]>=2, <3 # required for mteb test +sentence-transformers==5.2.0 # required for embedding tests num2words pqdm diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index d9572389eb..ed8827f5db 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -175,7 +175,7 @@ docopt==0.6.2 # via num2words docstring-parser==0.18.0 # via anthropic -dpcpp-cpp-rt==2026.0.0 +dpcpp-cpp-rt==2026.1.0 # via # onemkl-sycl-blas # onemkl-sycl-dft @@ -328,27 +328,27 @@ idna==3.18 # yarl ijson==3.5.1 # via -r requirements/test/../common.txt -impi-rt==2021.18.0 +impi-rt==2021.18.1 # via # oneccl # torch iniconfig==2.3.0 # via pytest -intel-cmplr-lib-rt==2026.0.0 +intel-cmplr-lib-rt==2026.1.0 # via # intel-sycl-rt # torch -intel-cmplr-lib-ur==2026.0.0 +intel-cmplr-lib-ur==2026.1.0 # via # intel-openmp # intel-sycl-rt # torch -intel-cmplr-lic-rt==2026.0.0 +intel-cmplr-lic-rt==2026.1.0 # via # intel-opencl-rt # intel-sycl-rt # torch -intel-opencl-rt==2026.0.0 +intel-opencl-rt==2026.1.0 # via # dpcpp-cpp-rt # onemkl-sycl-blas @@ -357,14 +357,14 @@ intel-opencl-rt==2026.0.0 # onemkl-sycl-rng # onemkl-sycl-sparse # torch -intel-openmp==2026.0.0 +intel-openmp==2026.1.0 # via # dpcpp-cpp-rt # mkl # torch -intel-pti==0.17.0 +intel-pti==1.0.1 # via torch -intel-sycl-rt==2026.0.0 +intel-sycl-rt==2026.1.0 # via # dpcpp-cpp-rt # oneccl @@ -463,7 +463,7 @@ mistral-common==1.11.7 # -c requirements/common.txt # -r requirements/test/../common.txt # -r requirements/test/xpu.in -mkl==2026.0.0 +mkl==2026.1.0 # via # onemkl-sycl-blas # onemkl-sycl-dft @@ -540,35 +540,34 @@ numpy==2.2.6 # sacrebleu # scikit-learn # scipy - # sentence-transformers # soundfile # soxr # tensorizer # torchvision # transformers # xgrammar -oneccl==2022.0.0 +oneccl==2022.1.1 # via # oneccl-devel # torch -oneccl-devel==2022.0.0 +oneccl-devel==2022.1.1 # via torch -onemkl-license==2026.0.0 +onemkl-license==2026.1.0 # via # mkl # torch -onemkl-sycl-blas==2026.0.0 +onemkl-sycl-blas==2026.1.0 # via # onemkl-sycl-lapack # onemkl-sycl-sparse # torch -onemkl-sycl-dft==2026.0.0 +onemkl-sycl-dft==2026.1.0 # via torch -onemkl-sycl-lapack==2026.0.0 +onemkl-sycl-lapack==2026.1.0 # via torch -onemkl-sycl-rng==2026.0.0 +onemkl-sycl-rng==2026.1.0 # via torch -onemkl-sycl-sparse==2026.0.0 +onemkl-sycl-sparse==2026.1.0 # via torch open-clip-torch==2.32.0 # via -r requirements/test/xpu.in @@ -836,7 +835,7 @@ pyyaml==6.0.3 # timm # transformers # uvicorn -pyzes==0.1.1 +pyzes==0.1.2 # via torch pyzmq==27.1.0 # via @@ -936,8 +935,10 @@ scipy==1.18.0 # pytrec-eval-terrier # scikit-learn # sentence-transformers -sentence-transformers==5.7.0 - # via mteb +sentence-transformers==5.2.0 + # via + # -r requirements/test/xpu.in + # mteb sentencepiece==0.2.1 # via -r requirements/test/../common.txt sentry-sdk==2.68.0 @@ -1007,7 +1008,7 @@ tabledata==1.3.5 # via pytablewriter tabulate==0.10.0 # via sacrebleu -tbb==2023.0.0 +tbb==2023.1.0 # via # intel-opencl-rt # mkl @@ -1047,9 +1048,8 @@ tokenizers==0.23.1 # -c requirements/common.txt # -r requirements/test/../common.txt # cohere - # sentence-transformers # transformers -torch==2.13.0+xpu +torch==2.14.0+xpu # via # -c requirements/xpu.txt # accelerate @@ -1062,7 +1062,7 @@ torch==2.13.0+xpu # timm # torchvision # xgrammar -torchvision==0.28.0+xpu +torchvision==0.29.0+xpu # via # open-clip-torch # timm @@ -1090,11 +1090,11 @@ transformers==5.17.0 # mteb # sentence-transformers # xgrammar -triton==3.7.2+xpu +triton==3.8.0+xpu # via # -c requirements/xpu.txt # xgrammar -triton-xpu==3.7.2 +triton-xpu==3.8.0 # via # torch # triton diff --git a/requirements/xpu.txt b/requirements/xpu.txt index d0e94ca189..fd5b3214f0 100644 --- a/requirements/xpu.txt +++ b/requirements/xpu.txt @@ -1,7 +1,7 @@ # Common dependencies -r common.txt --extra-index-url=https://wheels.vllm.ai/xpu/ -triton==3.7.2+xpu +triton==3.8.0+xpu ray>=2.9 cmake>=3.26.1 @@ -14,10 +14,10 @@ jinja2>=3.1.6 datasets # for benchmark scripts numba == 0.65.0 # Required for N-gram speculative decoding --extra-index-url=https://download.pytorch.org/whl/xpu -torch==2.13.0 +torch==2.14.0 torchaudio torchvision torchcodec >= 0.14 # Required for the torchcodec video decoding backend auto_round_lib==0.15.0 -vllm_xpu_kernels==0.1.14.1 +vllm_xpu_kernels==0.1.15.4 diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0a001833e9..5b122ea14c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8" +source = "git+https://github.com/smg-project/llm-multimodal?rev=f0985ef65967615db2c79279aa07818499301bfd#f0985ef65967615db2c79279aa07818499301bfd" dependencies = [ "anyhow", "base64 0.22.1", @@ -5903,6 +5903,7 @@ dependencies = [ name = "vllm-metrics" version = "0.1.0" dependencies = [ + "expect-test", "itertools 0.14.0", "prometheus-client", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index afcce8d7fd..40b48f85bc 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -61,7 +61,7 @@ indexmap = "2.13.0" indicatif = "0.18.4" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.24", features = ["pycompat"] } diff --git a/rust/README.md b/rust/README.md index 0cc33d64f8..140b027b02 100644 --- a/rust/README.md +++ b/rust/README.md @@ -83,7 +83,7 @@ To build the `vllm-rs` in isolation: ```bash # from the local checkout -./build_rust.sh +./tools/build_rust.sh ``` ### Engine-free renderer diff --git a/rust/clippy.toml b/rust/clippy.toml new file mode 100644 index 0000000000..d3ea524317 --- /dev/null +++ b/rust/clippy.toml @@ -0,0 +1,3 @@ +disallowed-types = [ + { path = "prometheus_client::metrics::histogram::Histogram", reason = "it locks on every observation; use `vllm_metrics::Histogram`" }, +] diff --git a/rust/src/bench/src/cli.rs b/rust/src/bench/src/cli.rs index fc0f5a29e4..302dbb4792 100644 --- a/rust/src/bench/src/cli.rs +++ b/rust/src/bench/src/cli.rs @@ -176,7 +176,7 @@ pub struct BenchServeArgs { #[arg(long)] pub tokenizer: Option, - /// Tokenizer mode (auto, hf, slow, mistral). Accepted for Python CLI + /// Tokenizer mode (auto, hf, mistral). Accepted for Python CLI /// compatibility; non-auto values are ignored with a warning. #[arg(long, default_value = "auto")] pub tokenizer_mode: String, diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 1f1f1974ea..a110020ec1 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -28,6 +28,7 @@ thiserror-ext.workspace = true time.workspace = true tokio.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true trait-set.workspace = true uuid.workspace = true vllm-engine-core-client.workspace = true @@ -48,7 +49,6 @@ rmp-serde.workspace = true serial_test.workspace = true tempfile.workspace = true tokio.workspace = true -tracing-subscriber.workspace = true uuid.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } vllm-tokenizer = { workspace = true, features = ["test-utils"] } diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index 52e26370af..7d124aaaf0 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -21,6 +21,8 @@ pub enum Error { InvalidReasoningEffort(String), #[error("{message}")] InvalidReasoningControl { message: String }, + #[error("chat role `{role}` is not supported by this chat renderer")] + UnsupportedChatRole { role: String }, #[error("multimodal input is not supported by this chat renderer")] UnsupportedMultimodalRenderer, #[error("unsupported multimodal content: {0}")] @@ -114,7 +116,8 @@ impl Error { | Self::InvalidReasoningControl { .. } | Self::DuplicateToolName { .. } | Self::ToolChoiceRequiresTools - | Self::ToolChoiceFunctionNotFound { .. } => true, + | Self::ToolChoiceFunctionNotFound { .. } + | Self::UnsupportedChatRole { .. } => true, Self::Text(error) => error.is_request_validation_error(), Self::UnsupportedMultimodalRenderer | Self::UnsupportedMultimodalContent(_) diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 8f41d72004..95c4332f74 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -72,6 +72,9 @@ pub struct ChatRequestProcessor { /// Effective model dtype reported by the engine. /// Absent for text-only frontends without an engine handshake. model_dtype: Option, + /// Effective runtime context length reported by the engine. + /// Absent for render-only frontends without an engine handshake. + max_model_len: Option, /// Tool-call parser selection used when preparing generation requests. tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. @@ -83,10 +86,11 @@ pub struct ChatRequestProcessor { impl ChatRequestProcessor { /// Create a processor with multimodal support using the effective model /// dtype reported by the engine. - fn new(backend: DynChatBackend, model_dtype: ModelDtype) -> Self { + fn new(backend: DynChatBackend, model_dtype: ModelDtype, max_model_len: u32) -> Self { Self { backend, model_dtype: Some(model_dtype), + max_model_len: Some(max_model_len as usize), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, tool_strict_level: ToolStrictLevel::Auto, @@ -98,6 +102,7 @@ impl ChatRequestProcessor { Self { backend, model_dtype: None, + max_model_len: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, tool_strict_level: ToolStrictLevel::Auto, @@ -137,6 +142,7 @@ impl ChatRequestProcessor { rendered, self.backend.multimodal_model_info(), model_dtype, + self.max_model_len, ) .await } @@ -161,7 +167,8 @@ impl ChatRequestProcessor { let features = match media { multimodal::MultimodalInput::Raw(parts) => { let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?; - info.prepare_multimodal(parts, token_ids, model_dtype).await? + info.prepare_multimodal(parts, token_ids, model_dtype, self.max_model_len) + .await? } multimodal::MultimodalInput::Preprocessed(features) => { info.prepare_preprocessed(features, token_ids.len())? @@ -256,10 +263,11 @@ impl ChatLlm { /// backend. pub fn new(text: TextLlm, backend: DynChatBackend) -> Self { let model_dtype = text.engine_core_client().model_dtype(); + let max_model_len = text.engine_core_client().max_model_len(); Self { text, - processor: ChatRequestProcessor::new(backend, model_dtype), + processor: ChatRequestProcessor::new(backend, model_dtype, max_model_len), } } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 4af49aa5c6..4138d39bda 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -23,6 +23,7 @@ use llm_multimodal::{ ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor, + VisionPreprocessingContext, }; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; @@ -55,6 +56,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer}; #[derive(Clone)] pub struct MultimodalModelInfo { context: MultimodalModelContext, + /// Rendered placeholder marker IDs for all resolved modalities. + placeholder_token_ids: Vec, image: Option, video: Option, audio: Option, @@ -420,8 +423,18 @@ impl MultimodalModelInfo { MediaConnectorConfig::default(), )?); + let placeholder_token_ids = [ + image.as_ref().map(|support| support.placeholder.marker_token_id), + video.as_ref().map(|support| support.placeholder.marker_token_id), + audio.as_ref().map(|support| support.placeholder.marker_token_id), + ] + .into_iter() + .flatten() + .collect(); + Ok(Some(Self { context, + placeholder_token_ids, image, video, audio, @@ -547,6 +560,20 @@ impl MultimodalModelInfo { Modality::ImageEmbeds => None, } } + + fn vision_preprocessing_context( + &self, + prompt_token_ids: &[u32], + max_model_len: Option, + ) -> VisionPreprocessingContext { + let text_prompt_length = prompt_token_ids + .iter() + .filter(|token_id| !self.placeholder_token_ids.contains(token_id)) + .count(); + VisionPreprocessingContext { + token_budget: max_model_len.map(|limit| limit.saturating_sub(text_prompt_length)), + } + } } /// Finalize a rendered chat prompt into text-generation input. @@ -559,6 +586,7 @@ pub(crate) async fn finalize_rendered_prompt( rendered: RenderedPrompt, info: Option<&MultimodalModelInfo>, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result<(Prompt, Option)> { let media_parts = extract_media_parts(request, rendered.media_order.as_deref())?; if media_parts.is_empty() { @@ -574,7 +602,12 @@ pub(crate) async fn finalize_rendered_prompt( Prompt::TokenIds(token_ids) => token_ids, }; let prepared = info - .prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype) + .prepare_multimodal( + media_parts, + &mut prompt_token_ids, + model_dtype, + max_model_len, + ) .instrument(mm_request_span(&request.request_id)) .await?; @@ -599,7 +632,8 @@ fn extract_media_parts( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => content, + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => content, ChatMessage::Assistant { .. } => { bail_multimodal!("renderer reported multimodal assistant content") } @@ -622,7 +656,8 @@ fn extract_media_parts( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => Some(content), + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => Some(content), ChatMessage::Assistant { .. } => None, }; match content { @@ -766,11 +801,13 @@ impl MultimodalModelInfo { media_parts: Vec, prompt_token_ids: &mut Vec, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result { if media_parts.is_empty() { return Ok(Vec::new()); } - self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await + self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype, max_model_len) + .await } /// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing` @@ -786,15 +823,24 @@ impl MultimodalModelInfo { media_parts: Vec, prompt_token_ids: &mut Vec, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result { let media_parts_len = media_parts.len(); self.validate_mm_limits(&media_parts)?; + let vision_context = self.vision_preprocessing_context(prompt_token_ids, max_model_len); let fetched = self.fetch_media(media_parts).await?; let mut prepared = Vec::new(); if !fetched.images.is_empty() { - prepared - .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?); + prepared.push( + self.prepare_images( + fetched.images, + fetched.image_uuids, + model_dtype, + vision_context, + ) + .await?, + ); } if !fetched.videos.is_empty() { prepared @@ -1042,6 +1088,100 @@ mod tests { ); } + #[test] + fn vision_preprocessing_context_uses_runtime_budget_and_text_tokens() { + let info = qwen3_vl_info(); + let context = info.vision_preprocessing_context( + &[11, QWEN3_IMAGE_PAD_ID, 12, QWEN3_VIDEO_PAD_ID, 13], + Some(4096), + ); + + assert_eq!(context.token_budget, Some(4096 - 3)); + assert_eq!( + info.vision_preprocessing_context(&[11, 12, 13], Some(2)).token_budget, + Some(0) + ); + assert_eq!( + info.vision_preprocessing_context(&[11], None).token_budget, + None + ); + } + + #[tokio::test] + async fn nemotron_budget_controls_preprocessing_and_engine_features() { + use vllm_engine_core_client::protocol::multimodal::MmKwargValue; + + let tokenizer = TestTokenizer::new() + .with_regular_token("", 1018) + .with_regular_token("", 1019) + .with_regular_token("", 1020); + let config = serde_json::json!({ + "model_type": "nemotron_h_omni", + "img_context_token_id": 1018, + "max_position_embeddings": 512, + "patch_size": 16, + "downsample_ratio": 0.5, + "norm_mean": [0.0, 0.0, 0.0], + "norm_std": [1.0, 1.0, 1.0], + "vision_config": { + "args": {"min_num_patches": 1024, "max_num_patches": 13312} + } + }); + let info = test_info("nemotron_h_omni", config, tokenizer); + let media = MediaContentPart::ImageUrl { + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=".to_string(), + detail: None, + uuid: None, + }; + let mut prompt = vec![11; 100]; + prompt.push(1018); + + let mut tokens = prompt.clone(); + let features = info + .prepare_multimodal( + vec![media.clone()], + &mut tokens, + ModelDtype::Float32, + Some(512), + ) + .await + .unwrap(); + assert_eq!(features.len(), 1); + let feature = &features[0]; + assert_eq!( + (feature.mm_position.offset, feature.mm_position.length), + (100, 258) + ); + assert_eq!(tokens.len(), 358); + assert_eq!(tokens[100], 1019); + assert!(tokens[101..357].iter().all(|&token| token == 1018)); + assert_eq!(tokens[357], 1020); + let data = feature.data.as_ref().unwrap(); + assert!(matches!( + data["pixel_values_flat"].data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) if tensor.shape == [3, 512, 512] + )); + assert_eq!( + data["num_tokens_per_image"].data, + Some(MmKwargValue::Int(256)) + ); + assert_eq!( + data["imgs_sizes"].data, + Some(MmKwargValue::List(vec![ + MmKwargValue::Int(512), + MmKwargValue::Int(512), + ])) + ); + + let error = info + .prepare_multimodal(vec![media], &mut prompt, ModelDtype::Float32, Some(100)) + .await + .unwrap_err(); + assert!( + matches!(error, Error::Multimodal(message) if message.contains("exceeding token_budget")) + ); + } + #[test] fn qwen3_vl_without_video_token_id_disables_video_support_only() { let config = serde_json::json!({ diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs index c230752f94..6b2fecf78d 100644 --- a/rust/src/chat/src/multimodal/audio.rs +++ b/rust/src/chat/src/multimodal/audio.rs @@ -259,7 +259,7 @@ mod tests { uuid: Some("audio-1".to_string()), }]; async { - info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32) + info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32, None) .instrument(mm_request_span("request-1")) .await .unwrap(); diff --git a/rust/src/chat/src/multimodal/image.rs b/rust/src/chat/src/multimodal/image.rs index 3062dae3e6..818bb72f1a 100644 --- a/rust/src/chat/src/multimodal/image.rs +++ b/rust/src/chat/src/multimodal/image.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs}; +use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs, VisionPreprocessingContext}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use super::timing::MM_STAGE_TARGET; @@ -30,11 +30,12 @@ impl MultimodalModelInfo { frames: Vec>, uuids: Vec>, model_dtype: ModelDtype, + preprocessing_context: VisionPreprocessingContext, ) -> Result { let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality { modality: Modality::Image.to_string(), })?; - let preprocessed = self.preprocess_images(support, &frames).await?; + let preprocessed = self.preprocess_images(support, &frames, preprocessing_context).await?; let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?; if replacements.len() != frames.len() { bail_multimodal!( @@ -65,13 +66,16 @@ impl MultimodalModelInfo { &self, support: &VisionModalitySupport, image_frames: &[Arc], + preprocessing_context: VisionPreprocessingContext, ) -> Result { let processor = Arc::clone(&support.processor); let images = image_frames.iter().map(|frame| frame.data().clone()).collect::>(); // TODO: is it still necessary given that we've already in a dedicated runtime? - tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?)) - .await - .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? + tokio::task::spawn_blocking(move || { + Ok(processor.preprocess_with_context(&images, &preprocessing_context)?) + }) + .await + .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? } } diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index 2c564fbda7..3991130989 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -18,7 +18,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal}; pub(super) enum KwargValue { /// Tensor with row-major flat data and shape. Tensor { wire: WireTensor }, - /// Non-tensor kwarg value that is shared or copied as-is. + /// Non-tensor kwarg value preserved in protocol form. Passthrough(ProtocolKwargValue), } @@ -150,7 +150,17 @@ impl KwargValue { /// Batched fields use their first axis as media-item index and drop that /// axis in the per-feature value, matching vLLM's batched-field semantics. pub(super) fn batched_wire_value_at(&self, index: usize) -> Result { - self.wire_value_range(index, index + 1, true) + match self { + Self::Passthrough(ProtocolKwargValue::List(values)) => values + .get(index) + .cloned() + .ok_or_else(|| multimodal!("batched passthrough has no item at index {index}")), + Self::Passthrough(value) if index == 0 => Ok(value.clone()), + Self::Passthrough(_) => Err(multimodal!( + "scalar batched passthrough has no item at index {index}" + )), + Self::Tensor { .. } => self.wire_value_range(index, index + 1, true), + } } /// Convert one media item's flat tensor range directly to wire bytes. @@ -336,6 +346,41 @@ mod tests { ); } + #[test] + fn batched_passthrough_value_at_selects_media_item() { + let image_sizes = KwargValue::from_model_specific( + ModelSpecificValue::TupleVec(vec![(384, 736), (512, 512)]), + ModelDtype::BFloat16, + ) + .unwrap(); + + assert_eq!( + image_sizes.batched_wire_value_at(1).unwrap(), + ProtocolKwargValue::List(vec![ + ProtocolKwargValue::Int(512), + ProtocolKwargValue::Int(512), + ]) + ); + assert!(image_sizes.batched_wire_value_at(2).is_err()); + + let token_counts = KwargValue::from_model_specific( + ModelSpecificValue::IntVec(vec![276, 400]), + ModelDtype::BFloat16, + ) + .unwrap(); + assert_eq!( + token_counts.batched_wire_value_at(1).unwrap(), + ProtocolKwargValue::Int(400) + ); + + let scalar = KwargValue::Passthrough(ProtocolKwargValue::Int(7)); + assert_eq!( + scalar.batched_wire_value_at(0).unwrap(), + ProtocolKwargValue::Int(7) + ); + assert!(scalar.batched_wire_value_at(1).is_err()); + } + #[test] fn flat_wire_value_range_keeps_first_axis() { let value = KwargValue::Tensor { diff --git a/rust/src/chat/src/multimodal/timing.rs b/rust/src/chat/src/multimodal/timing.rs index 5bdf207396..c57383c7a3 100644 --- a/rust/src/chat/src/multimodal/timing.rs +++ b/rust/src/chat/src/multimodal/timing.rs @@ -4,6 +4,7 @@ //! Multimodal preprocessing spans and their timing collector configuration. use tracing::{Span, info_span}; +use tracing_subscriber::filter::{Filtered, LevelFilter}; use vllm_tracing::timing::{RequestTimingLayer, RequestTimingStats}; /// Target of the multimodal preprocessing stage spans. @@ -12,7 +13,10 @@ pub(super) const MM_STAGE_TARGET: &str = "mm_processor_timing"; /// Create the timing layer and its stats handle for multimodal preprocessing /// stage spans (`vllm-bench mm-processor`), mirroring the Python /// `TimingContext` / `MultiModalTimingRegistry`. -pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) { +pub fn mm_timing_layer() -> ( + Filtered, + RequestTimingStats, +) { RequestTimingLayer::new(MM_STAGE_TARGET) } diff --git a/rust/src/chat/src/renderer/deepseek.rs b/rust/src/chat/src/renderer/deepseek.rs index 7892fd2497..1edc671d65 100644 --- a/rust/src/chat/src/renderer/deepseek.rs +++ b/rust/src/chat/src/renderer/deepseek.rs @@ -227,6 +227,9 @@ pub(super) fn render_request_with_media_order( ); } } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } if is_user_like_entry(message, current_render_index as usize, dialect) @@ -286,8 +289,8 @@ pub(super) fn resolve_reasoning( DsDialect::V41 => { let budget = match effort { EffortValue::String(effort) => match effort.as_str() { - "low" => 25, - "high" => 50, + "low" => 50, + "high" => 75, "xhigh" => 75, "max" => 100, _ => return Err(invalid_v41_reasoning_effort()), diff --git a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs index 74c7e4cbf2..33e6eb99d8 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs @@ -158,6 +158,7 @@ fn render_message( thinking_mode == ThinkingMode::Thinking && after_or_at_last_user_turn, content, ), + ChatMessage::Custom { role, .. } => Err(Error::UnsupportedChatRole { role: role.clone() }), } } diff --git a/rust/src/chat/src/renderer/deepseek_v4/tests.rs b/rust/src/chat/src/renderer/deepseek_v4/tests.rs index 21a33ec2d5..8702e8d73c 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/tests.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use expect_test::{ExpectFile, expect, expect_file}; use serde_json::Value; +use thiserror_ext::AsReport as _; use super::DeepSeekV4ChatRenderer; use crate::ChatRenderer; @@ -764,3 +765,20 @@ fn last_developer_omits_generation_prompt_when_disabled() { expect!["<|begin▁of▁sentence|><|User|>latest instruction"].assert_eq(&rendered); } + +#[test] +fn custom_roles_are_rejected_as_request_errors() { + let request = ChatRequest { + messages: vec![ + ChatMessage::custom("root", "identity"), + ChatMessage::user("hello"), + ], + ..ChatRequest::for_test() + }; + + let error = DeepSeekV4ChatRenderer::default().render(&request).unwrap_err(); + + assert!(error.is_request_validation_error()); + expect!["chat role `root` is not supported by this chat renderer"] + .assert_eq(&error.to_report_string()); +} diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt index 0897086688..a41d88e924 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt @@ -1,4 +1,4 @@ -<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) You are a helpful assistant. diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt index 4e93ce2cbb..fa481ca5cd 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt @@ -1,3 +1,3 @@ -<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) You are a helpful assistant.<|User|>Hello<|Assistant|>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|>The user asks about the capital of France. It is Paris.The capital of France is Paris.<|end▁of▁sentence|> \ No newline at end of file diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt index 99f4af0d0e..bd329b7928 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt @@ -1,4 +1,4 @@ -<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) <|User|>question<|User|>policy diff --git a/rust/src/chat/src/renderer/deepseek_v41/tests.rs b/rust/src/chat/src/renderer/deepseek_v41/tests.rs index 62cf6d2024..3f9dc061a6 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v41/tests.rs @@ -165,9 +165,9 @@ fn image_part(uuid: &str) -> ChatContentPart { #[test] fn maps_reasoning_effort_to_reference_numeric_budget() { for (effort, budget) in [ - (None, 50), - (Some(EffortValue::from("low")), 25), - (Some(EffortValue::from("high")), 50), + (None, 75), + (Some(EffortValue::from("low")), 50), + (Some(EffortValue::from("high")), 75), (Some(EffortValue::from("xhigh")), 75), (Some(EffortValue::from("max")), 100), (Some(EffortValue::Number(37.into())), 37), @@ -189,7 +189,7 @@ fn accepts_numeric_template_effort_with_top_level_precedence() { (json!(1), 1), (json!(42), 42), (json!(100), 100), - (json!("low"), 25), + (json!("low"), 50), ] { let mut request = request(); request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort); @@ -286,11 +286,11 @@ fn system_after_dropped_developer_becomes_leading_system() { ], ..ChatRequest::for_test() }; - expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"] + expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"] .assert_eq(&render(&request)); request.messages.push(ChatMessage::user("question")); - expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>question<|Assistant|>"] + expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>question<|Assistant|>"] .assert_eq(&render(&request)); } @@ -322,7 +322,7 @@ fn last_user_turn_omits_generation_prompt_when_disabled() { let mut request = request(); request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; - expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>question"] + expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>question"] .assert_eq(&render(&request)); } diff --git a/rust/src/chat/src/renderer/harmony/mod.rs b/rust/src/chat/src/renderer/harmony/mod.rs index 83b59e9198..d8dafadca7 100644 --- a/rust/src/chat/src/renderer/harmony/mod.rs +++ b/rust/src/chat/src/renderer/harmony/mod.rs @@ -188,7 +188,8 @@ fn peel_leading_instructions( )), ChatMessage::User { .. } | ChatMessage::Assistant { .. } - | ChatMessage::ToolResponse { .. } => Ok((None, None, messages)), + | ChatMessage::ToolResponse { .. } + | ChatMessage::Custom { .. } => Ok((None, None, messages)), } } @@ -328,6 +329,9 @@ fn to_harmony_message( .with_recipient("assistant"), ] } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } }) } diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index 911a73928a..f406e32da6 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -258,8 +258,8 @@ impl ChatRenderer for HfChatRenderer { // TODO: borrow more fields directly from the original `ChatMessage`. #[serde_with::skip_serializing_none] #[derive(Debug, Serialize)] -struct TemplateMessage { - role: &'static str, +struct TemplateMessage<'a> { + role: &'a str, content: TemplateContent, // Developer-role messages may provide message-local tools in the same shape // as top-level request tools. @@ -322,22 +322,22 @@ struct TemplateToolDefinition { } /// Convert chat messages into the JSON shape expected by Jinja chat templates. -fn to_template_messages( - messages: &[ChatMessage], +fn to_template_messages<'a>( + messages: &'a [ChatMessage], content_format: ChatTemplateContentFormat, multimodal: Option<&MultimodalRenderInfo>, -) -> Result> { +) -> Result>> { messages .iter() .map(|message| to_template_message(message, content_format, multimodal)) .collect() } -fn to_template_message( - message: &ChatMessage, +fn to_template_message<'a>( + message: &'a ChatMessage, content_format: ChatTemplateContentFormat, multimodal: Option<&MultimodalRenderInfo>, -) -> Result { +) -> Result> { Ok(match message { ChatMessage::System { content } => TemplateMessage { role: "system", @@ -394,6 +394,15 @@ fn to_template_message( tool_calls: None, tool_call_id: Some(tool_call_id.clone()), }, + ChatMessage::Custom { role, content } => TemplateMessage { + role, + content: to_template_content(content, content_format, multimodal)?, + tools: None, + reasoning: None, + reasoning_content: None, + tool_calls: None, + tool_call_id: None, + }, }) } @@ -524,7 +533,7 @@ const CONTINUE_FINAL_MESSAGE_TAG: &str = "CONTINUE_FINAL_MESSAGE_TAG "; // TODO: transformers v5 also allows continuing a non-`content` field (e.g. // `reasoning_content`) by passing a field name; only the boolean form is // supported here. -fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result { +fn append_continue_final_message_tag(message: &mut TemplateMessage<'_>) -> Result { let text = match &mut message.content { TemplateContent::String(text) => Some(text), // Pick the last text part in the message. @@ -1001,6 +1010,23 @@ mod tests { assert_eq!(rendered, "developer|policy|get_weather|city"); } + #[test] + fn chat_template_passes_custom_roles_unchanged() { + let request = sample_request(vec![ + ChatMessage::custom("root", "identity"), + ChatMessage::system("policy"), + ChatMessage::user("hello"), + ]); + + let rendered = render( + Some("{% for m in messages %}{{ m.role }}={{ m.content }};{% endfor %}"), + &request, + ) + .unwrap(); + + assert_eq!(rendered, "root=identity;system=policy;user=hello;"); + } + #[test] fn chat_template_keeps_string_text_for_openai_detected_templates() { let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]); diff --git a/rust/src/chat/src/renderer/hf/template.rs b/rust/src/chat/src/renderer/hf/template.rs index b890783701..de447e2c34 100644 --- a/rust/src/chat/src/renderer/hf/template.rs +++ b/rust/src/chat/src/renderer/hf/template.rs @@ -45,7 +45,7 @@ fn build_environment(template: String) -> Result> { #[serde_with::skip_serializing_none] #[derive(Default, Serialize)] pub(super) struct TemplateContext<'a> { - pub(super) messages: &'a [TemplateMessage], + pub(super) messages: &'a [TemplateMessage<'a>], pub(super) add_generation_prompt: bool, pub(super) continue_final_message: bool, pub(super) tools: Option<&'a [TemplateTool]>, diff --git a/rust/src/chat/src/renderer/inkling/mod.rs b/rust/src/chat/src/renderer/inkling/mod.rs index 9feee371aa..f792f70d32 100644 --- a/rust/src/chat/src/renderer/inkling/mod.rs +++ b/rust/src/chat/src/renderer/inkling/mod.rs @@ -345,6 +345,9 @@ impl ChatRenderer for InklingChatRenderer { &tool_call_id_to_name, )?; } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } } diff --git a/rust/src/chat/src/renderer/kimi_k3/encoding.rs b/rust/src/chat/src/renderer/kimi_k3/encoding.rs index b434d72c53..2424782947 100644 --- a/rust/src/chat/src/renderer/kimi_k3/encoding.rs +++ b/rust/src/chat/src/renderer/kimi_k3/encoding.rs @@ -205,6 +205,9 @@ pub(super) fn render_request_with_media_order( write_assistant_message(&mut out, content, thinking)?; } ChatMessage::ToolResponse { .. } => unreachable!("handled above"), + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } } flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?; diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 59e7ec6fba..f7ddd9b128 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -24,6 +24,9 @@ pub enum ChatRole { User, Assistant, ToolResponse, + /// Role outside the standard set, passed to chat templates unchanged. + #[serde(untagged)] + Custom(String), } /// One text-only chat content part in OpenAI-style block format. @@ -212,6 +215,10 @@ pub enum ChatMessage { content: ChatContent, tool_call_id: String, }, + /// Message content under a role outside the standard set, passed to chat + /// templates unchanged. + #[serde(untagged)] + Custom { role: String, content: ChatContent }, } impl ChatMessage { @@ -230,6 +237,7 @@ impl ChatMessage { use ChatMessage::tool_response() instead" ) } + ChatRole::Custom(role) => Self::custom(role, content), } } @@ -286,6 +294,14 @@ impl ChatMessage { } } + /// Construct one chat message with a role outside the standard set. + pub fn custom(role: impl Into, content: impl Into) -> Self { + Self::Custom { + role: role.into(), + content: content.into(), + } + } + /// Return the chat role of this message. pub fn role(&self) -> ChatRole { match self { @@ -294,6 +310,7 @@ impl ChatMessage { Self::User { .. } => ChatRole::User, Self::Assistant { .. } => ChatRole::Assistant, Self::ToolResponse { .. } => ChatRole::ToolResponse, + Self::Custom { role, .. } => ChatRole::Custom(role.clone()), } } @@ -303,7 +320,8 @@ impl ChatMessage { Self::System { content } | Self::Developer { content, .. } | Self::User { content } - | Self::ToolResponse { content, .. } => content.try_flatten_to_text(), + | Self::ToolResponse { content, .. } + | Self::Custom { content, .. } => content.try_flatten_to_text(), Self::Assistant { content } => Ok(content.text()), } } @@ -315,7 +333,8 @@ impl ChatMessage { Self::System { .. } | Self::Developer { .. } | Self::User { .. } - | Self::ToolResponse { .. } => None, + | Self::ToolResponse { .. } + | Self::Custom { .. } => None, } } @@ -325,7 +344,8 @@ impl ChatMessage { Self::System { content } | Self::Developer { content, .. } | Self::User { content } - | Self::ToolResponse { content, .. } => content.has_multimodal(), + | Self::ToolResponse { content, .. } + | Self::Custom { content, .. } => content.has_multimodal(), Self::Assistant { .. } => false, } } @@ -673,13 +693,14 @@ impl ChatRequest { impl ChatRole { /// Return the chat-template role string used by the current text-only chat /// backend. - pub fn as_str(&self) -> &'static str { + pub fn as_str(&self) -> &str { match self { Self::System => "system", Self::Developer => "developer", Self::User => "user", Self::Assistant => "assistant", Self::ToolResponse => "tool_response", + Self::Custom(role) => role, } } } diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 5bdf66da57..c5a7c75baf 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub enable_scale_out: bool, + /// Send an SSE keep-alive comment line every this many seconds when a + /// streaming response is idle (queued, prefill, or between tokens), to + /// prevent reverse proxies/tunnels with read timeouts from closing the + /// connection. Defaults to 0, which disables keep-alive comments entirely. + #[arg(long, default_value_t = 0)] + #[serde(default)] + pub sse_keep_alive_interval: u64, + /// If provided, the server will require one of these keys to be presented /// in the Authorization header. #[educe(Debug(ignore))] @@ -616,6 +624,8 @@ impl SharedRuntimeArgs { enable_prompt_tokens_details: self.enable_prompt_tokens_details, enable_request_id_headers: self.enable_request_id_headers, enable_scale_out: self.enable_scale_out, + sse_keep_alive_interval: (self.sse_keep_alive_interval > 0) + .then(|| Duration::from_secs(self.sse_keep_alive_interval)), } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index f7f6d0a658..dac00ac983 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ @@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ @@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index 8e514c9534..f65f766e84 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -246,6 +246,13 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub io_processor_plugin: Option, + /// Number of worker threads in the renderer thread pool. The pool is + /// consumed by the async renderer path to parallelize tokenization, chat + /// template rendering, and multimodal preprocessing across concurrent + /// requests. + #[arg(long)] + pub renderer_num_workers: Option, + /// Path to a dynamically reasoning parser plugin that can be dynamically /// loaded and registered. #[arg(long)] @@ -281,6 +288,16 @@ pub struct EngineUnsupportedArgs { )] pub data_parallel_external_lb: Option, + /// Enable fault tolerance for detailed error recovery, such as scaling + /// down fault DPEngineCore. + #[arg( + long, + visible_alias = "no-enable-fault-tolerance", + default_missing_value = "true", + num_args = 0..=1 + )] + pub enable_fault_tolerance: Option, + /// This feature is work in progress and no prefill optimization takes place /// with this flag enabled currently. #[arg( @@ -308,6 +325,28 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub mm_processor_cache_type: Option, + /// Hash algorithm to use for multi-modal input caching. Use `"sha256"` or + /// `"sha512"` for FIPS-compliant deployments. + #[arg(long)] + pub mm_hasher_algorithm: Option, + + /// IPC (inter-process communication) method for multimodal tensors. + /// - "direct_rpc": Use msgspec serialization via RPC + /// - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC + #[arg(long)] + pub mm_tensor_ipc: Option, + + /// Device the HF multi-modal processor runs the image/video transform on. + /// Convenience for `--mm-processor-kwargs '{"device": ...}'`. + #[arg(long)] + pub mm_processor_device: Option, + + /// Amount of GPU memory (in GiB) sequestered on the engine's device for + /// GPU-side multimodal work in the API-server (frontend) process, such as + /// hardware video decoding. + #[arg(long)] + pub mm_ipc_gpu_memory_gb: Option, + /// Dictionary mapping specific modalities to LoRA model paths. #[arg(long)] pub default_mm_loras: Option, @@ -324,6 +363,17 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub stream_interval: Option, + /// Maximum number of requests that can be in-flight (waiting or running) + /// at the same time. When the limit is reached, new requests are rejected + /// with HTTP 503 so the client can retry on another instance. + #[arg(long)] + pub max_num_queued_reqs: Option, + + /// Maximum total prompt tokens of requests currently in the prefill phase. + /// When the limit is reached, new requests are rejected with HTTP 503. + #[arg(long)] + pub max_num_queued_tokens: Option, + /// Structured outputs configuration. #[arg(long)] pub structured_outputs_config: Option, @@ -359,6 +409,17 @@ pub struct ServerUnsupportedArgs { )] pub trust_request_chat_template: Option, + /// Whether to trust per-request multimodal kwargs (`mm_processor_kwargs` + /// and `media_io_kwargs`). If False, the server rejects non-empty values + /// because they can change multimodal preprocessing resource usage. + #[arg( + long, + visible_alias = "no-trust-request-mm-kwargs", + default_missing_value = "true", + num_args = 0..=1 + )] + pub trust_request_mm_kwargs: Option, + /// The role name to return if `request.add_generation_prompt=true`. #[arg(long)] pub response_role: Option, @@ -419,6 +480,15 @@ pub struct ServerUnsupportedArgs { #[arg(long)] pub max_log_len: Option, + /// If set to True, include per-request timing metrics in API responses. + #[arg( + long, + visible_alias = "no-enable-per-request-metrics", + default_missing_value = "true", + num_args = 0..=1 + )] + pub enable_per_request_metrics: Option, + /// If set to True, enable tracking server_load_metrics in the app state. #[arg( long, @@ -492,6 +562,15 @@ pub struct ServerUnsupportedArgs { )] pub tokens_only: Option, + /// Controls the `system_fingerprint` field on responses: `full`, `hash`, + /// `custom`, or `none`. + #[arg(long)] + pub fingerprint_mode: Option, + + /// Literal fingerprint string used when `--fingerprint-mode=custom`. + #[arg(long)] + pub fingerprint_value: Option, + /// Log level for uvicorn. #[arg(long)] pub uvicorn_log_level: Option, @@ -562,4 +641,14 @@ pub struct ServerUnsupportedArgs { num_args = 0..=1 )] pub enable_offline_docs: Option, + + /// If set, run pooling score MaxSim on GPU in the API server process. + /// Can significantly improve late-interaction scoring performance. + #[arg( + long, + visible_alias = "no-enable-flash-late-interaction", + default_missing_value = "true", + num_args = 0..=1 + )] + pub enable_flash_late_interaction: Option, } diff --git a/rust/src/metrics/Cargo.toml b/rust/src/metrics/Cargo.toml index ab1a72098b..74e4d56d5b 100644 --- a/rust/src/metrics/Cargo.toml +++ b/rust/src/metrics/Cargo.toml @@ -8,5 +8,8 @@ license.workspace = true itertools.workspace = true prometheus-client.workspace = true +[dev-dependencies] +expect-test.workspace = true + [lints] workspace = true diff --git a/rust/src/metrics/src/api_server.rs b/rust/src/metrics/src/api_server.rs index 4decaa5573..855e9fbbb9 100644 --- a/rust/src/metrics/src/api_server.rs +++ b/rust/src/metrics/src/api_server.rs @@ -3,10 +3,9 @@ use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::metrics::family::Family; -use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; -use crate::U64Counter; +use crate::{Histogram, U64Counter}; const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0]; const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [ diff --git a/rust/src/metrics/src/histogram.rs b/rust/src/metrics/src/histogram.rs new file mode 100644 index 0000000000..f6d9afd86c --- /dev/null +++ b/rust/src/metrics/src/histogram.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::fmt; +use std::iter::once; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use prometheus_client::encoding::{EncodeMetric, MetricEncoder, NoLabelSet}; +use prometheus_client::metrics::{MetricType, TypedMetric}; + +/// Prometheus histogram with lock-free observations. +/// +/// `prometheus_client::metrics::histogram::Histogram` takes an exclusive lock for every +/// observation, which serializes all Tokio workers on hot histograms such as inter-token +/// latency. This histogram updates atomic cells instead, so observers never block each other +/// or a concurrent scrape. +/// +/// A scrape may see an observation's bucket count before its sum. `_count` is derived from the +/// bucket counts, so it always equals the `+Inf` bucket. +#[derive(Clone, Debug)] +pub struct Histogram { + inner: Arc, +} + +#[derive(Debug)] +struct Inner { + /// Upper bounds, ending with `f64::MAX`, which is encoded as `+Inf`. + upper_bounds: Box<[f64]>, + /// Non-cumulative observation count of each bucket. + bucket_counts: Box<[AtomicU64]>, + /// Bits of the `f64` sum of all observations. + sum_bits: AtomicU64, +} + +impl Histogram { + /// Create a histogram with the given finite bucket upper bounds, in ascending order. + pub fn new(buckets: impl IntoIterator) -> Self { + let upper_bounds: Box<[f64]> = buckets.into_iter().chain(once(f64::MAX)).collect(); + let bucket_counts = upper_bounds.iter().map(|_| AtomicU64::new(0)).collect(); + Self { + inner: Arc::new(Inner { + upper_bounds, + bucket_counts, + sum_bits: AtomicU64::new(0.0_f64.to_bits()), + }), + } + } + + pub fn observe(&self, value: f64) { + let inner = &*self.inner; + // Unlike `prometheus_client`, which counts `+inf` and NaN in `_count` but in no bucket, + // put them in the `+Inf` bucket as the Go client does. + let index = inner + .upper_bounds + .iter() + .position(|bound| value <= *bound) + .unwrap_or(inner.upper_bounds.len() - 1); + inner.bucket_counts[index].fetch_add(1, Ordering::Relaxed); + // The closure always returns `Some`, so the update cannot fail. + let _ = inner.sum_bits.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |bits| { + Some((f64::from_bits(bits) + value).to_bits()) + }); + } +} + +impl TypedMetric for Histogram { + const TYPE: MetricType = MetricType::Histogram; +} + +impl EncodeMetric for Histogram { + fn encode(&self, mut encoder: MetricEncoder) -> Result<(), fmt::Error> { + let inner = &*self.inner; + let buckets: Vec<(f64, u64)> = inner + .upper_bounds + .iter() + .zip(&inner.bucket_counts) + .map(|(bound, count)| (*bound, count.load(Ordering::Relaxed))) + .collect(); + let count = buckets.iter().map(|(_, count)| count).sum(); + let sum = f64::from_bits(inner.sum_bits.load(Ordering::Relaxed)); + encoder.encode_histogram::(sum, count, &buckets, None) + } + + fn metric_type(&self) -> MetricType { + Self::TYPE + } +} + +#[cfg(test)] +mod tests { + use expect_test::expect; + use prometheus_client::encoding::text::encode; + use prometheus_client::registry::Registry; + + use super::*; + + const BUCKETS: [f64; 3] = [0.1, 1.0, 10.0]; + + fn render(metric: impl EncodeMetric + fmt::Debug + Send + Sync + 'static) -> String { + let mut registry = Registry::default(); + registry.register("latency", "Latency", metric); + let mut output = String::new(); + encode(&mut output, ®istry).unwrap(); + output + } + + #[test] + #[expect( + clippy::disallowed_types, + reason = "compare against the replaced implementation" + )] + fn finite_observations_match_prometheus_client() { + let baseline = prometheus_client::metrics::histogram::Histogram::new(BUCKETS); + let histogram = Histogram::new(BUCKETS); + let mut values = vec![-1.0, 0.0, 100.0, f64::MAX]; + for bound in BUCKETS { + values.extend([bound.next_down(), bound, bound.next_up()]); + } + for value in values { + baseline.observe(value); + histogram.observe(value); + } + assert_eq!(render(histogram), render(baseline)); + } + + #[test] + fn concurrent_observations_are_not_lost() { + let histogram = Histogram::new(BUCKETS); + std::thread::scope(|scope| { + // Dyadic values keep the sum exact in any addition order. + for value in [0.0625, 0.5, 4.0, 64.0] { + let histogram = histogram.clone(); + scope.spawn(move || (0..10_000).for_each(|_| histogram.observe(value))); + } + }); + expect![[r##" + # HELP latency Latency. + # TYPE latency histogram + latency_sum 685625.0 + latency_count 40000 + latency_bucket{le="0.1"} 10000 + latency_bucket{le="1.0"} 20000 + latency_bucket{le="10.0"} 30000 + latency_bucket{le="+Inf"} 40000 + # EOF + "##]] + .assert_eq(&render(histogram)); + } + + #[test] + fn non_finite_observations_count_in_inf_bucket() { + let histogram = Histogram::new(BUCKETS); + histogram.observe(f64::INFINITY); + histogram.observe(f64::NAN); + expect![[r##" + # HELP latency Latency. + # TYPE latency histogram + latency_sum NaN + latency_count 2 + latency_bucket{le="0.1"} 0 + latency_bucket{le="1.0"} 0 + latency_bucket{le="10.0"} 0 + latency_bucket{le="+Inf"} 2 + # EOF + "##]] + .assert_eq(&render(histogram)); + } +} diff --git a/rust/src/metrics/src/lib.rs b/rust/src/metrics/src/lib.rs index 25ab4fc6ba..0324946676 100644 --- a/rust/src/metrics/src/lib.rs +++ b/rust/src/metrics/src/lib.rs @@ -9,14 +9,15 @@ use prometheus_client::encoding::text::encode; use prometheus_client::metrics::counter::Counter; pub use prometheus_client::metrics::family::Family; use prometheus_client::metrics::gauge::Gauge; -use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; mod api_server; +mod histogram; mod request; mod scheduler; pub use api_server::*; +pub use histogram::Histogram; pub use request::*; pub use scheduler::*; diff --git a/rust/src/metrics/src/request.rs b/rust/src/metrics/src/request.rs index 5bfc32e895..b6e7683e76 100644 --- a/rust/src/metrics/src/request.rs +++ b/rust/src/metrics/src/request.rs @@ -3,10 +3,9 @@ use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::metrics::family::Family; -use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; -use crate::{EngineLabels, HistogramFamily, U64Counter}; +use crate::{EngineLabels, Histogram, HistogramFamily, U64Counter}; const TTFT_BUCKETS: [f64; 22] = [ 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, diff --git a/rust/src/metrics/src/scheduler.rs b/rust/src/metrics/src/scheduler.rs index d7f52fd0a6..cdc25a244d 100644 --- a/rust/src/metrics/src/scheduler.rs +++ b/rust/src/metrics/src/scheduler.rs @@ -7,10 +7,9 @@ use std::sync::{Arc, Mutex}; use itertools::Itertools as _; use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder}; use prometheus_client::metrics::family::Family; -use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; -use crate::{F64Gauge, HistogramFamily, U64Counter, U64Gauge}; +use crate::{F64Gauge, Histogram, HistogramFamily, U64Counter, U64Gauge}; const KV_CACHE_RESIDENCY_BUCKETS: [f64; 21] = [ 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index afcfaeccfb..c1c580e75a 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -59,6 +59,9 @@ pub struct ApiServerOptions { pub enable_request_id_headers: bool, /// When `true`, register the scale-out `/inference/v1/generate` route. pub enable_scale_out: bool, + /// Idle interval after which streaming SSE responses send a keep-alive + /// comment. `None` disables keep-alive comments. + pub sse_keep_alive_interval: Option, } /// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive. diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 3523f173e3..79b68622cf 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -14,7 +14,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use thiserror_ext::AsReport as _; @@ -40,7 +40,7 @@ use crate::routes::openai::utils::logprobs::clamp_logprob; use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage}; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::resolve_request_context; +use crate::utils::{resolve_request_context, sse_response}; /// Validate one token-in/token-out request and proxy it into the shared /// `vllm-text` stack. @@ -114,7 +114,7 @@ pub async fn generate( ); let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span); - return Sse::new(sse_stream).into_response(); + return sse_response(sse_stream, api_server_options.sse_keep_alive_interval); } let collected = match raw_stream.collect_output().instrument(request_span.clone()).await { diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 474719a875..a87c52fb55 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use serde_json::Value; @@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::{ use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp}; pub(crate) fn lower_chat_request( request: ChatCompletionRequest, @@ -99,7 +99,7 @@ pub async fn chat_completions( ); let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span); - Sse::new(sse_stream).into_response() + sse_response(sse_stream, api_server_options.sse_keep_alive_interval) } else { let response = match collect_chat_completion( chat_stream, diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 1815856636..d4df614270 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -317,6 +317,9 @@ pub(crate) fn convert_message(message: ChatMessage) -> Result { + Ok(VllmChatMessage::custom(role, convert_content(content)?)) + } } } @@ -442,6 +445,7 @@ mod tests { use expect_test::expect; use llm_multimodal::ImageDetail; use serde_json::json; + use thiserror_ext::AsReport as _; use validator::Validate; use vllm_chat::{ AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage, @@ -1047,6 +1051,43 @@ mod tests { ); } + #[test] + fn prepare_chat_request_accepts_custom_role_messages() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "messages": [ + {"role": "root", "content": "Custom identity."}, + {"role": "user", "content": "hello"}, + ], + })) + .expect("parse custom role message"); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!( + prepared.chat_request.messages, + vec![ + VllmChatMessage::custom("root", "Custom identity."), + VllmChatMessage::user("hello"), + ] + ); + } + + #[test] + fn chat_http_message_keeps_standard_role_errors() { + let error = serde_json::from_value::(json!({ + "role": "tool", + "content": "Sunny", + })) + .unwrap_err(); + + expect!["missing field `tool_call_id`"].assert_eq(&error.to_report_string()); + } + #[test] fn prepare_chat_request_maps_image_url_content_parts() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 06874a43bd..7074b8c629 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use serde_json::Value; @@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::LogProbs; use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp}; pub(crate) fn lower_completion_request( request: CompletionRequest, @@ -111,7 +111,7 @@ pub async fn completions( ); let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span); - Sse::new(sse_stream).into_response() + sse_response(sse_stream, api_server_options.sse_keep_alive_interval) } else { let response = match collect_completion( text_stream, diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 5554b5d366..85a891341c 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -410,9 +410,17 @@ impl ToolReference { // Chat Messages // ============================================================================ +/// One chat message, tagged by `role`. +/// +/// The derived serde impls use `remote = "Self"` and cover only the standard +/// roles; the trait impls below dispatch on the role tag and handle +/// [`ChatMessage::Custom`] separately. A `#[serde(untagged)]` fallback variant +/// would also accept malformed standard messages, such as a `tool` message +/// without `tool_call_id`, and replace their field errors with serde's generic +/// untagged-enum error. #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "role")] +#[serde(tag = "role", remote = "Self")] pub enum ChatMessage { #[serde(rename = "system")] System { @@ -447,6 +455,67 @@ pub enum ChatMessage { tools: Option>, name: Option, }, + /// Message with a role outside the OpenAI set, such as `root`. Chat + /// templates receive the role string unchanged. + #[serde(skip)] + Custom { + role: String, + content: MessageContent, + }, +} + +/// Role tags of the standard [`ChatMessage`] variants. +const STANDARD_CHAT_ROLES: &[&str] = &[ + "system", + "user", + "assistant", + "tool", + "function", + "developer", +]; + +/// Wire shape of [`ChatMessage::Custom`]. +#[derive(Deserialize, Serialize)] +struct CustomChatMessage { + role: R, + content: C, +} + +impl Serialize for ChatMessage { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::Custom { role, content } => { + CustomChatMessage { role, content }.serialize(serializer) + } + _ => Self::serialize(self, serializer), + } + } +} + +impl<'de> Deserialize<'de> for ChatMessage { + /// Dispatch on the role tag: standard roles keep their derived errors, + /// and any other string role becomes [`ChatMessage::Custom`]. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let is_custom = value + .get("role") + .and_then(Value::as_str) + .is_some_and(|role| !STANDARD_CHAT_ROLES.contains(&role)); + + if is_custom { + let CustomChatMessage { role, content } = + CustomChatMessage::deserialize(value).map_err(serde::de::Error::custom)?; + Ok(Self::Custom { role, content }) + } else { + Self::deserialize(value).map_err(serde::de::Error::custom) + } + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 89ce35ce68..5c1fff2cae 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -555,7 +555,8 @@ fn render_fake_message_content( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder), + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => render_fake_content(content, placeholder), ChatMessage::Assistant { .. } => message.text_content(), } } diff --git a/rust/src/server/src/utils.rs b/rust/src/server/src/utils.rs index b58e89fd3f..c3a8449617 100644 --- a/rust/src/server/src/utils.rs +++ b/rust/src/server/src/utils.rs @@ -2,9 +2,13 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::HashMap; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use axum::BoxError; use axum::http::HeaderMap; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use futures::Stream; use serde_json::Value; use thiserror_ext::AsReport; use uuid::Uuid; @@ -27,6 +31,27 @@ pub fn unix_timestamp() -> u64 { .unwrap_or_default() } +/// Build a streaming SSE response, optionally sending keep-alive comments +/// while the stream is idle. +/// +/// With `keep_alive_interval` set, a `: keep-alive` comment is sent whenever no +/// event has been produced for that long (queued, prefill, between tokens, or +/// while a parser buffers a long tool call). SSE parsers ignore comments, but +/// they still count as bytes for read timeouts in reverse proxies and clients. +pub fn sse_response(stream: S, keep_alive_interval: Option) -> Response +where + S: Stream> + Send + 'static, + E: Into, +{ + let sse = Sse::new(stream); + match keep_alive_interval { + Some(interval) => sse + .keep_alive(KeepAlive::new().interval(interval).text("keep-alive")) + .into_response(), + None => sse.into_response(), + } +} + /// Construct an API error for a failed utility call to the engine core. pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError { ApiError::server_error(format!("failed to call {method}: {}", error.as_report())) @@ -157,3 +182,84 @@ pub fn resolve_base_request_id( id }) } + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + + use futures::{StreamExt as _, stream}; + use tokio::time::{Instant, sleep}; + + use super::*; + + /// Collect `(elapsed seconds, frame)` pairs from a response whose event + /// stream idles for 20 seconds between its two events. + async fn idle_stream_frames(keep_alive_interval: Option) -> Vec<(u64, String)> { + let events = stream::iter([Ok::<_, Infallible>(Event::default().data("first"))]).chain( + stream::once(async { + sleep(Duration::from_secs(20)).await; + Ok(Event::default().data("second")) + }), + ); + let start = Instant::now(); + sse_response(events, keep_alive_interval) + .into_body() + .into_data_stream() + .map(|frame| { + let frame = frame.expect("read frame"); + ( + start.elapsed().as_secs(), + String::from_utf8(frame.to_vec()).expect("utf-8 frame"), + ) + }) + .collect() + .await + } + + /// An idle gap is filled with `: keep-alive` comments at the configured + /// interval, and real events still pass through in order. + #[tokio::test(start_paused = true)] + async fn sse_response_sends_keep_alive_comments_while_idle() { + let frames = idle_stream_frames(Some(Duration::from_secs(8))).await; + expect_test::expect![[r#" + [ + ( + 0, + "data: first\n\n", + ), + ( + 8, + ": keep-alive\n\n", + ), + ( + 16, + ": keep-alive\n\n", + ), + ( + 20, + "data: second\n\n", + ), + ] + "#]] + .assert_debug_eq(&frames); + } + + /// Without an interval, an idle gap produces no extra bytes. + #[tokio::test(start_paused = true)] + async fn sse_response_without_keep_alive_interval_sends_no_comments() { + let frames = idle_stream_frames(None).await; + expect_test::expect![[r#" + [ + ( + 0, + "data: first\n\n", + ), + ( + 20, + "data: second\n\n", + ), + ] + "#]] + .assert_debug_eq(&frames); + } +} diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 849874bede..0768adf4ae 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -84,10 +84,11 @@ impl HfSpecialTokens { /// Minimal subset of `config.json` (the model's main HF config). /// -/// This intentionally supports only the two layouts we currently care about in +/// This intentionally supports only the layouts we currently care about in /// the Rust frontend: /// - pure text models that keep text metadata at the top level -/// - composite models that expose a single nested `text_config` +/// - composite models that expose a single nested `text_config` or its +/// `llm_config` alias (used by Nemotron) /// /// We do not support additional entry points such as `decoder`, `generator`, or /// `text_encoder`. @@ -102,6 +103,7 @@ pub struct ModelConfig { n_routed_experts: Option, num_local_experts: Option, block_configs: Vec, + #[serde(alias = "llm_config")] text_config: Option>, } @@ -188,7 +190,7 @@ impl ModelConfig { /// Return the config that the Rust frontend treats as the text/LLM config. /// /// This is deliberately narrower than Python/transformers: we only support - /// either the top-level config itself or a single nested `text_config`. + /// either the top-level config itself or a single nested text config. fn effective_text_config(&self) -> &Self { self.text_config.as_deref().unwrap_or(self) } @@ -233,7 +235,8 @@ impl ModelConfig { /// config. /// /// The only intentional simplification here is how we pick the text config: - /// Rust only looks at the top level or `text_config`, not the broader + /// Rust only looks at the top level or `text_config` (including its + /// `llm_config` alias), not the broader /// transformers composite-config surface. fn num_experts_from_block_configs(&self) -> u32 { self.effective_text_config() @@ -405,6 +408,27 @@ mod tests { assert_eq!(config.vocab_size().unwrap(), 151936); } + #[test] + fn model_config_uses_llm_config_for_nemotron_composite_models() { + let config: ModelConfig = serde_json::from_str( + r#"{ + "model_type": "nemotron_h_omni", + "llm_config": { + "model_type": "nemotron_h", + "vocab_size": 131072, + "n_routed_experts": 256, + "eos_token_id": 2 + } + }"#, + ) + .unwrap(); + + assert_eq!(config.model_type(), Some("nemotron_h_omni")); + assert_eq!(config.vocab_size().unwrap(), 131072); + assert_eq!(config.eos_token_ids(), &[2]); + assert_eq!(config.num_experts(), 256); + } + #[test] fn model_config_reads_top_level_eos_token_ids() { let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap(); diff --git a/rust/src/tracing/src/timing.rs b/rust/src/tracing/src/timing.rs index f9a50719fb..899e9eaea2 100644 --- a/rust/src/tracing/src/timing.rs +++ b/rust/src/tracing/src/timing.rs @@ -18,6 +18,7 @@ use std::time::Instant; use tracing::field::{Field, Visit}; use tracing::span::Attributes; use tracing::{Id, Subscriber}; +use tracing_subscriber::filter::{Filtered, LevelFilter}; use tracing_subscriber::layer::{Context, Layer}; use tracing_subscriber::registry::LookupSpan; @@ -66,7 +67,8 @@ impl Visit for SpanFields { /// Timed spans must have the configured target and a `stage` field. The /// `request_id` is taken from the timed span itself or its nearest ancestor /// carrying that field, regardless of the ancestor's target. Spans without -/// a request ID are ignored. Span names are unrestricted. +/// a request ID are ignored. Span names are unrestricted. Both the timed spans +/// and their `request_id` ancestors must be at `INFO` level or above. /// /// Both fields are read at span creation; subsequent [`tracing::Span::record`] /// calls are ignored. String values are used verbatim; other values use their @@ -93,13 +95,18 @@ pub struct RequestTimingStats { impl RequestTimingLayer { /// Create the layer and its stats handle. Only spans whose target equals /// `target` and that carry a `stage` field are timed. - pub fn new(target: &'static str) -> (Self, RequestTimingStats) { + /// + /// The layer is filtered to `INFO` so that, composed with a filtered log + /// layer, it does not raise the subscriber's max level hint and enable + /// `DEBUG`/`TRACE` callsites that the log layer would discard anyway. + pub fn new(target: &'static str) -> (Filtered, RequestTimingStats) { let stats = Arc::new(Mutex::new(HashMap::new())); + let layer = Self { + target, + stats: Arc::clone(&stats), + }; ( - Self { - target, - stats: Arc::clone(&stats), - }, + Filtered::new(layer, LevelFilter::INFO), RequestTimingStats { stats }, ) } @@ -165,8 +172,10 @@ where #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use super::*; - use tracing::info_span; + use tracing::{debug, debug_span, info_span}; use tracing_subscriber::layer::SubscriberExt as _; const TARGET: &str = "test_request_timing"; @@ -218,6 +227,63 @@ mod tests { assert!(stats.stat().is_empty()); } + #[test] + fn does_not_enable_debug_at_info() { + let (timing_layer, _) = RequestTimingLayer::new(TARGET); + let info_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::INFO); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(info_layer); + let evaluated_debug_fields = AtomicUsize::new(0); + + tracing::subscriber::with_default(subscriber, || { + assert_eq!(LevelFilter::current(), LevelFilter::INFO); + debug!( + value = evaluated_debug_fields.fetch_add(1, Ordering::Relaxed), + "debug event" + ); + assert!(debug_span!("debug span").is_disabled()); + }); + + assert_eq!(evaluated_debug_fields.load(Ordering::Relaxed), 0); + } + + #[test] + fn does_not_disable_explicit_debug_logging() { + let (timing_layer, _) = RequestTimingLayer::new(TARGET); + let debug_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::DEBUG); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(debug_layer); + let evaluated_debug_fields = AtomicUsize::new(0); + + tracing::subscriber::with_default(subscriber, || { + assert_eq!(LevelFilter::current(), LevelFilter::DEBUG); + debug!( + value = evaluated_debug_fields.fetch_add(1, Ordering::Relaxed), + "debug event" + ); + assert!(!debug_span!("debug span").is_disabled()); + }); + + assert_eq!(evaluated_debug_fields.load(Ordering::Relaxed), 1); + } + + #[test] + fn records_stage_timings_when_logging_at_warn() { + let (timing_layer, stats) = RequestTimingLayer::new(TARGET); + let warn_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::WARN); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(warn_layer); + tracing::subscriber::with_default(subscriber, || { + info_span!("request", request_id = "req-1").in_scope(|| { + drop(stage_span("media_fetch")); + }); + }); + assert!(stats.stat()["req-1"].contains_key("media_fetch_secs")); + } + #[test] fn other_targets_are_ignored() { let (layer, stats) = RequestTimingLayer::new(TARGET); diff --git a/setup.py b/setup.py index ed9616195f..67f192cdd3 100644 --- a/setup.py +++ b/setup.py @@ -1529,7 +1529,7 @@ setup( install_requires=get_requirements(), extras_require={ # AMD Zen CPU optimizations via zentorch - "zen": ["zentorch==2.13.0.0"], + "zen": ["zentorch==2.13.0.1"], "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.3.3"], diff --git a/tests/basic_correctness/cpu_offload/__init__.py b/tests/basic_correctness/cpu_offload/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/test_cpu_offload.py b/tests/basic_correctness/cpu_offload/test_cpu_offload.py similarity index 99% rename from tests/basic_correctness/test_cpu_offload.py rename to tests/basic_correctness/cpu_offload/test_cpu_offload.py index a63f1189aa..46bdb7218b 100644 --- a/tests/basic_correctness/test_cpu_offload.py +++ b/tests/basic_correctness/cpu_offload/test_cpu_offload.py @@ -15,7 +15,7 @@ from vllm.model_executor.offloader import ( ) from vllm.v1.worker.gpu.model_runner import GPUModelRunner -from ..utils import compare_two_settings +from ...utils import compare_two_settings @pytest.mark.parametrize("disable_pin_memory", [False, True]) diff --git a/tests/basic_correctness/memory/__init__.py b/tests/basic_correctness/memory/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/memory/cumem/__init__.py b/tests/basic_correctness/memory/cumem/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/memory/cumem/test_cumem.py b/tests/basic_correctness/memory/cumem/test_cumem.py new file mode 100644 index 0000000000..135a3fc1ed --- /dev/null +++ b/tests/basic_correctness/memory/cumem/test_cumem.py @@ -0,0 +1,302 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm.device_allocator.cumem as cumem +from vllm import LLM, SamplingParams +from vllm.device_allocator import get_mem_allocator_instance +from vllm.platforms import current_platform + +from ....utils import create_new_process_for_each_test + +DEVICE_TYPE = current_platform.device_type + + +def mapped_usage(allocator) -> int: + """Bytes the allocator still has mapped on the device. + + `torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` / + `hipMemGetInfo` query, so its delta also carries whatever else on the GPU + allocates or frees while we measure. The allocator's own bookkeeping is + process-local and exact. + """ + return sum( + data.handle[1] + for data in allocator.pointer_to_data.values() + if not data.is_asleep + ) + + +def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None: + """Wake discarded allocations with deterministic nonzero contents.""" + original_create_and_map = cumem.create_and_map + + def create_and_map_with_poison(handle) -> None: + original_create_and_map(handle) + _, size, ptr, _ = handle + cumem.libcudart.cudaMemset(ptr, byte_value, size) + + cumem.create_and_map = create_and_map_with_poison + try: + allocator.wake_up() + finally: + cumem.create_and_map = original_create_and_map + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_python_error(): + """Test if Python error occurs when there's low-level + error happening from the C++ side. + """ + allocator = get_mem_allocator_instance() + total_bytes = torch.accelerator.get_memory_info()[1] + alloc_bytes = int(total_bytes * 0.7) + tensors = [] + with allocator.use_memory_pool(): + # allocate 70% of the total memory + x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) + tensors.append(x) + # release the memory + allocator.sleep(offload_tags=()) + + # allocate more memory than the total memory + y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) + tensors.append(y) + with pytest.raises(RuntimeError): + # when the allocator is woken up, it should raise an error + # because we don't have enough memory + allocator.wake_up() + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_basic_cumem(): + # some tensors from default memory pool + shape = (1024, 1024) + x = torch.empty(shape, device=DEVICE_TYPE) + x.zero_() + + # some tensors from custom memory pool + allocator = get_mem_allocator_instance() + with allocator.use_memory_pool(): + # custom memory pool + y = torch.empty(shape, device=DEVICE_TYPE) + y.zero_() + y += 1 + z = torch.empty(shape, device=DEVICE_TYPE) + z.zero_() + z += 2 + + # they can be used together + output = x + y + z + assert torch.allclose(output, torch.ones_like(output) * 3) + + # Track the allocator's own mapped bytes, not device-wide free memory. + mapped_bytes = mapped_usage(allocator) + assert mapped_bytes >= y.nbytes + z.nbytes + allocator.sleep() + assert mapped_usage(allocator) == 0 + allocator.wake_up() + assert mapped_usage(allocator) == mapped_bytes + + # they can be used together + output = x + y + z + assert torch.allclose(output, torch.ones_like(output) * 3) + + +@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch): + """Rejected release and KV-only/full sleep cycles preserve greedy tokens.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + kv_cache_memory_bytes = 256 * 1024 * 1024 + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=kv_cache_memory_bytes, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes_by_tag(worker): + allocator = get_mem_allocator_instance() + mapped_bytes = mapped_usage(allocator) + kv_cache_bytes = sum( + data.handle[1] + for data in allocator.pointer_to_data.values() + if data.tag == "kv_cache" and not data.is_asleep + ) + return mapped_bytes, kv_cache_bytes + + mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0] + # Utility RPC transports the engine error as a plain Exception. + with pytest.raises(Exception, match="requires a completed pause"): + llm.release_kv_cache_memory() + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before + assert not llm.llm_engine.is_sleeping() + assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected + + llm.sleep(level=0) + llm.release_kv_cache_memory() + mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0] + assert kv_cache_before > 0 + assert kv_cache_after == 0 + assert mapped_before - mapped_after >= kv_cache_before + assert mapped_after > 0 + assert llm.llm_engine.is_sleeping() + + if full_sleep: + llm.sleep(level=full_sleep) + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0 + llm.wake_up(tags=None if full_sleep else ["kv_cache"]) + if full_sleep == 2: + llm.collective_rpc("reload_weights") + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + +@pytest.mark.parametrize("first_level", [1, 2]) +@pytest.mark.parametrize("second_level", [1, 2]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch): + """Repeated sleep after KV-only wake preserves mappings and recoverability.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=256 * 1024 * 1024, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + llm.sleep(level=first_level) + llm.wake_up(tags=["kv_cache"]) + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before > 0 + llm.sleep(level=second_level) + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + llm.wake_up(tags=["weights"]) + if first_level == 2: + llm.collective_rpc("reload_weights") + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_discard_tags(): + """Test that discard(tags) selectively frees GPU memory for specific + tags while keeping other tags mapped and usable.""" + allocator = get_mem_allocator_instance() + + with allocator.use_memory_pool("weights"): + weights = torch.ones(1024, 1024, device=DEVICE_TYPE) + + with allocator.use_memory_pool("kv_cache"): + kv = torch.ones(512, 512, device=DEVICE_TYPE) + + mapped_bytes = mapped_usage(allocator) + + # Discard kv_cache only — weights should remain valid + allocator.discard("kv_cache") + + mapped_bytes_after_discard = mapped_usage(allocator) + assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes + assert mapped_bytes_after_discard >= weights.nbytes + + # Weights are still usable + assert torch.allclose(weights, torch.ones_like(weights)) + + # Wake up and verify kv_cache is remapped; discarded contents are undefined. + allocator.wake_up() + assert kv.shape == (512, 512) + + # Full sleep/wake cycle still works after discard + allocator.sleep(offload_tags="weights") + allocator.wake_up() + assert torch.allclose(weights, torch.ones_like(weights)) + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") +def test_level2_discards_ordinary_tensor_with_weights_tag(): + """Reproduce the level-2 variant for an ordinary tensor in weights.""" + allocator = get_mem_allocator_instance() + + with allocator.use_memory_pool("weights"): + fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE) + ordinary_tensor = torch.full( + (4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE + ) + + pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) + allocator.sleep(offload_tags=()) + _wake_up_with_poisoned_mappings(allocator) + torch.accelerator.synchronize() + + assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers + assert torch.all(fake_weight == 0xA5) + assert torch.all(ordinary_tensor == 0xA5) + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU") +def test_cumem_with_cudagraph(): + allocator = get_mem_allocator_instance() + with allocator.use_memory_pool(): + weight = torch.eye(1024, device=DEVICE_TYPE) + with allocator.use_memory_pool(tag="discard"): + cache = torch.empty(1024, 1024, device=DEVICE_TYPE) + + def model(x): + out = x @ weight + cache[: out.size(0)].copy_(out) + return out + 1 + + x = torch.empty(128, 1024, device=DEVICE_TYPE) + + # warmup + model(x) + + # capture cudagraph + model_graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(model_graph): + y = model(x) + + mapped_bytes = mapped_usage(allocator) + assert mapped_bytes >= weight.nbytes + cache.nbytes + allocator.sleep() + assert mapped_usage(allocator) == 0 + allocator.wake_up() + assert mapped_usage(allocator) == mapped_bytes + + # after waking up, the content in the weight tensor + # should be restored, but the content in the cache tensor + # should be discarded + + # this operation is also compatible with cudagraph + + x.random_() + model_graph.replay() + + # cache content is as expected + assert torch.allclose(x, cache[: x.size(0)]) + + # output content is as expected + assert torch.allclose(y, x + 1) diff --git a/tests/basic_correctness/memory/sleep_mode/__init__.py b/tests/basic_correctness/memory/sleep_mode/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py new file mode 100644 index 0000000000..903fc7d3cb --- /dev/null +++ b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio + +import pytest +import torch + +import vllm.device_allocator.cumem as cumem +import vllm.envs as envs +from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams +from vllm.platforms import current_platform +from vllm.utils.mem_constants import GiB_bytes + +from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8 + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.parametrize( + "model", + [ + # sleep mode with safetensors + "hmellor/tiny-random-LlamaForCausalLM", + # sleep mode with pytorch checkpoint + "facebook/opt-125m", + ], +) +def test_end_to_end(model: str): + free, total = torch.accelerator.get_memory_info() + used_bytes_baseline = total - free # in case other process is running + llm = LLM(model, enable_sleep_mode=True) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + output = llm.generate(prompt, sampling_params) + + # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage, + # which is difficult to measure in the test. therefore, we only + # test sleep level 1 here. + llm.sleep(level=1) + + free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info() + used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline + # now the memory usage is mostly cudagraph memory pool, + # and it should be less than the model weights (1B model, 2GiB weights) + + # NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size) + # is captured but cannot be releasesd from PyTorch due to a known bug, + # therefore high memory usage after `llm.sleep` is called is expected. + # FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode + # in V1. + assert used_bytes < 7 * GiB_bytes + + llm.wake_up() + output2 = llm.generate(prompt, sampling_params) + # cmp output + assert output[0].outputs[0].text == output2[0].outputs[0].text + + llm.sleep(level=1) + llm.wake_up(tags=["weights"]) + + free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() + used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline + + # should just reallocate memory for weights (1B model, ~2GiB weights) + assert used_bytes < 10 * GiB_bytes + + # now allocate kv cache memory + llm.wake_up(tags=["kv_cache"]) + output3 = llm.generate(prompt, sampling_params) + + # cmp output + assert output[0].outputs[0].text == output3[0].outputs[0].text + + +@create_new_process_for_each_test() +def test_deep_sleep(gpu_memory_cleared): + model = "hmellor/tiny-random-LlamaForCausalLM" + free, total = torch.accelerator.get_memory_info() + used_bytes_baseline = total - free # in case other process is running + llm = LLM(model, enable_sleep_mode=True) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + output = llm.generate(prompt, sampling_params) + + # Put the engine to deep sleep + llm.sleep(level=2) + + free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info() + used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline + assert used_bytes < 3 * GiB_bytes + + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() + used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline + assert used_bytes < 4 * GiB_bytes + + # now allocate kv cache and cuda graph memory + llm.wake_up(tags=["kv_cache"]) + output2 = llm.generate(prompt, sampling_params) + + # cmp output + assert output[0].outputs[0].text == output2[0].outputs[0].text + + +@create_new_process_for_each_test() +def test_deep_sleep_lora(gpu_memory_cleared): + """Level-2 sleep/wake/reload with enable_lora=True. + + LoRA wrapping moves parameters under base_layer and adds LoRA + stacked tensors that are plain attributes, not restored by the + reload machinery — reload must forward checkpoint weights through + the wrappers and reset the LoRA state afterwards. + """ + model = "hmellor/tiny-random-LlamaForCausalLM" + llm = LLM( + model, + enable_sleep_mode=True, + enable_lora=True, + max_lora_rank=8, + enforce_eager=True, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + output = llm.generate(prompt, sampling_params) + + # Level-2 sleep discards all GPU memory + llm.sleep(level=2) + + # Reload weights from checkpoint + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + llm.wake_up(tags=["kv_cache"]) + output2 = llm.generate(prompt, sampling_params) + assert output[0].outputs[0].text == output2[0].outputs[0].text + + # Multiple cycles should not accumulate corruption + for _ in range(3): + llm.sleep(level=2) + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + llm.wake_up(tags=["kv_cache"]) + output3 = llm.generate(prompt, sampling_params) + assert output[0].outputs[0].text == output3[0].outputs[0].text + + +def _lora_logits_mapping_present(model) -> bool: + from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA + + return any( + isinstance(m, LogitsProcessorWithLoRA) + and m.sharded_to_full_mapping_gpu is not None + for m in model.modules() + ) + + +@multi_gpu_test(num_gpus=2) +def test_deep_sleep_lora_tp2(monkeypatch): + """Level-2 sleep/wake/reload with enable_lora=True and TP=2. + + With TP > 1 the LoRA logits processor carries + ``sharded_to_full_mapping_gpu``, a permanent index mapping used to + reorder gathered logits. Like the LoRA stacked tensors it is a plain + attribute allocated in the sleep-mode pool, so level-2 sleep destroys + its contents — it must be restored after reload. + """ + # Needed for apply_model to reach the multiproc TP workers below. + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + model = "hmellor/tiny-random-LlamaForCausalLM" + llm = LLM( + model, + enable_sleep_mode=True, + enable_lora=True, + max_lora_rank=8, + tensor_parallel_size=2, + enforce_eager=True, + ) + + # Guard against this test silently not exercising the TP>1 reindex + # path (e.g. if lm_head wrapping conditions change). + assert all(llm.apply_model(_lora_logits_mapping_present)) + + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + output = llm.generate(prompt, sampling_params) + + llm.sleep(level=2) + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + llm.wake_up(tags=["kv_cache"]) + output2 = llm.generate(prompt, sampling_params) + assert output[0].outputs[0].text == output2[0].outputs[0].text + + +@create_new_process_for_each_test() +def test_deep_sleep_async(gpu_memory_cleared): + async def test(): + model = "hmellor/tiny-random-LlamaForCausalLM" + free, total = torch.accelerator.get_memory_info() + used_bytes_baseline = total - free # in case other process is running + engine_args = AsyncEngineArgs( + model=model, + enable_sleep_mode=True, + ) + + llm = AsyncLLMEngine.from_engine_args(engine_args) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + outputs = llm.generate(prompt, sampling_params, request_id="test_request_id1") + async for output in outputs: + pass + + # Put the engine to deep sleep + await llm.sleep(level=2) + + await llm.wake_up(tags=["weights"]) + await llm.collective_rpc("reload_weights") + free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() + used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline + assert used_bytes < 4 * GiB_bytes + + # now allocate kv cache and cuda graph memory + await llm.wake_up(tags=["kv_cache"]) + outputs2 = llm.generate(prompt, sampling_params, request_id="test_request_id2") + async for output2 in outputs2: + pass + + # cmp output + assert output.outputs[0].text == output2.outputs[0].text + + asyncio.run(test()) + + +@requires_fp8 +def test_deep_sleep_fp8_kvcache_mrv1( + monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared +): + # Regression test for https://github.com/vllm-project/vllm/pull/28783. + # In particular, verify that MRV1 does not rely on post_kv_cache_wake_up() + # to restore correct output after level-2 sleep. + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") + envs.disable_envs_cache() + + model = "Qwen/Qwen2-0.5B" + used_bytes_baseline = current_platform.get_current_memory_usage() + + llm = LLM(model, enable_sleep_mode=True, kv_cache_dtype="fp8") + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + output = llm.generate(prompt, sampling_params) + + # Put the engine to deep sleep + llm.sleep(level=2) + + used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline + + # Rocm uses more memory for CudaGraphs, so we add 2 GiB more for the threshold + rocm_extra_mem_bytes = 2 * GiB_bytes if current_platform.is_rocm() else 0 + mem_threshold_after_sleep = 3 * GiB_bytes + rocm_extra_mem_bytes + assert used_bytes < mem_threshold_after_sleep + + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + + used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline + mem_threshold_after_wake_up = 4 * GiB_bytes + rocm_extra_mem_bytes + assert used_bytes < mem_threshold_after_wake_up + + # now allocate kv cache and cuda graph memory + llm.wake_up(tags=["kv_cache"]) + output2 = llm.generate(prompt, sampling_params) + + # cmp output + assert output[0].outputs[0].text == output2[0].outputs[0].text + + +@requires_fp8 +def test_deep_sleep_fp8_kvcache_mrv1_with_undefined_remap( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + envs.disable_envs_cache() + + llm = LLM( + "Qwen/Qwen2-0.5B", + enable_sleep_mode=True, + kv_cache_dtype="fp8", + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params) + + llm.sleep(level=2) + llm.wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights") + + original_create_and_map = cumem.create_and_map + + def create_and_map_with_poison(handle) -> None: + original_create_and_map(handle) + _, size, ptr, _ = handle + cumem.libcudart.cudaMemset(ptr, 0xA5, size) + + monkeypatch.setattr(cumem, "create_and_map", create_and_map_with_poison) + + # New requests must overwrite undefined remapped KV bytes before reading them. + llm.wake_up(tags=["kv_cache"]) + actual = llm.generate(prompt, sampling_params) + + assert expected[0].outputs[0].text == actual[0].outputs[0].text diff --git a/tests/basic_correctness/models/__init__.py b/tests/basic_correctness/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/test_basic_correctness.py b/tests/basic_correctness/models/test_basic_correctness.py similarity index 98% rename from tests/basic_correctness/test_basic_correctness.py rename to tests/basic_correctness/models/test_basic_correctness.py index 688d355c31..47777d6740 100644 --- a/tests/basic_correctness/test_basic_correctness.py +++ b/tests/basic_correctness/models/test_basic_correctness.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Compare the short outputs of HF and vLLM when using greedy sampling. -Run `pytest tests/basic_correctness/test_basic_correctness.py`. +Run `pytest tests/basic_correctness/models/test_basic_correctness.py`. """ import os @@ -17,9 +17,9 @@ from vllm import LLM from vllm.platforms import current_platform from vllm.v1.engine.llm_engine import LLMEngine -from ..conftest import HfRunner, VllmRunner -from ..models.utils import check_outputs_equal -from ..utils import multi_gpu_test +from ...conftest import HfRunner, VllmRunner +from ...models.utils import check_outputs_equal +from ...utils import multi_gpu_test ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"] diff --git a/tests/basic_correctness/prefetch_offload/__init__.py b/tests/basic_correctness/prefetch_offload/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/basic_correctness/test_prefetch_offload.py b/tests/basic_correctness/prefetch_offload/test_prefetch_offload.py similarity index 95% rename from tests/basic_correctness/test_prefetch_offload.py rename to tests/basic_correctness/prefetch_offload/test_prefetch_offload.py index 498887024e..a127ee34a4 100644 --- a/tests/basic_correctness/test_prefetch_offload.py +++ b/tests/basic_correctness/prefetch_offload/test_prefetch_offload.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Test prefetch offloading correctness with Llama model.""" -from ..utils import compare_two_settings +from ...utils import compare_two_settings def test_prefetch_offload_llama(): diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py deleted file mode 100644 index 6a483d26aa..0000000000 --- a/tests/basic_correctness/test_mem.py +++ /dev/null @@ -1,604 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import asyncio - -import pytest -import torch - -import vllm.device_allocator.cumem as cumem -import vllm.envs as envs -from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams -from vllm.device_allocator import get_mem_allocator_instance -from vllm.platforms import current_platform -from vllm.utils.mem_constants import GiB_bytes - -from ..utils import create_new_process_for_each_test, requires_fp8 - -DEVICE_TYPE = current_platform.device_type - - -def mapped_usage(allocator) -> int: - """Bytes the allocator still has mapped on the device. - - `torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` / - `hipMemGetInfo` query, so its delta also carries whatever else on the GPU - allocates or frees while we measure. The allocator's own bookkeeping is - process-local and exact. - """ - return sum( - data.handle[1] - for data in allocator.pointer_to_data.values() - if not data.is_asleep - ) - - -def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None: - """Wake discarded allocations with deterministic nonzero contents.""" - original_create_and_map = cumem.create_and_map - - def create_and_map_with_poison(handle) -> None: - original_create_and_map(handle) - _, size, ptr, _ = handle - cumem.libcudart.cudaMemset(ptr, byte_value, size) - - cumem.create_and_map = create_and_map_with_poison - try: - allocator.wake_up() - finally: - cumem.create_and_map = original_create_and_map - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -def test_python_error(): - """Test if Python error occurs when there's low-level - error happening from the C++ side. - """ - allocator = get_mem_allocator_instance() - total_bytes = torch.accelerator.get_memory_info()[1] - alloc_bytes = int(total_bytes * 0.7) - tensors = [] - with allocator.use_memory_pool(): - # allocate 70% of the total memory - x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) - tensors.append(x) - # release the memory - allocator.sleep(offload_tags=()) - - # allocate more memory than the total memory - y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) - tensors.append(y) - with pytest.raises(RuntimeError): - # when the allocator is woken up, it should raise an error - # because we don't have enough memory - allocator.wake_up() - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -def test_basic_cumem(): - # some tensors from default memory pool - shape = (1024, 1024) - x = torch.empty(shape, device=DEVICE_TYPE) - x.zero_() - - # some tensors from custom memory pool - allocator = get_mem_allocator_instance() - with allocator.use_memory_pool(): - # custom memory pool - y = torch.empty(shape, device=DEVICE_TYPE) - y.zero_() - y += 1 - z = torch.empty(shape, device=DEVICE_TYPE) - z.zero_() - z += 2 - - # they can be used together - output = x + y + z - assert torch.allclose(output, torch.ones_like(output) * 3) - - # Track the allocator's own mapped bytes, not device-wide free memory. - mapped_bytes = mapped_usage(allocator) - assert mapped_bytes >= y.nbytes + z.nbytes - allocator.sleep() - assert mapped_usage(allocator) == 0 - allocator.wake_up() - assert mapped_usage(allocator) == mapped_bytes - - # they can be used together - output = x + y + z - assert torch.allclose(output, torch.ones_like(output) * 3) - - -@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"]) -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch): - """Rejected release and KV-only/full sleep cycles preserve greedy tokens.""" - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - kv_cache_memory_bytes = 256 * 1024 * 1024 - llm = LLM( - "Qwen/Qwen3-0.6B", - enable_sleep_mode=True, - enforce_eager=True, - kv_cache_memory_bytes=kv_cache_memory_bytes, - max_model_len=1024, - max_num_seqs=4, - gpu_memory_utilization=0.05, - ) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids - - def get_mapped_bytes_by_tag(worker): - allocator = get_mem_allocator_instance() - mapped_bytes = mapped_usage(allocator) - kv_cache_bytes = sum( - data.handle[1] - for data in allocator.pointer_to_data.values() - if data.tag == "kv_cache" and not data.is_asleep - ) - return mapped_bytes, kv_cache_bytes - - mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0] - # Utility RPC transports the engine error as a plain Exception. - with pytest.raises(Exception, match="requires a completed pause"): - llm.release_kv_cache_memory() - assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before - assert not llm.llm_engine.is_sleeping() - assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected - - llm.sleep(level=0) - llm.release_kv_cache_memory() - mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0] - assert kv_cache_before > 0 - assert kv_cache_after == 0 - assert mapped_before - mapped_after >= kv_cache_before - assert mapped_after > 0 - assert llm.llm_engine.is_sleeping() - - if full_sleep: - llm.sleep(level=full_sleep) - assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0 - llm.wake_up(tags=None if full_sleep else ["kv_cache"]) - if full_sleep == 2: - llm.collective_rpc("reload_weights") - assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before - assert not llm.llm_engine.is_sleeping() - actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids - assert actual == expected - - -@pytest.mark.parametrize("first_level", [1, 2]) -@pytest.mark.parametrize("second_level", [1, 2]) -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch): - """Repeated sleep after KV-only wake preserves mappings and recoverability.""" - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - llm = LLM( - "Qwen/Qwen3-0.6B", - enable_sleep_mode=True, - enforce_eager=True, - kv_cache_memory_bytes=256 * 1024 * 1024, - max_model_len=1024, - max_num_seqs=4, - gpu_memory_utilization=0.05, - ) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids - - def get_mapped_bytes(worker): - return mapped_usage(get_mem_allocator_instance()) - - llm.sleep(level=first_level) - llm.wake_up(tags=["kv_cache"]) - mapped_before = llm.collective_rpc(get_mapped_bytes)[0] - assert mapped_before > 0 - llm.sleep(level=second_level) - assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before - llm.wake_up(tags=["weights"]) - if first_level == 2: - llm.collective_rpc("reload_weights") - assert not llm.llm_engine.is_sleeping() - actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids - assert actual == expected - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -def test_discard_tags(): - """Test that discard(tags) selectively frees GPU memory for specific - tags while keeping other tags mapped and usable.""" - allocator = get_mem_allocator_instance() - - with allocator.use_memory_pool("weights"): - weights = torch.ones(1024, 1024, device=DEVICE_TYPE) - - with allocator.use_memory_pool("kv_cache"): - kv = torch.ones(512, 512, device=DEVICE_TYPE) - - mapped_bytes = mapped_usage(allocator) - - # Discard kv_cache only — weights should remain valid - allocator.discard("kv_cache") - - mapped_bytes_after_discard = mapped_usage(allocator) - assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes - assert mapped_bytes_after_discard >= weights.nbytes - - # Weights are still usable - assert torch.allclose(weights, torch.ones_like(weights)) - - # Wake up and verify kv_cache is remapped; discarded contents are undefined. - allocator.wake_up() - assert kv.shape == (512, 512) - - # Full sleep/wake cycle still works after discard - allocator.sleep(offload_tags="weights") - allocator.wake_up() - assert torch.allclose(weights, torch.ones_like(weights)) - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") -def test_level2_discards_ordinary_tensor_with_weights_tag(): - """Reproduce the level-2 variant for an ordinary tensor in weights.""" - allocator = get_mem_allocator_instance() - - with allocator.use_memory_pool("weights"): - fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE) - ordinary_tensor = torch.full( - (4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE - ) - - pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) - allocator.sleep(offload_tags=()) - _wake_up_with_poisoned_mappings(allocator) - torch.accelerator.synchronize() - - assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers - assert torch.all(fake_weight == 0xA5) - assert torch.all(ordinary_tensor == 0xA5) - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU") -def test_cumem_with_cudagraph(): - allocator = get_mem_allocator_instance() - with allocator.use_memory_pool(): - weight = torch.eye(1024, device=DEVICE_TYPE) - with allocator.use_memory_pool(tag="discard"): - cache = torch.empty(1024, 1024, device=DEVICE_TYPE) - - def model(x): - out = x @ weight - cache[: out.size(0)].copy_(out) - return out + 1 - - x = torch.empty(128, 1024, device=DEVICE_TYPE) - - # warmup - model(x) - - # capture cudagraph - model_graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(model_graph): - y = model(x) - - mapped_bytes = mapped_usage(allocator) - assert mapped_bytes >= weight.nbytes + cache.nbytes - allocator.sleep() - assert mapped_usage(allocator) == 0 - allocator.wake_up() - assert mapped_usage(allocator) == mapped_bytes - - # after waking up, the content in the weight tensor - # should be restored, but the content in the cache tensor - # should be discarded - - # this operation is also compatible with cudagraph - - x.random_() - model_graph.replay() - - # cache content is as expected - assert torch.allclose(x, cache[: x.size(0)]) - - # output content is as expected - assert torch.allclose(y, x + 1) - - -@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") -@pytest.mark.parametrize( - "model", - [ - # sleep mode with safetensors - "hmellor/tiny-random-LlamaForCausalLM", - # sleep mode with pytorch checkpoint - "facebook/opt-125m", - ], -) -def test_end_to_end(model: str): - free, total = torch.accelerator.get_memory_info() - used_bytes_baseline = total - free # in case other process is running - llm = LLM(model, enable_sleep_mode=True) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - output = llm.generate(prompt, sampling_params) - - # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage, - # which is difficult to measure in the test. therefore, we only - # test sleep level 1 here. - llm.sleep(level=1) - - free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info() - used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline - # now the memory usage is mostly cudagraph memory pool, - # and it should be less than the model weights (1B model, 2GiB weights) - - # NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size) - # is captured but cannot be releasesd from PyTorch due to a known bug, - # therefore high memory usage after `llm.sleep` is called is expected. - # FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode - # in V1. - assert used_bytes < 7 * GiB_bytes - - llm.wake_up() - output2 = llm.generate(prompt, sampling_params) - # cmp output - assert output[0].outputs[0].text == output2[0].outputs[0].text - - llm.sleep(level=1) - llm.wake_up(tags=["weights"]) - - free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() - used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline - - # should just reallocate memory for weights (1B model, ~2GiB weights) - assert used_bytes < 10 * GiB_bytes - - # now allocate kv cache memory - llm.wake_up(tags=["kv_cache"]) - output3 = llm.generate(prompt, sampling_params) - - # cmp output - assert output[0].outputs[0].text == output3[0].outputs[0].text - - -@create_new_process_for_each_test() -def test_deep_sleep(): - model = "hmellor/tiny-random-LlamaForCausalLM" - free, total = torch.accelerator.get_memory_info() - used_bytes_baseline = total - free # in case other process is running - llm = LLM(model, enable_sleep_mode=True) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - output = llm.generate(prompt, sampling_params) - - # Put the engine to deep sleep - llm.sleep(level=2) - - free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info() - used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline - assert used_bytes < 3 * GiB_bytes - - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() - used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline - assert used_bytes < 4 * GiB_bytes - - # now allocate kv cache and cuda graph memory - llm.wake_up(tags=["kv_cache"]) - output2 = llm.generate(prompt, sampling_params) - - # cmp output - assert output[0].outputs[0].text == output2[0].outputs[0].text - - -@create_new_process_for_each_test() -def test_deep_sleep_lora(): - """Level-2 sleep/wake/reload with enable_lora=True. - - LoRA wrapping moves parameters under base_layer and adds LoRA - stacked tensors that are plain attributes, not restored by the - reload machinery — reload must forward checkpoint weights through - the wrappers and reset the LoRA state afterwards. - """ - model = "hmellor/tiny-random-LlamaForCausalLM" - llm = LLM( - model, - enable_sleep_mode=True, - enable_lora=True, - max_lora_rank=8, - enforce_eager=True, - ) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - output = llm.generate(prompt, sampling_params) - - # Level-2 sleep discards all GPU memory - llm.sleep(level=2) - - # Reload weights from checkpoint - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - llm.wake_up(tags=["kv_cache"]) - output2 = llm.generate(prompt, sampling_params) - assert output[0].outputs[0].text == output2[0].outputs[0].text - - # Multiple cycles should not accumulate corruption - for _ in range(3): - llm.sleep(level=2) - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - llm.wake_up(tags=["kv_cache"]) - output3 = llm.generate(prompt, sampling_params) - assert output[0].outputs[0].text == output3[0].outputs[0].text - - -def _lora_logits_mapping_present(model) -> bool: - from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA - - return any( - isinstance(m, LogitsProcessorWithLoRA) - and m.sharded_to_full_mapping_gpu is not None - for m in model.modules() - ) - - -@create_new_process_for_each_test() -def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch): - """Level-2 sleep/wake/reload with enable_lora=True and TP=2. - - With TP > 1 the LoRA logits processor carries - ``sharded_to_full_mapping_gpu``, a permanent index mapping used to - reorder gathered logits. Like the LoRA stacked tensors it is a plain - attribute allocated in the sleep-mode pool, so level-2 sleep destroys - its contents — it must be restored after reload. - """ - if num_gpus_available < 2: - pytest.skip("Requires at least 2 GPUs") - - # Needed for apply_model to reach the multiproc TP workers below. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - model = "hmellor/tiny-random-LlamaForCausalLM" - llm = LLM( - model, - enable_sleep_mode=True, - enable_lora=True, - max_lora_rank=8, - tensor_parallel_size=2, - enforce_eager=True, - ) - - # Guard against this test silently not exercising the TP>1 reindex - # path (e.g. if lm_head wrapping conditions change). - assert all(llm.apply_model(_lora_logits_mapping_present)) - - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - output = llm.generate(prompt, sampling_params) - - llm.sleep(level=2) - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - llm.wake_up(tags=["kv_cache"]) - output2 = llm.generate(prompt, sampling_params) - assert output[0].outputs[0].text == output2[0].outputs[0].text - - -@create_new_process_for_each_test() -def test_deep_sleep_async(): - async def test(): - model = "hmellor/tiny-random-LlamaForCausalLM" - free, total = torch.accelerator.get_memory_info() - used_bytes_baseline = total - free # in case other process is running - engine_args = AsyncEngineArgs( - model=model, - enable_sleep_mode=True, - ) - - llm = AsyncLLMEngine.from_engine_args(engine_args) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - outputs = llm.generate(prompt, sampling_params, request_id="test_request_id1") - async for output in outputs: - pass - - # Put the engine to deep sleep - await llm.sleep(level=2) - - await llm.wake_up(tags=["weights"]) - await llm.collective_rpc("reload_weights") - free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info() - used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline - assert used_bytes < 4 * GiB_bytes - - # now allocate kv cache and cuda graph memory - await llm.wake_up(tags=["kv_cache"]) - outputs2 = llm.generate(prompt, sampling_params, request_id="test_request_id2") - async for output2 in outputs2: - pass - - # cmp output - assert output.outputs[0].text == output2.outputs[0].text - - asyncio.run(test()) - - -@requires_fp8 -def test_deep_sleep_fp8_kvcache_mrv1(monkeypatch: pytest.MonkeyPatch): - # Regression test for https://github.com/vllm-project/vllm/pull/28783. - # In particular, verify that MRV1 does not rely on post_kv_cache_wake_up() - # to restore correct output after level-2 sleep. - monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") - envs.disable_envs_cache() - - model = "Qwen/Qwen2-0.5B" - used_bytes_baseline = current_platform.get_current_memory_usage() - - llm = LLM(model, enable_sleep_mode=True, kv_cache_dtype="fp8") - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - output = llm.generate(prompt, sampling_params) - - # Put the engine to deep sleep - llm.sleep(level=2) - - used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline - - # Rocm uses more memory for CudaGraphs, so we add 2 GiB more for the threshold - rocm_extra_mem_bytes = 2 * GiB_bytes if current_platform.is_rocm() else 0 - mem_threshold_after_sleep = 3 * GiB_bytes + rocm_extra_mem_bytes - assert used_bytes < mem_threshold_after_sleep - - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - - used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline - mem_threshold_after_wake_up = 4 * GiB_bytes + rocm_extra_mem_bytes - assert used_bytes < mem_threshold_after_wake_up - - # now allocate kv cache and cuda graph memory - llm.wake_up(tags=["kv_cache"]) - output2 = llm.generate(prompt, sampling_params) - - # cmp output - assert output[0].outputs[0].text == output2[0].outputs[0].text - - -@requires_fp8 -def test_deep_sleep_fp8_kvcache_mrv1_with_undefined_remap( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") - monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") - envs.disable_envs_cache() - - llm = LLM( - "Qwen/Qwen2-0.5B", - enable_sleep_mode=True, - kv_cache_dtype="fp8", - ) - prompt = "How are you?" - sampling_params = SamplingParams(temperature=0, max_tokens=10) - expected = llm.generate(prompt, sampling_params) - - llm.sleep(level=2) - llm.wake_up(tags=["weights"]) - llm.collective_rpc("reload_weights") - - original_create_and_map = cumem.create_and_map - - def create_and_map_with_poison(handle) -> None: - original_create_and_map(handle) - _, size, ptr, _ = handle - cumem.libcudart.cudaMemset(ptr, 0xA5, size) - - monkeypatch.setattr(cumem, "create_and_map", create_and_map_with_poison) - - # New requests must overwrite undefined remapped KV bytes before reading them. - llm.wake_up(tags=["kv_cache"]) - actual = llm.generate(prompt, sampling_params) - - assert expected[0].outputs[0].text == actual[0].outputs[0].text diff --git a/tests/compile/correctness_e2e/test_async_tp.py b/tests/compile/correctness_e2e/test_async_tp.py index 97248d153f..6b8c26cadc 100644 --- a/tests/compile/correctness_e2e/test_async_tp.py +++ b/tests/compile/correctness_e2e/test_async_tp.py @@ -2,18 +2,22 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +import logging import pytest +from tests.compile.fusions_e2e.common import FUSION_LOG_PATTERNS from tests.models.registry import HF_EXAMPLE_MODELS from tests.utils import ( compare_two_settings, create_new_process_for_each_test, + multi_gpu_test, ) from vllm.config import ( CompilationMode, ) from vllm.platforms import current_platform +from vllm.transformers_utils.repo_utils import hf_api from vllm.utils.flashinfer import has_flashinfer NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4" @@ -26,6 +30,78 @@ NVFP4_HF_OVERRIDES = { } +@multi_gpu_test(num_gpus=2) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract") +def test_rocm_async_tp_bf16_output_correctness(caplog_mp_spawn): + """Trained-model outputs must agree with unfused TP, including padded decode.""" + # A snapshot path pins both servers and the comparison helper's tokenizer. + model = hf_api().snapshot_download( + "Qwen/Qwen3-0.6B", revision="c1899de289a04d12100db370d81485cdf75e47ca" + ) + common_args = [ + "--dtype", + "bfloat16", + "--load-format", + "auto", + "--tensor-parallel-size", + "2", + "--distributed-executor-backend", + "mp", + "--max-model-len", + "512", + "--max-num-batched-tokens", + "512", + "--max-num-seqs", + "8", + "--kv-cache-memory-bytes", + str(64 * 1024**2), + "--gpu-memory-utilization", + "0.01", + "--attention-backend", + "TRITON_ATTN", + "--disable-custom-all-reduce", + ] + + def settings(enabled): + return [ + *common_args, + "--compilation-config", + json.dumps( + { + "mode": CompilationMode.VLLM_COMPILE, + "cudagraph_mode": "NONE", + "splitting_ops": [], + "inductor_compile_config": {"force_disable_caches": True}, + "pass_config": { + "enable_sp": enabled, + "fuse_gemm_comms": enabled, + "fuse_allreduce_rms": False, + "sp_min_token_num": 1, + }, + } + ), + ] + + env = {"VLLM_ROCM_USE_AITER": "0", "VLLM_DISABLE_COMPILE_CACHE": "1"} + with caplog_mp_spawn(logging.DEBUG) as logs: + compare_two_settings( + model, + settings(False), + settings(True), + env1=env, + env2=env, + method="generate", + force_v1_runner=True, + ) + matches = [ + int(count) for count in FUSION_LOG_PATTERNS["async_tp"].findall(logs.text) + ] + print(f"AsyncTP rewrite counts: {matches}") + assert len(matches) >= 2 and all(count > 0 for count in matches), ( + f"Expected positive AsyncTP rewrites on both TP workers, found {matches}" + ) + + @create_new_process_for_each_test() @pytest.mark.parametrize( "model_id", diff --git a/tests/compile/correctness_e2e/test_attn_quant.py b/tests/compile/correctness_e2e/test_attn_quant.py new file mode 100644 index 0000000000..c7fc849ae5 --- /dev/null +++ b/tests/compile/correctness_e2e/test_attn_quant.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +import logging + +import pytest +import regex as re + +from tests.utils import compare_two_settings +from vllm._aiter_ops import is_aiter_found_and_supported +from vllm.config import CompilationMode +from vllm.platforms import current_platform +from vllm.transformers_utils.repo_utils import hf_api + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not is_aiter_found_and_supported(), + reason="Requires ROCm AITER attention and quantization", +) +def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn): + """Trained FP8 outputs agree when actual AITER output quantization is fused.""" + model = hf_api().snapshot_download( + "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8", + revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a", + allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"], + ) + common_args = [ + "--dtype", + "bfloat16", + "--load-format", + "auto", + "--max-model-len", + "512", + "--max-num-batched-tokens", + "512", + "--max-num-seqs", + "8", + "--kv-cache-memory-bytes", + str(256 * 1024**2), + "--gpu-memory-utilization", + "0.01", + "--attention-backend", + "ROCM_AITER_UNIFIED_ATTN", + ] + + def settings(enabled): + return [ + *common_args, + "--compilation-config", + json.dumps( + { + "mode": CompilationMode.VLLM_COMPILE, + "cudagraph_mode": "NONE", + "splitting_ops": [], + "inductor_compile_config": {"force_disable_caches": True}, + "custom_ops": ["+quant_fp8", "-rms_norm"], + "pass_config": { + "fuse_attn_quant": enabled, + "fuse_norm_quant": False, + "fuse_act_quant": False, + "fuse_allreduce_rms": False, + }, + } + ), + ] + + env = { + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_USE_AITER_LINEAR": "1", + "VLLM_DISABLE_COMPILE_CACHE": "1", + } + with caplog_mp_spawn(logging.DEBUG) as logs: + compare_two_settings( + model, + settings(False), + settings(True), + env1=env, + env2=env, + method="generate", + force_v1_runner=True, + ) + matches = [ + int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text) + ] + print(f"Attention output quantization rewrite counts: {matches}") + assert matches and all(count == 32 for count in matches), ( + f"Expected output quantization fused in all 32 Llama layers, found {matches}" + ) diff --git a/tests/compile/fullgraph/test_full_cudagraph.py b/tests/compile/fullgraph/test_full_cudagraph.py index d91cd84816..4319a25c96 100644 --- a/tests/compile/fullgraph/test_full_cudagraph.py +++ b/tests/compile/fullgraph/test_full_cudagraph.py @@ -10,6 +10,7 @@ from tests.utils import wait_for_gpu_memory_to_clear from tests.v1.attention.utils import full_cg_backend_configs as backend_configs from vllm import LLM, SamplingParams from vllm.config import CompilationConfig, CUDAGraphMode +from vllm.outputs import RequestOutput from vllm.platforms import current_platform from vllm.utils.torch_utils import is_torch_equal_or_newer from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -36,7 +37,7 @@ def temporary_environ(env_vars): model_backends_full_cudagraph = [] # deepseek-ai/DeepSeek-V2-Lite with MLA -MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA"] +MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA", "FlashInferMLA"] for mla_backend in MLA_backends: model_backends_full_cudagraph.append( ("deepseek-ai/DeepSeek-V2-Lite", backend_configs[mla_backend]) @@ -53,27 +54,28 @@ for backend_config in other_backend_configs: @pytest.fixture(scope="class") def llm_pair(request): model, backend_config, use_inductor_graph_partition = request.param - backend_config.comp_config["use_inductor_graph_partition"] = ( - use_inductor_graph_partition - ) - if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"): pytest.skip("Inductor graph partition only supported in torch>=2.9") + backend = AttentionBackendEnum[backend_config.attention_config["backend"]] + if current_platform.is_rocm() and backend not in ( + AttentionBackendEnum.TRITON_ATTN, + AttentionBackendEnum.ROCM_ATTN, + ): + pytest.skip(f"{backend_config.name} is a CUDA-only attention backend") + if backend == AttentionBackendEnum.ROCM_ATTN and not current_platform.is_rocm(): + pytest.skip("ROCM_ATTN requires ROCm") + # Dynamically skip test if GPU capability is not met - if ( - backend_config.specific_gpu_arch - and backend_config.specific_gpu_arch != current_platform.get_device_capability() + if backend_config.specific_gpu_arch and ( + not current_platform.is_cuda() + or backend_config.specific_gpu_arch != current_platform.get_device_capability() ): if backend_config.specific_gpu_arch == (9, 0): pytest.skip("Only Hopper GPUs support FA3 and FlashMLA") elif backend_config.specific_gpu_arch == (10, 0): pytest.skip("Only Blackwell GPUs support Cutlass MLA") - # FlashInfer is not supported on ROCm - if backend_config == AttentionBackendEnum.FLASHINFER and current_platform.is_rocm(): - pytest.skip("FlashInfer is not supported on ROCm") - env_vars = { # Force native sampler to avoid potential nondeterminism in FlashInfer # when per-request generators are not used in V1. @@ -86,7 +88,11 @@ def llm_pair(request): trust_remote_code=True, max_model_len=1024, max_num_seqs=128, - compilation_config=CompilationConfig(**backend_config.comp_config), + attention_config=backend_config.attention_config, + compilation_config=CompilationConfig( + **backend_config.comp_config, + use_inductor_graph_partition=use_inductor_graph_partition, + ), generation_config="vllm", seed=42, ) @@ -96,8 +102,10 @@ def llm_pair(request): trust_remote_code=True, max_model_len=1024, max_num_seqs=128, + attention_config=backend_config.attention_config, compilation_config=CompilationConfig( - cudagraph_mode=CUDAGraphMode.PIECEWISE + cudagraph_mode=CUDAGraphMode.PIECEWISE, + use_inductor_graph_partition=use_inductor_graph_partition, ), generation_config="vllm", seed=42, @@ -117,7 +125,10 @@ def llm_pair(request): @pytest.mark.parametrize( "llm_pair", [ - pytest.param((model, backend_config, use_inductor_graph_partition)) + pytest.param( + (model, backend_config, use_inductor_graph_partition), + id=f"{backend_config.name}-partition={use_inductor_graph_partition}", + ) for model, backend_config in model_backends_full_cudagraph for use_inductor_graph_partition in [True, False] ], @@ -159,10 +170,20 @@ class TestFullCUDAGraph: temperature=0.0, max_tokens=max_tokens, top_p=1.0 ) - piecewise_responses = piecewise_llm.generate(prompts, sampling_params) - full_responses = full_cudagraph_llm.generate(prompts, sampling_params) + responses = [] + for llm in (piecewise_llm, full_cudagraph_llm): + # Queue the whole batch before scheduling so both engines exercise + # the requested batch size instead of timing-dependent sub-batches. + llm.sleep(level=0, mode="keep") + try: + llm.enqueue(prompts, sampling_params) + finally: + llm.wake_up(tags=["scheduling"]) + responses.append(llm.wait_for_completion(output_type=RequestOutput)) + piecewise_responses, full_responses = responses # Check that all responses are the same + assert len(piecewise_responses) == len(full_responses) == batch_size for piecewise_res, full_res in zip(piecewise_responses, full_responses): assert ( piecewise_res.outputs[0].text.lower() diff --git a/tests/compile/fusions_e2e/common.py b/tests/compile/fusions_e2e/common.py index 77c3ba5112..1f92ff5a19 100644 --- a/tests/compile/fusions_e2e/common.py +++ b/tests/compile/fusions_e2e/common.py @@ -128,8 +128,11 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = { ), "rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"), "act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"), + # The combined pass can consume these sites before the standalone pass. "norm_rope_fusion": re.compile( - r"qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on (\d+) sites" + r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|" + r"qk_norm_rope_kvcache_fusion.py:\d+] " + r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)" ), "attn_quant_fusion": re.compile( r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes" diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index f3df3c358e..a2cbe0135b 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -182,7 +182,11 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): # TODO: Remove log counting in unit tests # once all matchers implement VllmFusionPatternMatcherPass n_expected = tp_size * num_ranges_activated - if match_name not in ("attn_quant_fusion", "act_quant_fusion"): + if match_name not in ( + "attn_quant_fusion", + "act_quant_fusion", + "norm_rope_fusion", + ): assert len(log_matches) == n_expected, ( f"Could not find {n_expected} {match_name} " f"(found {len(log_matches)}) in:\n {log_holder.text}" @@ -190,7 +194,16 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): expected_matches = getattr(matches, match_name) - if match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check: + if match_name == "norm_rope_fusion": + # Opaque LayerName lets the combined pass consume short-range + # sites before the standalone pass; count either owner. + assert len(log_matches) >= n_expected + assert all(m in (0, expected_matches) for m in log_matches) + assert sum(log_matches) == expected_matches * n_expected, ( + f"{match_name} expected {expected_matches * n_expected} " + f"sites across both passes, found: {log_matches}" + ) + elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check: # AR+rms+quant takes precedence over rms+quant if activated. # That means we get full matching where ar+rms+quant was not # activated, and less where it was (only the smallest range). diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index 7879cbe650..eadf1739fd 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -44,6 +44,15 @@ ROCM_AITER_UNIFIED_ATTN = pytest.param( ), ) +ROCM_AITER_MLA_ATTN = pytest.param( + AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA), + id="ROCM_AITER_MLA", + marks=pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="ROCM_AITER_MLA only for AMD when AITER is installed", + ), +) + FLASHINFER_MLA_ATTN = pytest.param( AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA), id="FLASHINFER_MLA", diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index 4b3db06bc4..f35bc1bd9c 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -19,6 +19,7 @@ from .common import ( from .models import ( FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, + ROCM_AITER_MLA_ATTN, ROCM_AITER_UNIFIED_ATTN, ROCM_ATTN, TRITON_ATTN, @@ -40,25 +41,35 @@ pytestmark = pytest.mark.skipif( ) -@multi_gpu_test(num_gpus=2) -@pytest.mark.parametrize( - "model_name, matches_fn, model_kwargs, hf_overrides", - # qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported +# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet +# supported. ROCm carries only the payloads its two backends support. +TP2_FP8_MODELS = ( [ llama3_8b_fp8, llama4_scout_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8, deepseek_v3_fp8, - ], + ] + if current_platform.is_cuda() + else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8] ) + +TP2_FP8_ATTN_BACKENDS = ( + [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN] + if current_platform.is_cuda() + else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN] +) + + +@multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( - "attn_backend", [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN] + "model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS ) +@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS) @pytest.mark.parametrize("n_layers", [4]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm")) @pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION) -@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA") def test_tp2_ar_rms_fp8_fusions( model_name: str, matches_fn: Callable[[int], Matches], @@ -69,10 +80,27 @@ def test_tp2_ar_rms_fp8_fusions( custom_ops: str, inductor_graph_partition: bool, run_e2e_fusion_test, + monkeypatch, ): matches = matches_fn(n_layers) - block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower() + model_name_lower = model_name.lower() + # DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated + # against the native QuantFP8 pattern; every other DeepSeek stays group + # quantized as on CUDA. + rocm_static_fp8_mla = ( + current_platform.is_rocm() + and "deepseek-coder" in model_name_lower + and attn_backend.backend.name == "ROCM_AITER_MLA" + ) + block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower + if rocm_static_fp8_mla: + block_fp8 = False + if custom_ops != "-quant_fp8,-rms_norm": + pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm") + # AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER + # for MLA and the all-reduce, and let the MoE run on Triton. + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0") if block_fp8 and "-quant_fp8" in custom_ops: # This is why config forces +quant_fp8 by default pytest.skip("native QuantFP8 matching not supported for group quant") @@ -83,6 +111,13 @@ def test_tp2_ar_rms_fp8_fusions( model_kwargs["max_model_len"] = 1024 model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False} model_kwargs["disable_custom_all_reduce"] = False + if current_platform.is_rocm() and "llama-3" in model_name_lower: + model_kwargs.update( + gpu_memory_utilization=0.01, + kv_cache_memory_bytes=64 * 1024**2, + max_num_batched_tokens=1024, + max_num_seqs=32, + ) compilation_config = dict( use_inductor_graph_partition=inductor_graph_partition, @@ -103,6 +138,19 @@ def test_tp2_ar_rms_fp8_fusions( "ar_rms_fusion", ] + use_aiter = current_platform.is_rocm() + if use_aiter: + matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion) + # The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns + # before its AR+RMS-only ones, so the standalone AITER RMS-quant pass + # does not own the post-AR sites here. ROCm's contract is native + # QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS. + matches_check = [ + "norm_rope_fusion", + "attn_quant_fusion", + "aiter_ar_rms_fusion", + ] + run_e2e_fusion_test( model_name, matches, @@ -111,6 +159,7 @@ def test_tp2_ar_rms_fp8_fusions( compilation_config, matches_check, tp_size=2, + use_aiter=use_aiter, ) diff --git a/tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py b/tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py new file mode 100644 index 0000000000..3143143bec --- /dev/null +++ b/tests/compile/fusions_e2e/test_tp2_rocm_async_tp.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Bounded ROCm model coverage for actual sequence-parallel AsyncTP rewrites.""" + +import pytest + +from tests.utils import multi_gpu_test +from vllm.config import PassConfig +from vllm.platforms import current_platform + +from .common import INDUCTOR_GRAPH_PARTITION, custom_ops_combos +from .models import TRITON_ATTN, llama3_8b + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract") +@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION) +@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm"))) +@pytest.mark.parametrize("attn_backend", [TRITON_ATTN]) +def test_tp2_rocm_async_tp_bf16_fusions( + inductor_graph_partition, custom_ops, attn_backend, run_e2e_fusion_test +): + n_layers = 4 + model_kwargs = dict( + hf_overrides=dict( + num_hidden_layers=n_layers, + hidden_size=512, + intermediate_size=1024, + num_attention_heads=8, + num_key_value_heads=2, + head_dim=64, + ), + load_format="dummy", + max_model_len=1024, + kv_cache_memory_bytes=64 * 1024**2, + gpu_memory_utilization=0.01, + max_num_batched_tokens=1024, + max_num_seqs=32, + ) + run_e2e_fusion_test( + llama3_8b.model_name, + llama3_8b.matches(n_layers), + model_kwargs, + attn_backend, + dict( + use_inductor_graph_partition=inductor_graph_partition, + custom_ops=custom_ops.split(","), + pass_config=PassConfig( + enable_qk_norm_rope_fusion=True, + enable_sp=True, + fuse_gemm_comms=True, + fuse_allreduce_rms=False, + sp_min_token_num=512, + ), + ), + ["norm_rope_fusion", "sequence_parallel", "async_tp"], + tp_size=2, + ) diff --git a/tests/compile/passes/distributed/test_async_tp.py b/tests/compile/passes/distributed/test_async_tp.py index ff0b048962..c0fd4c00a8 100644 --- a/tests/compile/passes/distributed/test_async_tp.py +++ b/tests/compile/passes/distributed/test_async_tp.py @@ -303,6 +303,7 @@ def async_tp_pass_on_test_model( dtype: torch.dtype, dynamic: bool, distributed_init_method: str, + check_numerics: bool = False, ): set_random_seed(0) @@ -363,12 +364,24 @@ def async_tp_pass_on_test_model( hidden_states = torch.randn( (batch_size * seq_len, hidden_size), dtype=dtype, requires_grad=False ) + if check_numerics: + hidden_states += local_rank * 0.125 if dynamic: torch._dynamo.mark_dynamic(hidden_states, 0) compiled_model = torch.compile(model, backend=backend) - compiled_model(hidden_states) + actual = compiled_model(hidden_states) + + if check_numerics: + # These are new checks, so no existing tolerance is relaxed. + # Allow BF16 rounding vs. eager, but keep the lifetime check exact. + expected = model(hidden_states) + torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) + snapshot = actual.clone() + changed = compiled_model(-hidden_states) + torch.testing.assert_close(changed, -expected, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(actual, snapshot, rtol=0, atol=0) assert async_tp_pass.matched_count == 1 diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index 37f0d4126c..ccc7e624db 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -12,6 +12,7 @@ from tests.v1.attention.utils import ( create_common_attn_metadata, dense_kv_cache_views, ) +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant from vllm.compilation.passes.fusion.attn_quant_fusion import ( ATTN_OP, @@ -189,6 +190,17 @@ class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel): return self.fp8_linear(attn_output) +class TestAttentionAiterFp8StaticQuantPatternModel( + TestAttentionFp8StaticQuantPatternModel +): + aiter_quant = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fp8_linear.kernel.quant_fp8.use_aiter = True + self.fp8_linear.input_scale.fill_(0.125) + + class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel): """Test model for AttentionNvfp4QuantPattern fusion.""" @@ -455,11 +467,12 @@ def test_attention_quant_pattern( ) # Check quantization ops in the graph before and after fusion - quant_op = ( - torch.ops.aten.reciprocal - if "-quant_fp8" in custom_ops_list - else QUANT_OPS[quant_key] - ) + if "-quant_fp8" in custom_ops_list: + quant_op = torch.ops.aten.reciprocal + elif getattr(model_class, "aiter_quant", False): + quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default + else: + quant_op = QUANT_OPS[quant_key] if expect_fusion: # Note: for fp8, fully_replaced=False because query quant ops remain in @@ -523,3 +536,46 @@ def test_attention_quant_pattern( # Check that results are close torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2) + if getattr(model_class, "aiter_quant", False): + # Query quantization remains; exactly the output quant op is removed. + assert test_backend.op_count(quant_op, before=True) == ( + test_backend.op_count(quant_op) + 1 + ) + for model in (model_unfused, model_fused): + torch.testing.assert_close( + model.fp8_linear.input_scale, + torch.full_like(model.fp8_linear.input_scale, 0.125), + atol=0, + rtol=0, + ) + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not is_aiter_found_and_supported(), + reason="Requires ROCm AITER attention and quantization", +) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_rocm_aiter_static_quant_attention( + dtype, dist_init, disable_vllm_compile_cache, monkeypatch +): + """Fuse the actual AITER op without changing its static scale or outputs.""" + try: + with monkeypatch.context() as patch: + patch.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + test_attention_quant_pattern( + num_qo_heads=32, + num_kv_heads=8, + head_size=128, + batch_size=8, + dtype=dtype, + kv_cache_dtype="fp8", + custom_ops="+quant_fp8", + model_name="amd/Llama-3.1-8B-Instruct-FP8-KV", + model_class=TestAttentionAiterFp8StaticQuantPatternModel, + backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + dist_init=dist_init, + disable_vllm_compile_cache=disable_vllm_compile_cache, + ) + finally: + rocm_aiter_ops.refresh_env_variables() diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index f1de3dde0c..a945f00079 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -271,7 +271,6 @@ class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel): quant_key = kFp8Dynamic128Sym quant_config = Fp8Config( - is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128], ) @@ -362,13 +361,27 @@ if current_platform.is_cuda(): ] BACKENDS_MLA_FP8 = [AttentionBackendEnum.TRITON_MLA] BACKENDS_MLA_FP4 = [AttentionBackendEnum.TRITON_MLA] +elif current_platform.is_rocm(): + # ROCm supports the static-FP8 output-quant fusion through AITER MLA. + # Per-group FP8 still relies on the CUDA-only CUTLASS block-scaled + # kernel, and NVFP4 is NVIDIA-specific. + MLA_DIMS = [(16, 128, 64, 128, 512)] + PATTERN_TEST_MODELS_MLA_FP8 = [ + ( + "deepseek-ai/DeepSeek-V2-Lite", + TestMLAAttentionFp8StaticQuantPatternModel, + ) + ] + BACKENDS_MLA_FP8 = [AttentionBackendEnum.ROCM_AITER_MLA] @pytest.mark.parametrize( "num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank", MLA_DIMS, ) -@pytest.mark.parametrize("batch_size", [7, 256] if current_platform.is_cuda() else [8]) +@pytest.mark.parametrize( + "batch_size", [7, 256] if current_platform.is_cuda_alike() else [8] +) @pytest.mark.parametrize("dtype", [torch.bfloat16]) @pytest.mark.parametrize( "backend, model_name, model_class, custom_ops", @@ -406,6 +419,7 @@ def test_mla_attention_quant_pattern( backend: AttentionBackendEnum, dist_init, disable_vllm_compile_cache, + workspace_init, ): """Test MLA AttentionQuantPattern fusion pass.""" if ( diff --git a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py index 6965346a58..ee557235fd 100644 --- a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py +++ b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py @@ -8,6 +8,7 @@ import pytest import torch from packaging.version import Version +import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module import vllm.config from tests.compile.backend import TestBackend from tests.v1.attention.utils import ( @@ -15,8 +16,12 @@ from tests.v1.attention.utils import ( create_common_attn_metadata, dense_kv_cache_views, ) -from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops -from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP +from vllm._aiter_ops import ( + IS_AITER_FOUND, + is_aiter_found_and_supported, + rocm_aiter_ops, +) +from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import ( QkNormRopeKvCacheFusionPass, ) @@ -37,8 +42,12 @@ from vllm.config import ( from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding +from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding from vllm.platforms import current_platform +from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name from vllm.v1.attention.backend import ( AttentionBackend, CommonAttentionMetadata, @@ -94,6 +103,7 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): device: torch.device, rotary_dim: int | None = None, prefix: str = "model.layers.0.self_attn.attn", + expected_query_quant_group_shape: GroupShape | None = None, ): super().__init__() self.num_heads = num_heads @@ -133,6 +143,9 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): attn_backend=attn_backend.get_class(), ) self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend() + if expected_query_quant_group_shape is not None: + assert self.attn.query_quant is not None + assert self.attn.query_quant.group_shape == expected_query_quant_group_shape assert not self.attn_backend.forward_includes_kv_cache_update, ( f"Attention backend {self.attn_backend} does not support " "fuse_qk_norm_rope_kvcache." @@ -227,18 +240,15 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): self.kv_cache_dtype != self.dtype and self.attn.impl.supports_quant_query_input ): - q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE) - torch.ops.vllm.rocm_aiter_per_tensor_quant( - q_fp8, q, self.attn._q_scale, False - ) - q = q_fp8 + assert self.attn.query_quant is not None + q, _ = self.attn.query_quant(q, self.attn._q_scale) # Final views + KV cache update q = q.view(-1, self.num_heads, self.head_size) k = k.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size) kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( - k, v, self.layer_name + k, v, _encode_layer_name(self.layer_name) ) return q, k, v, kv_cache_dummy_dep @@ -260,6 +270,64 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default] +class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel): + def __init__( + self, + *, + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + **kwargs, + ): + super().__init__(**kwargs) + self.rotary_emb = MRotaryEmbedding( + self.head_size, + rotary_dim=self.rotary_dim, + max_position_embeddings=4096, + base=10000, + is_neox_style=self.is_neox, + dtype=self.dtype, + mrope_section=list(mrope_section), + mrope_interleaved=mrope_interleaved, + ) + self.enable_rope_custom_op = self.rotary_emb.enabled() + + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions) + output = torch.empty( + q.shape[0], + self.num_heads * self.head_size, + device=q.device, + dtype=self.dtype, + ) + torch.ops.vllm.unified_attention_with_output( + q, + k, + v, + output, + _encode_layer_name(self.layer_name), + kv_cache_dummy_dep=kv_cache_dummy_dep, + ) + # Production decoder attention consumes K from the cache rather than + # its direct K argument. Returning `output` keeps that production user + # in the graph without exposing K to an unsupported consumer. + return q, output, v, kv_cache_dummy_dep + + def ops_in_model_before(self) -> list[torch._ops.OpOverload]: + return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default] + + def ops_in_model_after(self) -> list[torch._ops.OpOverload]: + return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default] + + +class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel): + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions) + + def _run_qk_norm_rope_kvcache_fusion_test( *, attn_backend: AttentionBackendEnum, @@ -278,18 +346,33 @@ def _run_qk_norm_rope_kvcache_fusion_test( rms_norm_eps: float, custom_op: str, monkeypatch: pytest.MonkeyPatch, + mrope_section: tuple[int, int, int] | None = None, + mrope_interleaved: bool = False, + observe_mrope_k: bool = False, + expect_fusion: bool = True, + use_quark_scalar_query_scale: bool = False, ) -> None: device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda") torch.set_default_device(device) torch.set_default_dtype(dtype) torch.manual_seed(0) + # Set AITER toggles before VllmConfig initialization enables env caching. + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1") + rocm_aiter_ops.refresh_env_variables() + vllm_config = VllmConfig( model_config=ModelConfig(dtype=dtype), cache_config=CacheConfig( block_size=block_size, cache_dtype=kv_cache_dtype, ), + quant_config=( + QuarkConfig(quant_config={"exclude": []}) + if use_quark_scalar_query_scale + else None + ), compilation_config=CompilationConfig( mode=CompilationMode.VLLM_COMPILE, custom_ops=[custom_op], @@ -302,6 +385,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: m.setenv("VLLM_ROCM_USE_AITER", "1") + m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1") m.setenv( "VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0", @@ -309,18 +393,49 @@ def _run_qk_norm_rope_kvcache_fusion_test( m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout) rocm_aiter_ops.refresh_env_variables() - model = QKNormRoPEKVCacheTestModel( - vllm_config=vllm_config, - attn_backend=attn_backend, - num_heads=num_heads, - num_kv_heads=num_kv_heads, - head_size=head_size, - rotary_dim=rotary_dim, - is_neox=is_neox, - rms_norm_eps=rms_norm_eps, - dtype=dtype, - device=torch.get_default_device(), - ) + expected_query_quant_group_shape = None + if use_quark_scalar_query_scale: + expected_query_quant_group_shape = ( + GroupShape(-1, num_heads * head_size) + if num_kv_heads == 1 + else GroupShape.PER_TENSOR + ) + + model_kwargs = { + "vllm_config": vllm_config, + "attn_backend": attn_backend, + "num_heads": num_heads, + "num_kv_heads": num_kv_heads, + "head_size": head_size, + "rotary_dim": rotary_dim, + "is_neox": is_neox, + "rms_norm_eps": rms_norm_eps, + "dtype": dtype, + "device": torch.get_default_device(), + "expected_query_quant_group_shape": expected_query_quant_group_shape, + } + if mrope_section is None: + model = QKNormRoPEKVCacheTestModel(**model_kwargs) + else: + alternate_section = (8, 8, 8) + m.setattr( + fusion_module, + "_discover_mrope_configs", + lambda _: ( + (mrope_section, mrope_interleaved), + (alternate_section, not mrope_interleaved), + ), + ) + model_cls = ( + QKNormMRoPEObservableKTestModel + if observe_mrope_k + else QKNormMRoPEKVCacheTestModel + ) + model = model_cls( + **model_kwargs, + mrope_section=mrope_section, + mrope_interleaved=mrope_interleaved, + ) fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config) passes = [ @@ -337,7 +452,12 @@ def _run_qk_norm_rope_kvcache_fusion_test( num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype, ) - pos = torch.arange(num_tokens, dtype=torch.long) + if mrope_section is None: + pos = torch.arange(num_tokens, dtype=torch.long) + else: + # Real Qwen-VL position IDs can be a strided [3, N] view. + pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2] + assert not pos.is_contiguous() qkv_unfused = qkv.clone() pos_unfused = pos.clone() @@ -356,7 +476,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( # Run fused (compiled) forward torch._dynamo.mark_dynamic(qkv, 0) - torch._dynamo.mark_dynamic(pos, 0) + torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1) with set_forward_context(None, vllm_config): model_fused = torch.compile(model, backend=backend) forward_context = get_forward_context() @@ -369,14 +489,33 @@ def _run_qk_norm_rope_kvcache_fusion_test( kv_cache_fused = attn_layer.kv_cache del dummy - assert fusion_pass.matched_count == 1 + assert fusion_pass.matched_count == int(expect_fusion) - backend.check_before_ops(model.ops_in_model_before()) - backend.check_after_ops(model.ops_in_model_after()) + if expect_fusion: + backend.check_before_ops(model.ops_in_model_before()) + backend.check_after_ops(model.ops_in_model_after()) + else: + for op in model.ops_in_model_before(): + assert backend.op_count(op, before=True) > 0 + assert backend.op_count(op) > 0 + + if use_quark_scalar_query_scale: + expected_quant_op = ( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default + if num_kv_heads != 1 + else torch.ops._C.static_scaled_fp8_quant.default + ) + assert backend.op_count(expected_quant_op, before=True) > 0 + # Query quantization remains separate after the prologue fusion. + assert backend.op_count(expected_quant_op) > 0 # Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2; # AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2. - ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2) + ATOL, RTOL = ( + (2e-2, 2e-2) + if enable_aiter_triton_rope or mrope_section is not None + else (1e-2, 1e-2) + ) is_fp8_cache = model.kv_cache_dtype != dtype if q_fused.dtype == FP8_DTYPE: @@ -387,7 +526,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( else: torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL) - if not is_fp8_cache: + if not expect_fusion or (not is_fp8_cache and mrope_section is None): # The AITER PTS kernel populates k_out only for non-FP8 caches. # With FP8, the kernel writes quantized K directly to the cache # and may leave k_out uninitialised. In production this is fine @@ -397,15 +536,16 @@ def _run_qk_norm_rope_kvcache_fusion_test( # Should be bit exact since no processing had been done on v for both paths torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0) - # fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1. - if is_fp8_cache and enable_aiter_triton_rope: + # Fused and unfused arithmetic can straddle an FP8 rounding boundary. + # Allow one E4M3 quantization step while still comparing every block. + if is_fp8_cache: cache_atol = cache_rtol = 1.25e-1 else: cache_atol, cache_rtol = ATOL, RTOL torch.testing.assert_close( - kv_cache_unfused[0].float(), - kv_cache_fused[0].float(), + kv_cache_unfused.float(), + kv_cache_fused.float(), atol=cache_atol, rtol=cache_rtol, ) @@ -494,3 +634,225 @@ def test_qk_norm_rope_kvcache_fusion( custom_op=custom_op, monkeypatch=monkeypatch, ) + + +@pytest.mark.parametrize( + "mrope_section", + [ + pytest.param((16, 24, 24), id="full_rotary"), + pytest.param((4, 6, 6), id="partial_rotary"), + ], +) +@pytest.mark.parametrize("mrope_interleaved", [False, True]) +@pytest.mark.parametrize("is_neox", [False, True]) +@pytest.mark.parametrize( + "kv_layout", + [ + pytest.param(KVCacheLayout.LBHNC, id="head_major"), + pytest.param(KVCacheLayout.LBNHC, id="token_major"), + pytest.param(KVCacheLayout.LHBNC, id="block_major"), + ], +) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.skipif( + not IS_AITER_FOUND, + reason="Requires AITER", +) +def test_qk_norm_mrope_kvcache_fusion( + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + is_neox: bool, + kv_layout: KVCacheLayout, + kv_cache_dtype: str, + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=2 * sum(mrope_section), + block_size=16, + is_neox=is_neox, + use_shuffle_kv_layout="0", + kv_layout=kv_layout, + dtype=torch.bfloat16, + kv_cache_dtype=kv_cache_dtype, + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=mrope_section, + mrope_interleaved=mrope_interleaved, + ) + + +@pytest.mark.parametrize( + ("num_heads", "num_kv_heads"), + [(32, 2), (16, 1), (8, 1)], + ids=["tp2", "tp4", "tp8"], +) +def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale( + num_heads: int, + num_kv_heads: int, + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=128, + rotary_dim=128, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(24, 20, 20), + mrope_interleaved=True, + use_quark_scalar_query_scale=True, + ) + + +def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=128, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="auto", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(16, 24, 24), + observe_mrope_k=True, + expect_fusion=False, + ) + + +def test_mrope_fusion_rejects_unaligned_rotary_dim( + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=6, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="auto", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(1, 1, 1), + expect_fusion=False, + ) + + +@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName") +def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch): + device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda") + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=torch.bfloat16), + cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + custom_ops=["+rotary_embedding"], + pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True), + ), + ) + + with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: + m.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + model_kwargs = { + "vllm_config": vllm_config, + "attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + "num_heads": 8, + "num_kv_heads": 2, + "head_size": 128, + "is_neox": True, + "rms_norm_eps": 1e-6, + "dtype": torch.bfloat16, + "device": torch.device(device), + } + models = [ + QKNormRoPEKVCacheTestModel( + **model_kwargs, + prefix=f"model.layers.{index}.self_attn.attn", + ) + for index in range(2) + ] + + # Opaque layer names are wildcards in the pattern graph. Registering + # once per physical layer raises "Duplicate pattern" on torch >= 2.11. + fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config) + + assert models + assert fusion_pass.patterns + + +def _call_fused_mrope_impl(q_out: torch.Tensor) -> None: + device = q_out.device + fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl( + q_out=q_out, + qkv=torch.zeros(2, 16, device=device), + positions=torch.arange(2, device=device), + q_weight=torch.ones(4, device=device), + k_weight=torch.ones(4, device=device), + rms_norm_eps=1e-6, + cos_sin_cache=torch.zeros(16, 4, device=device), + is_neox=True, + mrope_section_t=1, + mrope_section_h=1, + mrope_section_w=0, + is_interleaved=False, + rotary_dim=4, + layer_name="model.layers.0.self_attn.attn", + ) + + +def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + fusion_module, + "get_attention_context", + lambda _: (None, None, None, None), + ) + q_out = torch.full((2, 8), torch.nan, device="cpu") + + _call_fused_mrope_impl(q_out) + + torch.testing.assert_close(q_out, torch.zeros_like(q_out)) + + +def test_mrope_attention_context_failure_propagates( + monkeypatch: pytest.MonkeyPatch, +): + def fail_context(_): + raise RuntimeError("missing attention context") + + monkeypatch.setattr(fusion_module, "get_attention_context", fail_context) + + with pytest.raises(RuntimeError, match="missing attention context"): + _call_fused_mrope_impl(torch.empty(2, 8, device="cpu")) diff --git a/tests/compile/rocm/__init__.py b/tests/compile/rocm/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/compile/rocm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/compile/rocm/test_async_tp.py b/tests/compile/rocm/test_async_tp.py new file mode 100644 index 0000000000..a4462265f8 --- /dev/null +++ b/tests/compile/rocm/test_async_tp.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm AsyncTP numerical coverage kept outside the shared distributed suite.""" + +import pytest +import torch + +from tests.compile.passes.distributed.test_async_tp import ( + TestAGMMModel, + TestMMRSModel, + async_tp_pass_on_test_model, +) +from tests.utils import multi_gpu_test +from vllm.distributed import ( + tensor_model_parallel_all_gather, + tensor_model_parallel_reduce_scatter, +) +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_file_store_init_method + + +class TestRocmMMRSModel(TestMMRSModel): + def forward(self, hidden_states): + view = hidden_states.reshape(-1, self.hidden_size) + mm = torch.ops.vllm.rocm_unquantized_gemm(view, self.gate_proj, None) + return tensor_model_parallel_reduce_scatter(mm, dim=0) + + def ops_in_model_before(self): + return super().ops_in_model_before() + [ + torch.ops.vllm.rocm_unquantized_gemm.default + ] + + +class TestRocmAGMMModel(TestAGMMModel): + def __init__(self, hidden_size=16, dtype=torch.bfloat16): + super().__init__(hidden_size, dtype) + self.weight = torch.nn.Parameter( + torch.empty((hidden_size * 2, hidden_size)), requires_grad=False + ) + torch.nn.init.normal_(self.weight, std=0.02) + + def forward(self, hidden_states): + view = hidden_states.reshape(-1, self.hidden_size) + gathered = tensor_model_parallel_all_gather(view, dim=0) + return torch.ops.vllm.rocm_unquantized_gemm(gathered, self.weight, None) + + def ops_in_model_before(self): + return super().ops_in_model_before() + [ + torch.ops.vllm.rocm_unquantized_gemm.default + ] + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend contract") +@pytest.mark.parametrize( + "test_model", [TestMMRSModel, TestAGMMModel, TestRocmMMRSModel, TestRocmAGMMModel] +) +@pytest.mark.parametrize("dynamic", [False, True]) +def test_rocm_async_tp_bf16_rewrite_and_correctness(test_model, dynamic): + """Real symmetric-memory AG/GEMM and GEMM/RS must replace the collectives.""" + torch.multiprocessing.spawn( + async_tp_pass_on_test_model, + args=( + 2, + test_model, + 8, + 16, + 128, + torch.bfloat16, + dynamic, + get_file_store_init_method(), + True, + ), + nprocs=2, + ) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 7465ea442c..74ed0d34db 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -70,6 +70,37 @@ def test_get_raw_stream_patch(): assert get_raw_stream is _cuda_getCurrentRawStream +# Inductor can initialize CUDA even for CPU inputs. +@pytest.mark.forked +@pytest.mark.parametrize("use_v2", [False, True]) +def test_e8m0_custom_op_fullgraph(use_v2, use_fresh_inductor_cache): + """E8M0 inputs must not prevent mutable custom-op decomposition.""" + with torch.library._scoped_library("test_e8m0", "FRAGMENT") as lib: + lib.define("scale_(Tensor(a!) out, Tensor scale) -> ()") + + @torch.library.impl(lib, "scale_", "CPU") + def scale_impl(out, scale): + out.mul_(scale.float()) + + @torch.library.register_fake("test_e8m0::scale_", lib=lib) + def scale_fake(out, scale): + return None + + def forward(x, scale): + out = x.clone() + torch.ops.test_e8m0.scale_(out, scale) + return out + + x = torch.randn(4, 4) + scale = torch.full((4,), 128, dtype=torch.uint8).view(torch.float8_e8m0fnu) + compiled = torch.compile( + forward, + fullgraph=True, + options={"enable_auto_functionalized_v2": use_v2}, + ) + torch.testing.assert_close(compiled(x, scale), x * 2) + + def test_copy_pass(): vllm_config = VllmConfig() inductor_pass = FixFunctionalizationPass(vllm_config) @@ -233,6 +264,21 @@ def test_enforce_eager(vllm_runner, monkeypatch): pass +@pytest.mark.parametrize("enable_fault_tolerance", [False, True]) +def test_enforce_eager_jit_warmup(enable_fault_tolerance): + """Enforce-eager disables JIT warmup unless fault tolerance is on. + + FT fault detection runs against deadlines that in-inference Triton + compilation latency spikes can blow past, so warmup stays enabled. + """ + config = VllmConfig( + model_config=ModelConfig(model="facebook/opt-125m", enforce_eager=True), + parallel_config=ParallelConfig(enable_fault_tolerance=enable_fault_tolerance), + ) + assert config.compilation_config.mode == CompilationMode.NONE + assert config.kernel_config.enable_jit_warmup == enable_fault_tolerance + + @pytest.mark.forked def test_torch_compile_disable(vllm_runner, monkeypatch): monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") diff --git a/tests/config/test_bailing_mtp_config.py b/tests/config/test_bailing_mtp_config.py index bb7cf3ad56..cd842856a9 100644 --- a/tests/config/test_bailing_mtp_config.py +++ b/tests/config/test_bailing_mtp_config.py @@ -3,7 +3,7 @@ from typing import get_args -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.speculative import MTPModelTypes, SpeculativeConfig from vllm.transformers_utils.model_arch_config_convertor import ( @@ -11,8 +11,8 @@ from vllm.transformers_utils.model_arch_config_convertor import ( ) -def _bailing_config() -> PretrainedConfig: - config = PretrainedConfig( +def _bailing_config() -> PreTrainedConfig: + config = PreTrainedConfig( architectures=["BailingMoeV2_5ForCausalLM"], hidden_size=4096, kv_lora_rank=512, diff --git a/tests/config/test_config_generation.py b/tests/config/test_config_generation.py index c7edf2b971..a6908f35e5 100644 --- a/tests/config/test_config_generation.py +++ b/tests/config/test_config_generation.py @@ -109,3 +109,60 @@ def test_unrecognized_env(monkeypatch): fail_on_environ_validation=True, ) engine_args.create_engine_config() + + +def test_get_diff_sampling_param_includes_penalties(): + """``--override-generation-config`` must apply presence_penalty and + frequency_penalty as server-side defaults, like the other sampling + params. Regression test for the whitelist silently dropping them. + + Called unbound with a lightweight stand-in so no model is loaded: + generation_config="vllm" makes the method skip try_get_generation_config + and read only override_generation_config. + """ + from types import SimpleNamespace + + from vllm.config.model import ModelConfig + + fake = SimpleNamespace( + generation_config="vllm", + override_generation_config={ + "presence_penalty": 1.5, + "frequency_penalty": 0.5, + "temperature": 0.6, + }, + ) + + diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type] + + assert diff["presence_penalty"] == 1.5 + assert diff["frequency_penalty"] == 0.5 + assert diff["temperature"] == 0.6 + + +def test_generation_config_penalties_survive_diff_sampling_filter(): + """``generation_config.json`` penalties must survive the diff-sampling + whitelist (regression for #50767, auto path). + + Covers the ``generation_config="auto"`` branch, where the override + dict is empty and the values come from ``try_get_generation_config``. + """ + from types import SimpleNamespace + + from vllm.config.model import ModelConfig + + fake = SimpleNamespace( + generation_config="auto", + override_generation_config={}, + try_get_generation_config=lambda: { + "presence_penalty": 1.1, + "frequency_penalty": 0.7, + "temperature": 0.9, + }, + ) + + diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type] + + assert diff["presence_penalty"] == 1.1 + assert diff["frequency_penalty"] == 0.7 + assert diff["temperature"] == 0.9 diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py index b68820c0d5..ae4c7220f2 100644 --- a/tests/config/test_model_arch_config.py +++ b/tests/config/test_model_arch_config.py @@ -10,13 +10,15 @@ from typing import cast from unittest.mock import Mock import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.models.gemma4.configuration_gemma4 import Gemma4TextConfig from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig from vllm.config.model_arch import ModelArchitectureConfig +from vllm.transformers_utils.configs.diffusion_gemma import DiffusionGemmaConfig from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config from vllm.transformers_utils.model_arch_config_convertor import ( + MODEL_ARCH_CONFIG_CONVERTORS, Gemma4ModelArchConfigConvertor, ModelArchConfigConvertorBase, ) @@ -123,7 +125,7 @@ def _assert_model_config_methods( def test_head_size_falls_back_when_head_dim_is_zero(): """Regression test for configs that materialize missing head_dim as 0.""" - hf_config = PretrainedConfig( + hf_config = PreTrainedConfig( model_type="deepseek_vl_v2", hidden_size=1280, num_attention_heads=10, @@ -138,7 +140,7 @@ def test_head_size_falls_back_when_head_dim_is_zero(): def test_qk_rope_head_dim_recovery_uses_model_revision(monkeypatch: pytest.MonkeyPatch): - hf_config = PretrainedConfig( + hf_config = PreTrainedConfig( model_type="deepseek_v2", qk_rope_head_dim=128, qk_nope_head_dim=128, @@ -174,7 +176,7 @@ def test_qk_rope_head_dim_recovery_uses_model_revision(monkeypatch: pytest.Monke ], ) def test_num_experts_per_tok_aliases(attribute: str): - hf_config = PretrainedConfig(**{attribute: 4}) + hf_config = PreTrainedConfig(**{attribute: 4}) convertor = ModelArchConfigConvertorBase(hf_config, hf_config) model_config = cast( ModelConfig, @@ -189,7 +191,7 @@ def test_num_experts_per_tok_aliases(attribute: str): def test_num_experts_per_tok_none_is_normalized(): - hf_config = PretrainedConfig(top_k_experts=None) + hf_config = PreTrainedConfig(top_k_experts=None) convertor = ModelArchConfigConvertorBase(hf_config, hf_config) assert convertor.get_num_experts_per_token() == 0 @@ -205,7 +207,7 @@ def test_legacy_modelopt_config_without_producer_is_normalized(): "modelopt_quant_config": {"quant_cfg": {}}, } } - hf_config = PretrainedConfig(quantization_config=quantization_config) + hf_config = PreTrainedConfig(quantization_config=quantization_config) convertor = ModelArchConfigConvertorBase(hf_config, hf_config) @@ -368,7 +370,7 @@ def test_gemma4_uniform_head_dims_are_homogeneous(): assert arch[3] is arch -class _HeterogeneousConfig(PretrainedConfig): +class _HeterogeneousConfig(PreTrainedConfig): """A heterogeneous config with no convertor of its own. Mirrors the parts vLLM uses: per-layer configs are shallow copies with the @@ -382,7 +384,7 @@ class _HeterogeneousConfig(PretrainedConfig): super().__init__(**kwargs) @property - def per_layer_config(self) -> list[PretrainedConfig]: + def per_layer_config(self) -> list[PreTrainedConfig]: layers = [] for i in range(self.num_hidden_layers): layer = copy(self) @@ -496,3 +498,26 @@ def test_deepseek_v4_convertor_splits_vision_architecture(): conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg) assert conv.get_architectures() == [draft_arch] assert draft_cfg.architectures == [draft_arch] + + +def test_diffusion_gemma_head_dims_vary_by_layer_type(): + """Checkpoints carry model_type `diffusion_gemma` at the top level. Under + the base convertor every layer reports the sliding head dim, Gemma4Config + skips its head-dim handling, and the full-attention KV-cache group falls to + FlashInfer, which cannot take a per-request causal tensor.""" + text_config = dict( + num_hidden_layers=6, + hidden_size=64, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=16, + global_head_dim=32, + layer_types=["sliding_attention"] * 5 + ["full_attention"], + ) + hf_config = DiffusionGemmaConfig(text_config=text_config) + + convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS[hf_config.model_type] + assert convertor_cls is Gemma4ModelArchConfigConvertor + arch = convertor_cls(hf_config, hf_config.text_config).convert() + + assert [arch[i].head_size for i in range(6)] == [16] * 5 + [32] diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 225f41bd01..b36f5776da 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock, patch import pytest import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from tests.models.utils import build_model_context from vllm.config.ec_transfer import ECRole, ECTransferConfig @@ -136,7 +136,7 @@ def test_language_model_only_disables_via_supports_multimodal_inputs(): def test_convertor_clears_mm_prefix_when_multimodal_disabled(): - hf_config = PretrainedConfig( + hf_config = PreTrainedConfig( model_type="gemma3", architectures=["Gemma3ForConditionalGeneration"], ) diff --git a/tests/config/test_speculative_draft_hf_overrides.py b/tests/config/test_speculative_draft_hf_overrides.py index 8505172da3..e7014faa47 100644 --- a/tests/config/test_speculative_draft_hf_overrides.py +++ b/tests/config/test_speculative_draft_hf_overrides.py @@ -15,20 +15,20 @@ import functools from unittest.mock import MagicMock, patch import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.parallel import ParallelConfig from vllm.config.speculative import SpeculativeConfig -def _make_hf_config(**kwargs) -> PretrainedConfig: +def _make_hf_config(**kwargs) -> PreTrainedConfig: defaults = dict( architectures=["LlamaForCausalLM"], model_type="llama", num_hidden_layers=64, ) defaults.update(kwargs) - return PretrainedConfig(**defaults) + return PreTrainedConfig(**defaults) @pytest.mark.cpu_test @@ -47,12 +47,97 @@ def test_none_overrides_fall_back_to_arch_mapping(): assert composed is SpeculativeConfig.hf_config_override +def _make_speculative_config( + hf_kwargs: dict, architecture: str, **speculative_kwargs +) -> SpeculativeConfig: + hf_config = SpeculativeConfig.hf_config_override(_make_hf_config(**hf_kwargs)) + draft = MagicMock( + hf_config=hf_config, architectures=hf_config.architectures, max_model_len=128 + ) + draft.registry.inspect_model_cls.return_value = (None, architecture) + target = MagicMock(max_model_len=128, quantization=None, hf_overrides={}) + target.hf_config.model_type = hf_kwargs["model_type"] + + with patch("vllm.config.speculative.ModelConfig", return_value=draft): + return SpeculativeConfig( + model="draft", + **speculative_kwargs, + target_model_config=target, + target_parallel_config=ParallelConfig(), + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("hf_kwargs", "n_predict", "architecture"), + [ + ( + dict( + model_type="deepseek_v4", + num_nextn_predict_layers=3, + dspark_block_size=5, + ), + 3, + "DSparkDraftModel", + ), + ( + dict( + model_type="deepseek_v41", + num_nextn_predict_layers=3, + dspark_block_size=5, + ), + 3, + "DSparkV41DraftModel", + ), + ( + dict( + model_type="gemma4_text", + architectures=["Gemma4DSparkModel"], + block_size=5, + ), + None, + "Gemma4DSparkModel", + ), + ], +) +@pytest.mark.parametrize("speculative_kwargs", [{"num_speculative_tokens": 5}, {}]) +def test_dspark_width_is_independent_of_mtp_stages( + hf_kwargs, n_predict, architecture, speculative_kwargs +): + config = _make_speculative_config( + hf_kwargs, architecture, method="dspark", **speculative_kwargs + ) + + assert config.num_speculative_tokens == 5 + assert getattr(config.draft_model_config.hf_config, "n_predict", None) == n_predict + assert config.draft_model_config.hf_config.architectures == [architecture] + + +@pytest.mark.cpu_test +def test_mtp_stages_are_independent_of_dspark_width(): + hf_kwargs = dict( + model_type="deepseek_v4", num_nextn_predict_layers=3, dspark_block_size=5 + ) + config = _make_speculative_config( + hf_kwargs, "DeepSeekV4MTPModel", method="mtp", num_speculative_tokens=6 + ) + + assert config.num_speculative_tokens == 6 + assert config.draft_model_config.hf_config.n_predict == 3 + assert config.draft_model_config.hf_config.architectures == ["DeepSeekV4MTPModel"] + + with pytest.raises(ValueError, match="must be divisible by n_predict=3"): + _make_speculative_config( + hf_kwargs, "DeepSeekV4MTPModel", method="mtp", num_speculative_tokens=5 + ) + + @pytest.mark.cpu_test def test_callable_overrides_reach_the_draft_config(): """A callable override (config-to-config transform) composes with the architecture-mapping override and is applied to the draft config.""" - def shrink(hf_config: PretrainedConfig) -> PretrainedConfig: + def shrink(hf_config: PreTrainedConfig) -> PreTrainedConfig: hf_config.num_hidden_layers = 1 return hf_config @@ -70,7 +155,7 @@ def test_arch_mapping_applies_before_callable_override(): observes (and may adjust) the post-mapping config.""" seen_architectures: list[str] = [] - def record(hf_config: PretrainedConfig) -> PretrainedConfig: + def record(hf_config: PreTrainedConfig) -> PreTrainedConfig: seen_architectures.append(hf_config.architectures[0]) return hf_config @@ -117,7 +202,7 @@ def test_inkling_override_exposes_all_mtp_depths(): assert out.local_layer_ids == [0, 2, 4] -def _module_level_shrink(hf_config: PretrainedConfig) -> PretrainedConfig: +def _module_level_shrink(hf_config: PreTrainedConfig) -> PreTrainedConfig: hf_config.num_hidden_layers = 1 return hf_config diff --git a/tests/config/test_speculative_draft_max_position_embeddings.py b/tests/config/test_speculative_draft_max_position_embeddings.py index 117ae5a66d..f5c5524acd 100644 --- a/tests/config/test_speculative_draft_max_position_embeddings.py +++ b/tests/config/test_speculative_draft_max_position_embeddings.py @@ -17,7 +17,7 @@ import logging from typing import Literal import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.model import ModelConfig from vllm.config.parallel import ParallelConfig @@ -46,7 +46,7 @@ def _override_logged(caplog: pytest.LogCaptureFixture) -> bool: @pytest.mark.cpu_test def test_override_raises_smaller_value(vllm_caplog: pytest.LogCaptureFixture): - hf_config = PretrainedConfig(max_position_embeddings=2048) + hf_config = PreTrainedConfig(max_position_embeddings=2048) SpeculativeConfig._maybe_override_draft_max_position_embeddings( hf_config, target_max_model_len=8192 ) @@ -56,7 +56,7 @@ def test_override_raises_smaller_value(vllm_caplog: pytest.LogCaptureFixture): @pytest.mark.cpu_test def test_override_keeps_sufficient_value(vllm_caplog: pytest.LogCaptureFixture): - hf_config = PretrainedConfig(max_position_embeddings=8192) + hf_config = PreTrainedConfig(max_position_embeddings=8192) SpeculativeConfig._maybe_override_draft_max_position_embeddings( hf_config, target_max_model_len=8192 ) @@ -66,7 +66,7 @@ def test_override_keeps_sufficient_value(vllm_caplog: pytest.LogCaptureFixture): @pytest.mark.cpu_test def test_override_ignores_missing_attribute(vllm_caplog: pytest.LogCaptureFixture): - hf_config = PretrainedConfig() + hf_config = PreTrainedConfig() hf_config.__dict__.pop("max_position_embeddings", None) SpeculativeConfig._maybe_override_draft_max_position_embeddings( hf_config, target_max_model_len=8192 diff --git a/tests/conftest.py b/tests/conftest.py index cd1d11917b..37bfee1f8b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -317,14 +317,6 @@ def example_system_message() -> str: return f.read() -class DecoderPromptType(Enum): - """For encoder/decoder models only.""" - - CUSTOM = 1 - NONE = 2 - EMPTY_STR = 3 - - @pytest.fixture def example_long_prompts() -> list[str]: return [prompt for filename in _LONG_PROMPTS for prompt in _read_prompts(filename)] @@ -928,7 +920,7 @@ class HfRunner: # shutdown. This is helpful in cases where the HfRunner is # initialized after significant GPU memory is already occupied, # e.g. in - # tests/basic_correctness/test_basic_correctness.py::test_models_distributed + # tests/basic_correctness/models/test_basic_correctness.py::test_models_distributed # where vllm worker processes are still alive and holding GPU # memory when hf_runner.__exit__ is called. from tests.utils import record_gpu_memory_usage_stats @@ -982,6 +974,8 @@ class VllmRunner: - `enable_chunked_prefill`: Set to `False` instead of `None` for test reproducibility. - `enforce_eager`: Set to `False` to test CUDA graph. + - `kernel_config.enable_jit_warmup`: Set to `False` to reduce test startup + time. """ def __init__( @@ -1010,6 +1004,8 @@ class VllmRunner: else set_default_torch_num_threads(default_torch_num_threads) ) + kwargs.setdefault("kernel_config", {"enable_jit_warmup": False}) + if not kwargs.get("compilation_config", None): # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is @@ -1426,16 +1422,20 @@ def temporary_enable_log_propagate(): import logging logger = logging.getLogger("vllm") + previous_propagate = logger.propagate logger.propagate = True yield - logger.propagate = False + logger.propagate = previous_propagate @pytest.fixture() def caplog_vllm(temporary_enable_log_propagate, caplog): - # To capture vllm log, we should enable propagate=True temporarily - # because caplog depends on logs propagated to the root logger. - yield caplog + # caplog depends on logs propagated to the root logger. The vLLM logger + # inherits this INFO level until runtime initialization configures it. + import logging + + with caplog.at_level(logging.INFO): + yield caplog @pytest.fixture() @@ -1786,6 +1786,24 @@ def disable_deepgemm_ue8m0(monkeypatch): is_deep_gemm_e8m0_used.cache_clear() +@pytest.fixture +def gpu_memory_cleared(): + """Wait for prior tests to release GPU memory on gfx950.""" + if not current_platform.is_rocm(): + return + + from tests.utils import wait_for_gpu_memory_to_clear + from vllm.platforms.rocm import on_gfx950 + + if on_gfx950(): + wait_for_gpu_memory_to_clear( + devices=[0], + threshold_ratio=0.08, + timeout_s=30, + stable_duration_s=1, + ) + + def _should_clean_gpu_memory_between_tests() -> bool: # This must stay opt-in: a function-scoped fixture cannot distinguish # stale VRAM from allocations owned by longer-lived module/session fixtures. diff --git a/tests/device_allocator/test_alloc_conf.py b/tests/device_allocator/test_alloc_conf.py new file mode 100644 index 0000000000..8a9128c5f0 --- /dev/null +++ b/tests/device_allocator/test_alloc_conf.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.device_allocator import alloc_conf + +pytestmark = pytest.mark.cpu_test + +ES = alloc_conf.EXPANDABLE_SEGMENTS + + +@pytest.mark.parametrize( + "environ,expected", + [ + ({}, False), + ({"PYTORCH_ALLOC_CONF": "expandable_segments:True"}, True), + ({"PYTORCH_ALLOC_CONF": "expandable_segments:true"}, True), + ({"PYTORCH_ALLOC_CONF": "expandable_segments:1"}, True), + ({"PYTORCH_ALLOC_CONF": "expandable_segments:False"}, False), + ({"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"}, True), + # ROCm users set this one; it was previously not looked at. + ({"PYTORCH_HIP_ALLOC_CONF": "expandable_segments:True"}, True), + ( + {"PYTORCH_ALLOC_CONF": "max_split_size_mb:512,expandable_segments:True"}, + True, + ), + ({"PYTORCH_ALLOC_CONF": "max_split_size_mb:512"}, False), + ({"PYTORCH_ALLOC_CONF": "not_expandable_segments_really:True"}, False), + ({"PYTORCH_ALLOC_CONF": ""}, False), + # torch's precedence: the accelerator-agnostic variable wins. + ( + { + "PYTORCH_ALLOC_CONF": "expandable_segments:False", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + }, + False, + ), + ], +) +def test_expandable_segments_from_env(environ, expected): + assert alloc_conf.expandable_segments_enabled_from_env(environ) is expected + + +@pytest.mark.parametrize( + "conf,key,expected", + [ + ("", ES, False), + ("expandable_segments:True", ES, True), + ("expandable_segments:TRUE", ES, True), + ("expandable_segments:1", ES, True), + ("expandable_segments:0", ES, False), + (" expandable_segments : True ", ES, True), + ("max_split_size_mb:512", ES, False), + ("max_split_size_mb:512", "max_split_size_mb", False), + ("garbage_collection_threshold:0.9", ES, False), + ], +) +def test_conf_flag_enabled(conf, key, expected): + assert alloc_conf.conf_flag_enabled(conf, key) is expected + + +@pytest.mark.parametrize( + "conf,enabled,expected", + [ + ("", False, "expandable_segments:False"), + ("expandable_segments:True", False, "expandable_segments:False"), + ("expandable_segments:False", True, "expandable_segments:True"), + ( + "max_split_size_mb:512,expandable_segments:True", + False, + "max_split_size_mb:512,expandable_segments:False", + ), + ( + "garbage_collection_threshold:0.9,max_split_size_mb:512", + False, + "garbage_collection_threshold:0.9,max_split_size_mb:512," + "expandable_segments:False", + ), + ], +) +def test_with_conf_flag_preserves_other_fields(conf, enabled, expected): + assert alloc_conf.with_conf_flag(conf, ES, enabled) == expected + + +def test_with_conf_flag_round_trips(): + """The regression this guards: torch resets every option absent from the + string it is handed, so a one-field write drops the rest permanently.""" + conf = ( + "max_split_size_mb:512,expandable_segments:True," + "garbage_collection_threshold:0.9" + ) + off = alloc_conf.with_conf_flag(conf, ES, False) + assert "max_split_size_mb:512" in off + assert "garbage_collection_threshold:0.9" in off + assert alloc_conf.with_conf_flag(off, ES, True) == conf + + +def test_alloc_conf_from_env_precedence(): + assert ( + alloc_conf.alloc_conf_from_env( + { + "PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256", + "PYTORCH_HIP_ALLOC_CONF": "max_split_size_mb:128", + } + ) + == "max_split_size_mb:256" + ) + assert alloc_conf.alloc_conf_from_env({}) == "" + + +def test_current_alloc_conf_prefers_live_state(monkeypatch): + monkeypatch.setattr(alloc_conf, "live_alloc_conf", lambda: "max_split_size_mb:64") + monkeypatch.setattr(alloc_conf, "alloc_conf_from_env", lambda: "stale:True") + assert alloc_conf.current_alloc_conf() == "max_split_size_mb:64" + + monkeypatch.setattr(alloc_conf, "live_alloc_conf", lambda: None) + assert alloc_conf.current_alloc_conf() == "stale:True" + + +def test_expandable_segments_enabled_returns_none_when_unreadable(monkeypatch): + """'cannot read' must stay distinct from 'disabled'.""" + + class _NoSnapshot: + @staticmethod + def _snapshot(): + raise RuntimeError("no allocator here") + + monkeypatch.setattr(alloc_conf.torch.cuda, "memory", _NoSnapshot) + assert alloc_conf.expandable_segments_enabled() is None diff --git a/tests/distributed/test_custom_all_gather_reduce_scatter.py b/tests/distributed/test_custom_all_gather_reduce_scatter.py index dde5fdf7ff..61666110e5 100644 --- a/tests/distributed/test_custom_all_gather_reduce_scatter.py +++ b/tests/distributed/test_custom_all_gather_reduce_scatter.py @@ -23,6 +23,7 @@ import torch import torch.distributed as dist from vllm.distributed.parallel_state import get_tp_group +from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter from vllm.platforms import current_platform from ..utils import ( @@ -53,6 +54,11 @@ def _supports_multimem(): ) +def _supports_sm100_or_sm103(): + capability = current_platform.get_device_capability() + return current_platform.is_cuda() and capability in ((10, 0), (10, 3)) + + def _payload(value, device): return torch.full((_ROWS, _HIDDEN), value, dtype=torch.bfloat16, device=device) @@ -324,3 +330,65 @@ def test_mnnvl_lamport_stage_cleanup(monkeypatch: pytest.MonkeyPatch): if torch.accelerator.device_count() < 2: pytest.skip("Need at least two GPUs to run the test.") multi_process_parallel(monkeypatch, 2, 1, _run_stage_cleanup_test) + + +@ray.remote(num_gpus=1, max_calls=1) +def _run_multimem_reduce_scatter_test( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pp_size, + rank, + distributed_init_port, +): + with monkeypatch.context() as m: + m.delenv("CUDA_VISIBLE_DEVICES", raising=False) + m.delenv("HIP_VISIBLE_DEVICES", raising=False) + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) + ensure_model_parallel_initialized(tp_size, pp_size) + + fa = get_tp_group().device_communicator.ca_comm + assert fa is not None and not fa.disabled + assert fa.mnnvl_multimem_rs_supported + assert not fa.mnnvl_multimem_rs_initialized + assert not fa.mnnvl_multimem_rs_multicast_ptr + + message_bytes = { + "mnnvl_lamport": 1 * 1024 * 1024, + "mnnvl_multimem": 20 * 1024 * 1024, + "nccl": 64 * 1024 * 1024 + 128, + } + for _ in range(2): + for backend, size_bytes in message_bytes.items(): + shard_elements = size_bytes // torch.bfloat16.itemsize // tp_size + inp = torch.full( + (tp_size, shard_elements), + rank + 1, + dtype=torch.bfloat16, + device=device, + ) + selected = fa._select_reduce_scatter_backend(inp) + if backend == "nccl": + assert selected is None + out = sp_reduce_scatter(inp) + else: + assert selected == backend + out = fa.custom_reduce_scatter(inp) + assert out is not None + if backend == "mnnvl_multimem": + assert fa.mnnvl_multimem_rs_initialized + assert fa.mnnvl_multimem_rs_multicast_ptr + expected = tp_size * (tp_size + 1) // 2 + torch.testing.assert_close(out, torch.full_like(out, expected)) + + +@pytest.mark.skipif( + not _supports_sm100_or_sm103(), + reason="The low-SM MNNVL reduce-scatter path requires SM100 or SM103.", +) +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +def test_mnnvl_multimem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, tp_size: int): + if torch.accelerator.device_count() < tp_size: + pytest.skip(f"Need at least {tp_size} GPUs to run the test.") + multi_process_parallel(monkeypatch, tp_size, 1, _run_multimem_reduce_scatter_test) diff --git a/tests/distributed/test_custom_all_reduce.py b/tests/distributed/test_custom_all_reduce.py index 7119e596a0..23d66ee973 100644 --- a/tests/distributed/test_custom_all_reduce.py +++ b/tests/distributed/test_custom_all_reduce.py @@ -25,12 +25,43 @@ for i, v in enumerate(test_sizes): test_sizes[i] -= v % 8 +def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int: + """Largest distance between two BF16 tensors in units in the last place.""" + + def ordered(t: torch.Tensor) -> torch.Tensor: + bits = t.view(torch.int16).int() & 0xFFFF + return torch.where(bits >= 0x8000, 0x8000 - bits, bits) + + return int((ordered(a) - ordered(b)).abs().max()) + + +def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps): + """All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32 + order, rounding to BF16 where the unfused path returns BF16.""" + reduced = peers[0].float() + for peer in peers[1:]: + reduced = reduced + peer.float() + reduced = reduced.bfloat16().float() + output = torch.empty_like(residual) + collapse = torch.zeros_like(reduced) + for target in range(4): + mixed = reduced * post[:, target : target + 1] + for source in range(4): + mixed = torch.addcmul( + mixed, residual[:, source].float(), comb[:, source, target, None] + ) + output[:, target] = mixed.bfloat16() + collapse = torch.addcmul( + collapse, output[:, target].float(), pre[:, target : target + 1] + ) + prenorm = collapse.bfloat16().float() + inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps) + return output, (prenorm * inv_rms * weight.float()).bfloat16() + + @ray.remote(num_gpus=1, max_calls=1) def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port): - from vllm.model_executor.kernels.mhc.tilelang import ( - mhc_post_tilelang, - mhc_pre_delayed_tilelang, - ) + from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC with monkeypatch.context() as m: m.delenv("CUDA_VISIBLE_DEVICES", raising=False) @@ -38,98 +69,114 @@ def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port): torch.accelerator.set_device_index(device) init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) ensure_model_parallel_initialized(tp_size, pp_size) - comm = get_tp_group().device_communicator.ca_comm - assert comm is not None and comm.mnnvl_lamport_ag_multicast_ptr - fn = torch.zeros(24, 20480, device=device) - scale = torch.ones(3, device=device) - base = torch.zeros(24, device=device) + op = AllReduceMHC( + hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device + ) - # Changing shapes and interleaving all-gather exercise the shared - # Lamport stages, including cleanup of a larger previous payload. - def run(n): - torch.manual_seed(42 + rank) - x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16) - # Packed +0/-0 pairs collide with the Lamport sentinel. - x[:, :16] = 0 - x[:, 9:16:2] = -0.0 + def check_eager_and_replayed(fused, check, halved, doubled): + check(fused()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(5): + fused() + captured = fused() + for _ in range(20): + graph.replay() + check(captured) + # Replays must pick up in-place input changes. + halved.mul_(0.5) + doubled.mul_(2) + graph.replay() + check(captured) + + def mhc_inputs(n): torch.manual_seed(123) residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16) post = torch.rand(n, 4, device=device) comb = torch.randn(n, 4, 4, device=device) * 0.1 pre = torch.rand(n, 4, device=device) weight = torch.randn(5120, device=device, dtype=torch.bfloat16) - output = torch.empty_like(residual) - normalized = torch.empty_like(x) + return residual, post, comb, pre, weight + + def run(n): + torch.manual_seed(42 + rank) + x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16) + # Packed +0/-0 pairs collide with the Lamport sentinel. + x[:, :16] = 0 + x[:, 9:16:2] = -0.0 + residual, post, comb, pre, weight = mhc_inputs(n) def fused(): - torch.ops._C_custom_ar.all_reduce_mhc( - x, + return op(x, residual, post, comb, pre, weight, 1e-6) + + def check(outputs): + peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120) + output, normalized = _unfused_all_reduce_mhc( + peers, residual, post, comb, pre, weight, 1e-6 + ) + # The mixed hc streams match bit for bit. The RMSNorm sums the + # squares in another order and uses an approximate rsqrt. + assert torch.equal(outputs[0], output) + assert _bf16_ulps(outputs[1], normalized) <= 1 + + check_eager_and_replayed(fused, check, x, residual) + + def run_finalize(n): + torch.manual_seed(7 + rank) + # A padded permuted GEMM2 buffer, like the MoE's. + rows = n * 6 + 5 + gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16) + permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int() + # A route to an expert this rank does not hold. + permuted[0, -1] = -1 + weights = torch.rand(n, 6, device=device) + shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16) + residual, post, comb, pre, weight = mhc_inputs(n) + + def fused(): + return op.finalize( + gemm2, + weights, + permuted, + shared, residual, post, comb, pre, weight, - output, - normalized, - comm.mnnvl_lamport_ag_local_ptr, - comm.mnnvl_lamport_ag_multicast_ptr, - comm.mnnvl_lamport_epochs[0], - rank, - comm.mnnvl_buffer_size, 1e-6, ) - def check(): - gathered = comm.custom_all_gather(x) - assert gathered is not None - peers = gathered.view(tp_size, n, 5120).float() - reduced = peers[0].clone() - for peer in peers[1:]: - reduced.add_(peer) - expected = mhc_post_tilelang( - reduced.bfloat16(), residual, post.unsqueeze(-1), comb - ) - expected_norm = mhc_pre_delayed_tilelang( - expected, - fn, - scale, - base, - 1e-6, - 1e-6, - 1e-6, - 2.0, - 20, - pre_mix=pre, - norm_weight=weight, - )[2] - torch.testing.assert_close(output, expected, rtol=0.008, atol=1e-6) - torch.testing.assert_close( - normalized, expected_norm, rtol=0.008, atol=0.008 - ) + def check(outputs): + # One FP32 FMA per route in route order, the shared add, one + # rounding, then the plain path: only the finalize differs. + acc = torch.zeros(n, 5120, device=device) + for k in range(6): + valid = (permuted[:, k] >= 0).unsqueeze(-1) + rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float() + torch.addcmul( + acc, + torch.where(valid, rows_k, 0.0), + torch.where(valid, weights[:, k : k + 1], 0.0), + out=acc, + ) + x = (acc + shared.float()).bfloat16() + expected = op(x, residual, post, comb, pre, weight, 1e-6) + assert torch.equal(outputs[0], expected[0]) + assert torch.equal(outputs[1], expected[1]) - fused() - check() - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - for _ in range(6): - fused() - for _ in range(20): - graph.replay() - check() - x.mul_(0.5) - residual.mul_(2) - graph.replay() - check() + check_eager_and_replayed(fused, check, gemm2, shared) - # Cover fixed Q6 and shrinking/growing adaptive verification batches. + # Changing shapes cover shrinking and growing batches in one mailbox. for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1): run(n) + run_finalize(n) @pytest.mark.skipif( not current_platform.is_device_capability_family(100), reason="Requires SM100" ) -def test_all_reduce_mhc_preserves_bf16_boundaries_and_graph_replay(monkeypatch): +def test_all_reduce_mhc_matches_unfused_path(monkeypatch): if torch.accelerator.device_count() < 4: pytest.skip("Requires four GPUs with NVLink multicast") multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc) @@ -151,9 +198,9 @@ def test_custom_allreduce_filters_dtype( ) -> None: communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) communicator.disabled = False + communicator._ptr = 0 communicator.world_size = 2 communicator.max_size = 1024 - communicator._ptr = 0 assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected @@ -219,6 +266,268 @@ def test_local_multicast_support_rejects_non_cuda(monkeypatch): assert not car._has_local_multicast_support(torch.device("cuda:0")) +@pytest.mark.parametrize( + ("world_size", "device_capability", "local_multicast", "expected"), + [ + (2, (10, 0), True, True), + (4, (10, 3), True, True), + (8, (10, 0), True, True), + (8, (10, 3), True, True), + (6, (10, 3), True, False), + (8, (10, 1), True, False), + (8, (9, 0), True, False), + (8, (10, 3), False, False), + ], +) +def test_mnnvl_multimem_reduce_scatter_platform_gate( + monkeypatch, + world_size, + device_capability, + local_multicast, + expected, +): + def is_device_capability(capability, device_id): + assert capability in ((10, 0), (10, 3)) + assert device_id == 3 + return device_capability == capability + + monkeypatch.setattr( + car.current_platform, + "is_device_capability", + is_device_capability, + ) + monkeypatch.setattr( + car, + "_has_local_multicast_support", + lambda _device: local_multicast, + ) + + supported = car._supports_mnnvl_multimem_reduce_scatter( + torch.device("cuda:3"), world_size + ) + assert supported is expected + + +@pytest.mark.parametrize( + ( + "message_bytes", + "multimem_ptr", + "multimem_initialized", + "batch_invariant", + "expected", + ), + [ + (16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"), + (16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"), + (64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"), + (64 * 1024 * 1024 + 128, 1, True, False, None), + (32 * 1024 * 1024, 0, True, False, None), + (32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"), + (8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"), + (32 * 1024 * 1024, 1, True, True, None), + ], +) +@pytest.mark.parametrize("world_size", [2, 4, 8]) +def test_mnnvl_reduce_scatter_backend_gate( + monkeypatch, + world_size, + message_bytes, + multimem_ptr, + multimem_initialized, + batch_invariant, + expected, +): + monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant) + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator._ptr = 0 + communicator.world_size = world_size + communicator.mnnvl_only = False + communicator.fully_connected = True + communicator.mnnvl_multicast_ptr = 1 + communicator.mnnvl_multimem_rs_supported = True + communicator.mnnvl_multimem_rs_initialized = multimem_initialized + communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr + communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024 + communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024 + communicator.max_reduce_scatter_size = 16 * 1024 * 1024 + inp = torch.empty( + (world_size, message_bytes // torch.bfloat16.itemsize // world_size), + dtype=torch.bfloat16, + ) + + assert inp.nbytes == message_bytes + assert communicator._select_reduce_scatter_backend(inp) == expected + assert communicator.should_custom_reduce_scatter(inp) is (expected is not None) + assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is ( + expected == "mnnvl_multimem" + ) + + +def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure( + monkeypatch, +): + events = [] + + class FakeSymmMem: + @staticmethod + def empty(*_args, **_kwargs): + events.append("empty") + return torch.empty(1, dtype=torch.uint8) + + @staticmethod + def rendezvous(*_args, **_kwargs): + events.append("rendezvous") + return None + + def report_peer_allocation_failure(group_value, **_kwargs): + events.append("all_reduce") + assert group_value.item() == 1 + group_value.zero_() + + monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem) + monkeypatch.setattr(car.ops, "meta_size", lambda: 128) + monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure) + warnings = [] + monkeypatch.setattr( + car.logger, + "warning_once", + lambda message, *_args, **_kwargs: warnings.append(message), + ) + + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = True + communicator._ptr = 0 + communicator.group = object() + communicator.device = torch.device("cuda:0") + communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024 + communicator.mnnvl_multimem_rs_supported = True + communicator.mnnvl_multimem_rs_initialized = False + communicator.mnnvl_multimem_rs_buffer = None + communicator.mnnvl_multimem_rs_multicast_ptr = 0 + + communicator._init_mnnvl_multimem_reduce_scatter_buffer() + + assert events == ["empty", "all_reduce"] + assert communicator.mnnvl_multimem_rs_initialized + assert communicator.mnnvl_multimem_rs_buffer is None + assert communicator.mnnvl_multimem_rs_multicast_ptr == 0 + assert warnings == [ + "MNNVL multimem reduce-scatter symmetric-memory allocation " + "failed on at least one rank; falling back to NCCL." + ] + + +def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch): + events = [] + + class FakeSymmMem: + @staticmethod + def empty(*_args, **_kwargs): + events.append("empty") + return torch.empty(1, dtype=torch.uint8) + + @staticmethod + def rendezvous(*_args, **_kwargs): + events.append("rendezvous") + raise RuntimeError("rendezvous failed") + + def preserve_local_result(_group_value, **_kwargs): + events.append("all_reduce") + + warnings = [] + monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem) + monkeypatch.setattr(car.ops, "meta_size", lambda: 128) + monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result) + monkeypatch.setattr( + car.logger, + "warning_once", + lambda message, *_args, **_kwargs: warnings.append(message), + ) + + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = True + communicator._ptr = 0 + communicator.group = type("Group", (), {"group_name": "test"})() + communicator.device = torch.device("cuda:0") + communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024 + communicator.mnnvl_multimem_rs_supported = True + communicator.mnnvl_multimem_rs_initialized = False + communicator.mnnvl_multimem_rs_buffer = None + communicator.mnnvl_multimem_rs_multicast_ptr = 0 + + communicator._init_mnnvl_multimem_reduce_scatter_buffer() + + assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"] + assert communicator.mnnvl_multimem_rs_initialized + assert communicator.mnnvl_multimem_rs_buffer is None + assert communicator.mnnvl_multimem_rs_multicast_ptr == 0 + assert warnings == [ + "MNNVL multimem reduce-scatter symmetric-memory rendezvous " + "failed on at least one rank; falling back to NCCL." + ] + + +def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch): + events = [] + buffers = [] + + class FakeHandle: + multicast_ptr = 0x3000 + + class FakeSymmMem: + @staticmethod + def empty(size, **_kwargs): + events.append(("empty", size)) + buffer = torch.ones(size, dtype=torch.uint8) + buffers.append(buffer) + return buffer + + @staticmethod + def rendezvous(*_args, **_kwargs): + events.append(("rendezvous", None)) + return FakeHandle() + + def preserve_local_result(*_args, **_kwargs): + events.append(("all_reduce", None)) + + monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem) + monkeypatch.setattr(car.ops, "meta_size", lambda: 128) + monkeypatch.setattr( + car.torch.accelerator, + "synchronize", + lambda: events.append(("synchronize", None)), + ) + monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result) + + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = True + communicator._ptr = 0 + communicator.group = type("Group", (), {"group_name": "test"})() + communicator.device = torch.device("cpu") + communicator.max_mnnvl_multimem_reduce_scatter_size = 129 + communicator.mnnvl_multimem_rs_supported = True + communicator.mnnvl_multimem_rs_initialized = False + communicator.mnnvl_multimem_rs_buffer = None + communicator.mnnvl_multimem_rs_multicast_ptr = 0 + + communicator._init_mnnvl_multimem_reduce_scatter_buffer() + + assert events == [ + ("empty", 257), + ("all_reduce", None), + ("rendezvous", None), + ("synchronize", None), + ("all_reduce", None), + ] + assert torch.all(buffers[0][:128] == 0) + assert torch.all(buffers[0][128:] == 1) + assert communicator.mnnvl_multimem_rs_buffer_size == 129 + assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128 + assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080 + + @ray.remote(num_gpus=1, max_calls=1) def graph_allreduce( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py index e5764954bd..b3d9fdddd3 100644 --- a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py +++ b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py @@ -455,13 +455,29 @@ def test_mla_dcp_manager_selects_fallback_backends(monkeypatch): ) -def test_dcp_workspace_covers_parallel_drafting(): +@pytest.mark.parametrize( + ("parallel_drafting", "use_v2_model_runner", "expected"), + [ + # Model runner V1's parallel drafter widens requests to 1 + 2 * 3. + (True, False, 28), + (True, True, 16), + # A diffusion canvas of 3 without a speculative config. + (None, True, 16), + ], +) +def test_dcp_workspace_covers_decode_width( + parallel_drafting, use_v2_model_runner, expected +): config = _manager_config() config.scheduler_config.max_num_batched_tokens = 128 config.num_speculative_tokens = 3 - config.speculative_config = MagicMock(parallel_drafting=True) + config.use_v2_model_runner = use_v2_model_runner + if parallel_drafting is not None: + config.speculative_config = MagicMock( + num_speculative_tokens=3, parallel_drafting=parallel_drafting + ) - assert dcp.get_dcp_workspace_max_num_tokens(config) == 28 + assert dcp.get_dcp_workspace_max_num_tokens(config) == expected def test_mla_dcp_manager_selects_pcp_combine(monkeypatch): diff --git a/tests/distributed/test_weight_transfer_m2n.py b/tests/distributed/test_weight_transfer_m2n.py new file mode 100644 index 0000000000..0591adff4d --- /dev/null +++ b/tests/distributed/test_weight_transfer_m2n.py @@ -0,0 +1,868 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the NCCL M2N weight transfer backend. + +These cover how a transfer is *described* — layout encoding and the wire-type +validation that keeps a bad plan from reaching a collective, since a mismatch +there is a hang rather than an error. The transfer itself needs the m2n runtime +and multiple GPUs, so it is exercised separately. +""" + +import logging +from functools import wraps +from types import MethodType +from unittest.mock import Mock + +import pybase64 as base64 +import pytest +import ray +import torch + +from vllm.distributed.weight_transfer import ( + WeightTransferEngineFactory, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.m2n_common import ( + REPLICATE, + REPLICATED, + M2NMesh, + M2NParamMeta, + check_placements, + check_transferable, + publish_destination_placements, + resolve_layout, + validate_layout, +) +from vllm.distributed.weight_transfer.m2n_engine import ( + M2NWeightTransferEngine, + M2NWeightTransferInitInfo, + M2NWeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.m2n_layout import ( + M2NDestination, + resolve_parameter_destinations, +) +from vllm.distributed.weight_transfer.m2n_source import ( + mesh_from_tensor, + placements_from_tensor, +) +from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.platforms import current_platform + +VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode() + + +class TestLayout: + def test_replicated_splits_nothing(self): + """A replicated tensor imposes no divisibility constraint. The shape + dims are deliberately coprime with the mesh size: a layout that actually + split the tensor would reject them.""" + mesh = M2NMesh((2, 2), start_rank=1) + indivisible_shape = (7, 13) # neither dim divisible by any mesh axis + + resolved_mesh, placements = resolve_layout(mesh, REPLICATED) + validate_layout(resolved_mesh, placements, indivisible_shape, "destination") + + def test_replicated_keeps_the_same_ranks(self): + """Replication re-factors the mesh to get a size-1 axis for its no-op + shard. That is only sound if it still covers exactly the same GPUs.""" + mesh = M2NMesh((2, 3), start_rank=4) + resolved_mesh, _ = resolve_layout(mesh, REPLICATED) + assert resolved_mesh.size == mesh.size + assert resolved_mesh.start_rank == mesh.start_rank + + def test_sharded_keeps_its_own_factorization(self): + """Rank order decides who owns which shard, so a sharded tensor must + not be re-factored the way a replicated one is.""" + mesh = M2NMesh((2, 3), start_rank=4) + resolved_mesh, placements = resolve_layout(mesh, (REPLICATE, 0)) + assert resolved_mesh == mesh + assert placements == (REPLICATE, 0) + + def test_two_shard_axes_rejected(self): + """One axis has to replicate; a 2-D mesh that shards both is not + something a single reshard can express.""" + with pytest.raises(ValueError, match="shards both"): + check_placements((0, 1)) + + def test_placement_below_replicate_rejected(self): + with pytest.raises(ValueError, match=r"parameter 'w'.*-2"): + check_placements((-2, REPLICATE), "parameter 'w' source placements") + + def test_shard_dim_must_exist(self): + with pytest.raises(ValueError, match="rank 2"): + validate_layout(M2NMesh((1, 2), 0), (REPLICATE, 2), (8, 16), "source") + + def test_shard_must_divide_evenly(self): + with pytest.raises(ValueError, match="does not divide evenly"): + validate_layout(M2NMesh((1, 3), 0), (REPLICATE, 0), (8, 16), "source") + + +class TestSourceLayout: + def test_plain_tensor_is_replicated_across_trainer_ranks(self): + """A tensor with no DTensor metadata is the same on every trainer rank, + so the source describes it as replicated over all of them.""" + assert placements_from_tensor(torch.zeros(4)) is REPLICATED + assert mesh_from_tensor(torch.zeros(4), 4) == M2NMesh((4, 1), 0) + + +class TestTransferable: + def test_unsupported_dtype_names_the_parameter(self): + with pytest.raises(ValueError, match="'w'"): + check_transferable("w", torch.complex64, (4,)) + + def test_rank_four_rejected(self): + with pytest.raises(ValueError, match="rank 4"): + check_transferable("w", torch.bfloat16, (2, 2, 2, 2)) + + +class TestWireTypes: + def _init_info(self, **overrides): + fields = dict( + master_address="127.0.0.1", + master_port=1234, + rank_offset=1, + world_size=3, + src_mesh_dims=[1, 1], + dst_mesh_dims=[2, 1], + names=["w"], + dtype_names=["bfloat16"], + shapes=[[16, 16]], + src_placements=[None], + ) + fields.update(overrides) + return M2NWeightTransferInitInfo(**fields) + + def test_accepts_a_consistent_plan(self): + assert self._init_info().names == ["w"] + + def test_accepts_pre_shared_nccl_unique_id(self): + info = self._init_info( + master_address=None, + master_port=None, + nccl_unique_id_b64=VALID_UID_B64, + ) + assert info.nccl_unique_id_bytes == b"\x00" * 128 + assert VALID_UID_B64 not in repr(info) + + def test_destination_plan_uses_uid_communicator_without_group(self): + comm = Mock(rank=0, device=torch.device("cpu"), group=None) + + def receive_plan(tensor, src): + assert src == 1 + tensor.copy_(torch.tensor([[-1, 0], [-2, -2]], dtype=torch.int8)) + + comm.broadcast.side_effect = receive_plan + + placements = publish_destination_placements(comm, 1, None, 2) + + assert placements == [(REPLICATE, 0), REPLICATED] + + def test_rejects_both_rendezvous_modes(self): + with pytest.raises(ValueError, match="not both"): + self._init_info(nccl_unique_id_b64=VALID_UID_B64) + + def test_trainer_accepts_pre_shared_nccl_unique_id(self): + info = M2NTrainerInitInfo( + nccl_unique_id_b64=VALID_UID_B64, + world_size=3, + num_trainer_ranks=1, + rank=0, + ) + assert info.nccl_unique_id_bytes == b"\x00" * 128 + assert VALID_UID_B64 not in repr(info) + + def test_ragged_plan_rejected(self): + with pytest.raises(ValueError, match="`shapes`"): + self._init_info(shapes=[]) + + def test_destination_mesh_must_cover_the_workers(self): + """The trainer declares the inference mesh, so one that does not cover + the workers is a config error — and it has to fail the init RPC, since + a mismatched mesh would otherwise surface as a hung collective.""" + with pytest.raises(ValueError, match="dst_mesh_dims"): + self._init_info(dst_mesh_dims=[3, 1]) # 3 != the 2 workers + + def test_world_must_hold_a_trainer_and_a_worker(self): + with pytest.raises(ValueError, match="rank_offset"): + self._init_info(rank_offset=3, world_size=3) + + @pytest.mark.parametrize("dtype_name", ["not_a_dtype", "Tensor"]) + def test_invalid_dtype_name_names_parameter(self, monkeypatch, dtype_name): + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine.import_m2n", + lambda: object(), + ) + engine = object.__new__(M2NWeightTransferEngine) + + with pytest.raises(ValueError, match=r"parameter 'w'.*dtype"): + engine.init_transfer_engine(self._init_info(dtype_names=[dtype_name])) + + def _init_32_to_4_plan(self, monkeypatch, destination): + m2n = Mock() + m2n.Handle.create.return_value = object() + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine.import_m2n", lambda: m2n + ) + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine." + "resolve_parameter_destinations", + lambda *args, **kwargs: [destination], + ) + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine.worker_init_process_group", + lambda *args, **kwargs: object(), + ) + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine." + "publish_destination_placements", + lambda *args: args[2], + ) + + engine = object.__new__(M2NWeightTransferEngine) + engine.parallel_config = Mock(pipeline_parallel_size=1) + engine.model_config = Mock(quantization=None) + engine.model = torch.nn.Module() + engine.init_transfer_engine( + self._init_info( + rank_offset=32, + world_size=36, + src_mesh_dims=[1, 32], + dst_mesh_dims=[1, 4], + shapes=[[32, 8]], + src_placements=[[REPLICATE, 0]], + ) + ) + + def test_limits_use_resolved_sharded_destination(self, monkeypatch): + destination = M2NDestination("w", (REPLICATE, 0), torch.empty(8, 8)) + + self._init_32_to_4_plan(monkeypatch, destination) + + def test_limits_still_reject_replicated_destination(self, monkeypatch): + destination = M2NDestination("w", REPLICATED, None) + + with pytest.raises(ValueError, match=r"32 source shards.*MAX_SOURCES=16"): + self._init_32_to_4_plan(monkeypatch, destination) + + def test_update_preflights_all_names_before_reshard(self): + engine = object.__new__(M2NWeightTransferEngine) + engine._handle = object() + engine.model_update_group = object() + engine._index = {"valid": 0} + engine._metas = [M2NParamMeta("valid", torch.float32, (4,), REPLICATED)] + engine._reshard = Mock() + + with pytest.raises(ValueError, match=r"parameter 'unknown'"): + engine.receive_weights( + M2NWeightTransferUpdateInfo(names=["valid", "unknown"]) + ) + + engine._reshard.assert_not_called() + + def test_fallback_tensors_share_one_load_weights_invocation(self, monkeypatch): + class CoupledModel: + def __init__(self): + self.calls = 0 + self.loaded = [] + + def load_weights(self, weights): + self.calls += 1 + received = list(weights) + if [name for name, _ in received] == ["pair.weight", "pair.scale"]: + self.loaded = [(name, tensor.clone()) for name, tensor in received] + + model = CoupledModel() + direct = torch.zeros(1) + engine = object.__new__(M2NWeightTransferEngine) + engine._handle = object() + engine.model_update_group = object() + engine.device = torch.device("cpu") + engine.model = model + engine._index = {"pair.weight": 0, "direct": 1, "pair.scale": 2} + engine._metas = [ + M2NParamMeta(name, torch.float32, (1,), REPLICATED) + for name in engine._index + ] + engine._parameter_destinations = [ + M2NDestination("pair.weight", REPLICATED, None), + M2NDestination("direct", REPLICATED, direct), + M2NDestination("pair.scale", REPLICATED, None), + ] + reshard_order = [] + + def reshard(comm, stream, meta, placements, buffer): + reshard_order.append(meta.name) + buffer.fill_(len(reshard_order)) + + engine._reshard = reshard + stream = Mock() + monkeypatch.setattr( + "vllm.distributed.weight_transfer.m2n_engine.comm_ptr", lambda group: 0 + ) + monkeypatch.setattr(torch.cuda, "current_stream", lambda: stream) + + engine.receive_weights( + M2NWeightTransferUpdateInfo(names=["pair.weight", "direct", "pair.scale"]) + ) + + assert model.calls == 1 + assert [name for name, _ in model.loaded] == ["pair.weight", "pair.scale"] + assert [tensor.item() for _, tensor in model.loaded] == [1, 3] + assert direct.item() == 2 + assert reshard_order == ["pair.weight", "direct", "pair.scale"] + assert stream.synchronize.call_count == 2 + + def test_trainer_rank_must_be_a_trainer_rank(self): + """A trainer rank must fall within the trainer portion of the group.""" + with pytest.raises(ValueError, match="num_trainer_ranks"): + M2NTrainerInitInfo( + master_address="127.0.0.1", + master_port=1234, + world_size=4, + num_trainer_ranks=2, + rank=2, + ) + + def test_trainer_destination_mesh_must_cover_the_workers(self): + """The destination mesh must include every inference worker.""" + with pytest.raises(ValueError, match="dst_mesh_dims"): + M2NTrainerInitInfo( + master_address="127.0.0.1", + master_port=1234, + world_size=6, + num_trainer_ranks=2, + dst_mesh_dims=(3, 1), # 3 != the 4 inference workers + rank=0, + ) + + def test_destination_mesh_defaults_to_flat(self): + """A replicated destination does not care how the mesh is factored, so + callers that do not shard it need not supply one.""" + info = M2NTrainerInitInfo( + master_address="127.0.0.1", + master_port=1234, + world_size=6, + num_trainer_ranks=2, + rank=0, + ) + assert info.destination_mesh_dims == (4, 1) + + def test_sender_is_trainer_rank_zero(self): + """Trainer rank 0 drives the inference control plane.""" + info = M2NTrainerInitInfo( + master_address="127.0.0.1", master_port=1234, world_size=4, rank=0 + ) + assert info.is_sender + + +class TestRegistration: + def test_both_registries_expose_the_backend(self): + """Both worker and trainer factories register nccl_m2n.""" + assert "nccl_m2n" in WeightTransferEngineFactory._registry + assert "nccl_m2n" in WeightTransferTrainerFactory._registry + + def test_init_info_dispatches_to_the_backend(self): + """Trainer init info selects the nccl_m2n factory entry.""" + assert M2NTrainerInitInfo.backend == "nccl_m2n" + + +# --------------------------------------------------------------------------- +# End-to-end transfer +# +# Transport-only, in the style of the NCCL/sparse tests in +# test_weight_transfer.py: two Ray tasks with one GPU each drive the two engines +# directly, so no HTTP server or LLM instance is involved. The worker is not an +# RPC endpoint here, so the trainer engine gets a no-op control-plane client -- +# the NCCL rendezvous and the reshard itself are the real thing. +# --------------------------------------------------------------------------- + +SHAPE = [64, 32] +DTYPE = "float32" + + +def _init_ray() -> None: + if ray.is_initialized(): + return + ray.init( + ignore_reinit_error=True, + runtime_env={ + "env_vars": { + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", + "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1", + } + }, + ) + + +def _assigned_device() -> "torch.device": + gpu_ids = ray.get_gpu_ids() + device = torch.device(f"cuda:{int(gpu_ids[0])}" if gpu_ids else "cuda:0") + current_platform.set_device(device) + return device + + +@ray.remote(num_gpus=1) +def _m2n_trainer_send(nccl_unique_id_b64: str, world_size: int) -> bool: + """Send one parameter through the real trainer engine.""" + device = _assigned_device() + + from vllm.distributed.weight_transfer import WeightTransferTrainerFactory + from vllm.distributed.weight_transfer.m2n_source import DTensorModuleSource + from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo + + class NoopClient: + def init_weight_transfer_engine(self, init_info): + pass + + def start_weight_update(self): + pass + + def update_weights(self, update_info): + pass + + def finish_weight_update(self, weight_version=None): + pass + + class Tiny(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter( + torch.arange( + SHAPE[0] * SHAPE[1], dtype=torch.float32, device=device + ).reshape(SHAPE) + ) + + engine = WeightTransferTrainerFactory.trainer_init( + init_info=M2NTrainerInitInfo( + nccl_unique_id_b64=nccl_unique_id_b64, + world_size=world_size, + num_trainer_ranks=1, + dst_mesh_dims=(1, world_size - 1), + rank=0, + ), + client=NoopClient(), + source=DTensorModuleSource(Tiny(), num_trainer_ranks=1), + ) + engine.send_weights() + torch.accelerator.synchronize() + engine.shutdown() + return True + + +@ray.remote(num_gpus=1) +def _m2n_worker_receive( + nccl_unique_id_b64: str, + world_size: int, + worker_rank: int = 0, +) -> dict: + """Receive that parameter through the real worker engine.""" + import contextlib + from unittest.mock import MagicMock + + device = _assigned_device() + + from vllm.config.parallel import ParallelConfig + from vllm.config.weight_transfer import WeightTransferConfig + from vllm.distributed.weight_transfer.m2n_engine import ( + M2NWeightTransferEngine, + M2NWeightTransferInitInfo, + M2NWeightTransferUpdateInfo, + ) + + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + if world_size > 2: + self.weight = torch.nn.Parameter( + torch.zeros( + SHAPE[0] // (world_size - 1), + SHAPE[1], + dtype=torch.float32, + device=device, + ) + ) + self.weight.output_dim = 0 + self.weight.weight_loader = MethodType( + ColumnParallelLinear.weight_loader, + Mock(tp_size=world_size - 1), + ) + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + + parallel_config = MagicMock(spec=ParallelConfig) + parallel_config.rank = worker_rank + parallel_config.world_size = world_size - 1 + parallel_config.data_parallel_rank = 0 + parallel_config.data_parallel_index = 0 + parallel_config.tensor_parallel_size = world_size - 1 + parallel_config.pipeline_parallel_size = 1 + vllm_config = MagicMock() + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() + vllm_config.model_config.quantization = None + + recorder = Recorder() + engine = M2NWeightTransferEngine( + WeightTransferConfig(backend="nccl_m2n"), vllm_config, device, recorder + ) + # Transport-only: receive_weights enters set_current_vllm_config, and + # vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() + + engine.init_transfer_engine( + M2NWeightTransferInitInfo( + nccl_unique_id_b64=nccl_unique_id_b64, + rank_offset=1, # trainer occupies rank 0 + world_size=world_size, + src_mesh_dims=[1, 1], + dst_mesh_dims=[1, world_size - 1], + names=["weight"], + dtype_names=[DTYPE], + shapes=[SHAPE], + src_placements=[None], # replicated on the single trainer rank + ) + ) + engine.receive_weights(M2NWeightTransferUpdateInfo(names=["weight"])) + torch.accelerator.synchronize() + + full = torch.arange( + SHAPE[0] * SHAPE[1], dtype=torch.float32, device=device + ).reshape(SHAPE) + direct = world_size > 2 + name: str | None + got: torch.Tensor | None + if direct: + name, got = "weight", recorder.weight + expected = full.chunk(world_size - 1, dim=0)[worker_rank] + else: + name, got = recorder.received[0] if recorder.received else (None, None) + expected = full + result = { + "count": len(recorder.received), + "name": name, + "shape": list(got.shape) if got is not None else None, + "exact": bool(torch.equal(got, expected)) if got is not None else False, + "direct": direct, + } + engine.shutdown() + return result + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 2, + reason="Need at least 2 GPUs: one trainer rank and one inference worker.", +) +def test_m2n_weight_transfer_between_processes(): + """A parameter survives a real reshard from a trainer process to a worker. + + This is the only test here that moves bytes: it builds both engines, joins + one NCCL communicator across two processes, and reshards. Everything else + in this file checks how a transfer is *described*. + """ + pytest.importorskip("nccl.m2n", reason="nccl_m2n backend needs the m2n runtime") + _init_ray() + + from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary + + nccl = NCCLLibrary() + nccl_unique_id_b64 = base64.b64encode( + bytes(nccl.ncclGetUniqueId().internal) + ).decode() + world_size = 2 # 1 trainer + 1 inference worker + + worker = _m2n_worker_receive.remote(nccl_unique_id_b64, world_size) + trainer = _m2n_trainer_send.remote(nccl_unique_id_b64, world_size) + trainer_ok, result = ray.get([trainer, worker]) + + assert trainer_ok, "trainer engine did not complete" + assert result["count"] == 1, f"expected one parameter, got {result['count']}" + assert result["name"] == "weight" + assert result["shape"] == SHAPE + assert result["exact"], "resharded tensor does not match what the trainer sent" + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 3, + reason="Need at least 3 GPUs: one trainer rank and two inference workers.", +) +def test_m2n_weight_transfer_to_tp2_shards(): + """Two inference workers receive their own TP shard in live model storage.""" + pytest.importorskip("nccl.m2n", reason="nccl_m2n backend needs the m2n runtime") + _init_ray() + + from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary + + nccl = NCCLLibrary() + nccl_unique_id_b64 = base64.b64encode( + bytes(nccl.ncclGetUniqueId().internal) + ).decode() + world_size = 3 + + workers = [ + _m2n_worker_receive.remote(nccl_unique_id_b64, world_size, rank) + for rank in range(2) + ] + trainer = _m2n_trainer_send.remote(nccl_unique_id_b64, world_size) + trainer_ok, *results = ray.get([trainer, *workers]) + + assert trainer_ok, "trainer engine did not complete" + for rank, result in enumerate(results): + assert result["count"] == 0, f"worker {rank} unexpectedly called load_weights" + assert result["name"] == "weight" + assert result["shape"] == [SHAPE[0] // 2, SHAPE[1]] + assert result["direct"] + assert result["exact"], f"worker {rank} received the wrong shard" + + +class _Model(torch.nn.Module): + """Stand-in for a TP=2 shard of a tiny model. + + `column` is sharded on dim 0 and `row` on dim 1 (the two shapes a TP linear + layer produces), `norm` is replicated, and `fused` has no checkpoint-name + counterpart — the shape a fused parameter presents to the resolver. + """ + + def __init__(self) -> None: + super().__init__() + self.column = torch.nn.Parameter(torch.zeros(8, 16)) + self.column.output_dim = 0 + self.column.weight_loader = MethodType( + ColumnParallelLinear.weight_loader, Mock(tp_size=2) + ) + self.row = torch.nn.Parameter(torch.zeros(16, 8)) + self.row.input_dim = 1 + self.row.weight_loader = MethodType( + RowParallelLinear.weight_loader, Mock(tp_size=2) + ) + self.norm = torch.nn.Parameter(torch.zeros(16)) + self.norm.weight_loader = default_weight_loader + self.fused = torch.nn.Parameter(torch.zeros(24, 16)) + + +class _MixedModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.proj = torch.nn.Module() + self.proj.weight = torch.nn.Parameter(torch.zeros(16, 8)) + self.proj.weight.input_dim = 1 + self.proj.weight.weight_loader = MethodType( + RowParallelLinear.weight_loader, Mock(tp_size=2) + ) + self.proj.bias = torch.nn.Parameter(torch.zeros(16)) + + +def _resolve(names, dtypes, shapes, **kwargs): + defaults = dict(num_workers=2, shard_axis_size=2, allow_direct=True) + defaults.update(kwargs) + return resolve_parameter_destinations(_Model(), names, dtypes, shapes, **defaults) + + +class TestDestinationResolution: + @pytest.mark.parametrize( + "name,expected_dim", + [("column", 0), ("row", 1)], + ) + def test_sharded_parameter_resolves_to_its_tp_dim(self, name, expected_dim): + [destination] = _resolve([name], [torch.float32], [(16, 16)]) + assert destination.direct + assert destination.placements == (REPLICATE, expected_dim) + + def test_replicated_parameter_needs_no_placement(self): + """A parameter every rank holds in full is REPLICATED, so it works + whatever way the inference mesh happens to be factored.""" + [destination] = _resolve(["norm"], [torch.float32], [(16,)]) + assert destination.direct + assert destination.placements is REPLICATED + + def test_direct_destination_is_the_live_parameter(self): + model = _Model() + [destination] = resolve_parameter_destinations( + model, + ["column"], + [torch.float32], + [(16, 16)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert destination.tensor.data_ptr() == model.column.data_ptr() + + def test_known_transforming_model_submodule_falls_back(self): + class GPT2Model(_Model): + pass + + GPT2Model.__module__ = "vllm.model_executor.models.gpt2" + model = torch.nn.Module() + model.transforming = GPT2Model() + [destination] = resolve_parameter_destinations( + model, + ["transforming.row"], + [torch.float32], + [(16, 16)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + + assert not destination.direct + assert destination.placements is REPLICATED + + def test_unknown_name_falls_back(self): + """Fused parameters reach the worker under checkpoint names that do not + exist in the model; those must take the full-tensor path.""" + [destination] = _resolve(["mlp.gate_proj.weight"], [torch.float32], [(16, 16)]) + assert not destination.direct + assert destination.placements is REPLICATED + + def test_fallback_parameter_demotes_direct_sibling_in_same_module(self): + """Layerwise finalization must not overwrite a directly loaded weight + when a sibling bias takes the fallback path.""" + destinations = resolve_parameter_destinations( + _MixedModel(), + ["proj.weight", "proj.bias"], + [torch.float32, torch.float32], + [(16, 16), (16,)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + + assert not any(destination.direct for destination in destinations) + + def test_unknown_loader_falls_back_despite_matching_name_and_shape(self): + model = _Model() + model.norm.weight_loader = lambda param, weight: param.data.copy_(weight) + [destination] = resolve_parameter_destinations( + model, + ["norm"], + [torch.float32], + [(16,)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_missing_loader_is_not_assumed_to_be_a_plain_copy(self): + model = _Model() + del model.norm.weight_loader + [destination] = resolve_parameter_destinations( + model, + ["norm"], + [torch.float32], + [(16,)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_wrapped_known_loader_falls_back(self): + model = _Model() + + @wraps(default_weight_loader) + def wrapped_loader(param, weight): + default_weight_loader(param, weight) + + model.norm.weight_loader = wrapped_loader + [destination] = resolve_parameter_destinations( + model, + ["norm"], + [torch.float32], + [(16,)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_replicated_loader_with_transform_metadata_falls_back(self): + model = _Model() + model.norm.is_transposed = True + [destination] = resolve_parameter_destinations( + model, + ["norm"], + [torch.float32], + [(16,)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_packed_dim_zero_falls_back(self): + model = _Model() + model.column.packed_dim = 0 + [destination] = resolve_parameter_destinations( + model, + ["column"], + [torch.float32], + [(16, 16)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_loader_declared_dim_wins_over_shape_guessing(self): + model = _Model() + model.column.output_dim = 1 + [destination] = resolve_parameter_destinations( + model, + ["column"], + [torch.float32], + [(16, 16)], + num_workers=2, + shard_axis_size=2, + allow_direct=True, + ) + assert not destination.direct + + def test_shape_the_tp_factor_cannot_explain_falls_back(self): + [destination] = _resolve(["fused"], [torch.float32], [(16, 16)]) + assert not destination.direct + + def test_dtype_mismatch_falls_back(self): + """A parameter stored in a different dtype than the wire dtype means a + quantized or otherwise transformed layout — never write into it.""" + [destination] = _resolve(["column"], [torch.bfloat16], [(16, 16)]) + assert not destination.direct + + def test_allow_direct_false_forces_every_parameter_to_fall_back(self): + destinations = _resolve( + ["column", "norm"], + [torch.float32] * 2, + [(16, 16), (16,)], + allow_direct=False, + ) + assert not any(d.direct for d in destinations) + + def test_plan_reports_byte_coverage(self, caplog_vllm): + """A small direct parameter must not hide a large fallback tensor.""" + with caplog_vllm.at_level( + logging.INFO, + logger="vllm.distributed.weight_transfer.m2n_layout", + ): + _resolve( + ["norm", "missing_fused_weight"], + [torch.float32, torch.bfloat16], + [(16,), (1024, 1024)], + ) + + assert ( + "1/2 parameters resharded directly into the model, 1 via " + "full-tensor fallback; direct byte coverage: 64/2097216 bytes " + "(0.0%)" in caplog_vllm.text + ) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 480c4f1b8a..924d2bafec 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -480,6 +480,18 @@ def test_compilation_config(): ) +def test_trust_request_mm_kwargs_cli(): + from vllm.entrypoints.launchers.cli_args import FrontendArgs + + parser = FrontendArgs.add_cli_args(FlexibleArgumentParser()) + + args = parser.parse_args([]) + assert not args.trust_request_mm_kwargs + + args = parser.parse_args(["--trust-request-mm-kwargs"]) + assert args.trust_request_mm_kwargs + + def test_attention_config(): from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 226908d673..aa68b6bff0 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -15,8 +15,9 @@ Also covers cache usage computation in ``_build_anthropic_usage``. import json from argparse import Namespace from http import HTTPStatus +from types import SimpleNamespace from typing import Annotated -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI @@ -1382,6 +1383,11 @@ Q35_TEMPLATE = ( "{%- endfor %}" ) +PERMISSIVE_TEMPLATE = ( + "{%- for message in messages %}{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" +) + class TestDetectMergeInlineSystem: """Verify _detect_merge_inline_system auto-detection. @@ -1401,11 +1407,7 @@ class TestDetectMergeInlineSystem: def test_no_restriction_no_merge(self): """Template without restriction accepts mid-conversation system.""" assert ( - AnthropicServingMessages._detect_merge_inline_system( - "{%- for message in messages %}" - "{{- message.role }}: {{ message.content }}\n" - "{%- endfor %}" - ) + AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE) is False ) @@ -1413,6 +1415,38 @@ class TestDetectMergeInlineSystem: """No chat_template → conservative default: merge.""" assert AnthropicServingMessages._detect_merge_inline_system(None) is True + @pytest.mark.parametrize( + ("resolved", "expected"), + [(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)], + ) + def test_probes_resolved_template_without_cli_override( + self, monkeypatch, resolved, expected + ): + """Without --chat-template, probe the tokenizer's template (#58727). + + Merging defeats prefix caching, so it is warned about at startup. + """ + import vllm.entrypoints.anthropic.serving as serving_mod + from vllm.renderers.hf import HfRenderer + + monkeypatch.setattr( + serving_mod, "resolve_chat_template", lambda *a, **kw: resolved + ) + mock_logger = MagicMock() + monkeypatch.setattr(serving_mod, "logger", mock_logger) + renderer = MagicMock(spec=HfRenderer) + renderer.tokenizer = MagicMock() + online_renderer = SimpleNamespace( + renderer=renderer, + chat_template=None, + model_config=None, + ) + assert ( + AnthropicServingMessages._should_merge_inline_system(online_renderer) + is expected + ) + assert mock_logger.warning.called is expected + # ====================================================================== # Full (non-streaming) response conversion: messages_full_converter @@ -1727,3 +1761,197 @@ class TestClientErrorResponses: assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json()["error"]["type"] == "BadRequestError" + + +# ====================================================================== +# thinking configuration pass-through +# ====================================================================== + + +class TestThinkingConfig: + def test_absent_thinking_leaves_reasoning_untouched(self): + """Requests without `thinking` must convert exactly as before.""" + request = _make_request([{"role": "user", "content": "Hello"}]) + + result = _convert(request) + assert result.reasoning_effort is None + assert result.thinking_token_budget is None + assert result.include_reasoning is True + + def test_disabled_clears_reasoning_effort(self): + """`disabled` maps to reasoning_effort="none", which is what clears + enable_thinking for templates that honor it.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "none" + + def test_disabled_overrides_output_config_effort(self): + """`thinking` is applied after output_config so an explicit opt-out wins + over an inherited effort ceiling.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "high"}, + thinking={"type": "disabled"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "none" + + def test_enabled_sets_thinking_token_budget(self): + request = AnthropicMessagesRequest( + model="test-model", + max_tokens=4096, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled", "budget_tokens": 2048}, + ) + + result = _convert(request) + assert result.thinking_token_budget == 2048 + assert result.reasoning_effort is None + + @pytest.mark.parametrize( + "thinking", + [ + pytest.param({"budget_tokens": 2048}, id="missing-type"), + pytest.param({"type": "enabled"}, id="enabled-missing-budget"), + pytest.param( + {"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024" + ), + pytest.param( + {"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max" + ), + pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"), + ], + ) + def test_rejects_invalid_thinking(self, thinking): + """Mirror the Anthropic API's BetaThinkingConfigParam constraints.""" + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model="test-model", + max_tokens=4096, + messages=[{"role": "user", "content": "Hello"}], + thinking=thinking, + ) + + def test_pd_prefill_leg_skips_budget_check(self): + """P/D sidecars resend the request to the prefill node with + max_tokens=1; the budget is enforced on the decode leg instead.""" + thinking = {"type": "enabled", "budget_tokens": 1024} + request = AnthropicMessagesRequest( + model="test-model", + max_tokens=1, + messages=[{"role": "user", "content": "Hello"}], + thinking=thinking, + kv_transfer_params={"do_remote_decode": True}, + ) + + assert _convert(request).thinking_token_budget == 1024 + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model="test-model", + max_tokens=1, + messages=[{"role": "user", "content": "Hello"}], + thinking=thinking, + kv_transfer_params={"do_remote_decode": False}, + ) + + def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self): + """`adaptive` lets the model choose depth, so only the ceiling from + output_config.effort should survive.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "low"}, + thinking={"type": "adaptive"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "low" + assert result.thinking_token_budget is None + + def test_disabled_uses_configured_effort(self): + """Models that always think (GLM-5.3) or reject "none" (Harmony) are + served with a low effort instead.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + result = _convert(request, disabled_thinking_effort="low") + assert result.reasoning_effort == "low" + + @pytest.mark.parametrize("display", ["omitted", "summarized", "updates"]) + def test_display_keeps_reasoning_included(self, display): + """Suppressing reasoning would mark it ended for structured outputs and + drop it from multi-turn history, so `display` is ignored.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive", "display": display}, + ) + + result = _convert(request) + assert result.include_reasoning is True + + def test_claude_code_payload(self): + """The combination Claude Code sends on every request: an effort ceiling + plus adaptive thinking with reasoning display omitted.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "high"}, + thinking={"type": "adaptive", "display": "omitted"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "high" + assert result.include_reasoning is True + assert result.thinking_token_budget is None + + +class TestProbeDisabledThinkingEffort: + """``auto`` falls back to ``low`` when ``none`` cannot turn thinking off.""" + + @staticmethod + async def _probe(render): + obj = MagicMock(spec=AnthropicServingMessages) + obj.online_renderer = MagicMock() + obj.online_renderer.render_chat = AsyncMock( + side_effect=lambda req: ([], [render(req.reasoning_effort)]) + ) + obj._extract_prompt_components = lambda engine_input: SimpleNamespace( + token_ids=engine_input, text=None + ) + obj._render_probe_prompt = ( + AnthropicServingMessages._render_probe_prompt.__get__(obj) + ) + return await AnthropicServingMessages._probe_disabled_thinking_effort(obj) + + @staticmethod + def _reject_none(effort): + """Harmony (gpt-oss) raises on ``none``.""" + if effort == "none": + raise ValueError(f"unsupported {effort=}") + return [1] + + @staticmethod + def _none_as_max(effort): + """GLM-5.3 treats efforts other than low/high as max.""" + return [1, {"low": 0, "high": 1}.get(effort, 2)] + + @pytest.mark.asyncio + async def test_template_honors_none(self): + assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none" + + @pytest.mark.asyncio + async def test_template_ignores_effort(self): + assert await self._probe(lambda effort: [1]) == "low" + + @pytest.mark.asyncio + async def test_template_renders_none_as_thinking_effort(self): + assert await self._probe(self._none_as_max) == "low" + + @pytest.mark.asyncio + async def test_renderer_rejects_none(self): + assert await self._probe(self._reject_none) == "low" diff --git a/tests/entrypoints/anthropic/test_protocol_exports.py b/tests/entrypoints/anthropic/test_protocol_exports.py index 466f40e3cc..1123175793 100644 --- a/tests/entrypoints/anthropic/test_protocol_exports.py +++ b/tests/entrypoints/anthropic/test_protocol_exports.py @@ -19,6 +19,7 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) @@ -35,6 +36,7 @@ SERVING_PROTOCOL_EXPORTS = ( AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) diff --git a/tests/entrypoints/launchers/test_cli_args.py b/tests/entrypoints/launchers/test_cli_args.py index 0bcca8cb41..de32e82ba4 100644 --- a/tests/entrypoints/launchers/test_cli_args.py +++ b/tests/entrypoints/launchers/test_cli_args.py @@ -8,6 +8,7 @@ import pytest import vllm.entrypoints.launchers.cli_args as cli_args_module from tests.utils import VLLM_PATH +from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.launchers.cli_args import ( make_arg_parser, validate_parsed_serve_args, @@ -69,6 +70,24 @@ def test_config_arg_parsing(serve_parser, cli_config_file): assert args.port == 9000 +def test_logging_config_cli_args(serve_parser): + with pytest.warns(UserWarning, match="--log-config-file is deprecated"): + args = serve_parser.parse_args( + [ + "--logging-config", + '{"log_level":"WARNING","pylogging_config_file":"/tmp/json.json"}', + "--log-level", + "DEBUG", + "--log-config-file", + "/tmp/flat.json", + ] + ) + + config = AsyncEngineArgs.from_cli_args(args).create_logging_config() + assert config.log_level == "DEBUG" + assert config.pylogging_config_file == "/tmp/flat.json" + + ### Tests for LoRA module parsing def test_valid_key_value_format(serve_parser): # Test old format: name=path diff --git a/tests/entrypoints/llm/test_struct_output_generate.py b/tests/entrypoints/llm/test_struct_output_generate.py index 030b0207e4..e6b02721f4 100644 --- a/tests/entrypoints/llm/test_struct_output_generate.py +++ b/tests/entrypoints/llm/test_struct_output_generate.py @@ -455,7 +455,7 @@ def test_structured_output( structured_outputs=StructuredOutputsParams(grammar="not a grammar"), ) with pytest.raises( - VLLMValidationError, match="Failed to convert the grammar " + VLLMValidationError, match="Invalid grammar specification" ): runner.llm.generate( ( diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py index 5031cbea1a..7b0f3c89c9 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py @@ -26,13 +26,11 @@ def server(): "--enforce-eager", "--limit-mm-per-prompt", json.dumps({"audio": 3, "video": 3}), + "--trust-request-mm-kwargs", *ROCM_EXTRA_ARGS, ] - with RemoteOpenAIServer( - MODEL_NAME, - args, - ) as remote_server: + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py index 682eb97386..a6ca410e07 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py @@ -37,6 +37,7 @@ def server(): json.dumps({"video": MAXIMUM_VIDEOS}), "--media-io-kwargs", json.dumps({"video": {"num_frames": 32}}), + "--trust-request-mm-kwargs", ] # ROCm: Increase timeouts to handle potential network delays and slower diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index a1665a71de..a4126ba5f7 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -45,6 +45,8 @@ class MockModelConfig: trust_remote_code = False tokenizer_mode = "auto" max_model_len = 100 + revision = None + code_revision = None tokenizer_revision = None multimodal_config = MultiModalConfig() hf_config = MockHFConfig() @@ -475,6 +477,81 @@ def test_json_schema_response_format_missing_schema(): ) +@pytest.mark.asyncio +async def test_online_renderer_rejects_mm_processor_kwargs_by_default(): + model_config = MockModelConfig() + model_config.multimodal_config = MultiModalConfig() + online_renderer = OnlineRenderer( + model_config=model_config, + renderer=MagicMock(), + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hello"}], + mm_processor_kwargs={ + "patch_size": 1, + "vision_min_num_patches": 3_000_000_000, + }, + ) + + with pytest.raises( + VLLMValidationError, + match="Per-request mm_processor_kwargs are disabled", + ): + await online_renderer.preprocess_chat( + request, + request.messages, + default_template=None, + default_template_content_format="auto", + default_template_kwargs=None, + ) + + +@pytest.mark.asyncio +async def test_online_renderer_rejects_media_io_kwargs_by_default(): + model_config = MockModelConfig() + model_config.multimodal_config = MultiModalConfig() + online_renderer = OnlineRenderer( + model_config=model_config, + renderer=MagicMock(), + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hello"}], + media_io_kwargs={"video": {"num_frames": 1_000_000}}, + ) + + with pytest.raises( + VLLMValidationError, + match="Per-request media_io_kwargs are disabled", + ): + await online_renderer.preprocess_chat( + request, + request.messages, + default_template=None, + default_template_content_format="auto", + default_template_kwargs=None, + ) + + +def test_trust_request_mm_kwargs_opt_in_allows_overrides(): + from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs + + validate_request_mm_kwargs( + mm_processor_kwargs={"use_audio_in_video": True}, + media_io_kwargs={"video": {"num_frames": 4}}, + trust_request_mm_kwargs=True, + ) + + @pytest.mark.parametrize("format_value", [None, {}]) def test_structural_tag_response_format_invalid(format_value): """Malformed structural tags should be rejected during request validation.""" diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 9f92e1ce35..6022f5ce77 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -44,6 +44,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import QueueOverflowError, VLLMValidationError from vllm.inputs import TokensPrompt +from vllm.logprobs import Logprob from vllm.multimodal.inputs import PlaceholderRange from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser import HarmonyParser @@ -797,38 +798,401 @@ async def test_chat_streaming_metrics_ride_on_usage_chunk(): @pytest.mark.asyncio -async def test_streaming_reasoning_usage_counts_across_deltas(): - serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) - serving._include_reasoning_tokens_details = True +@pytest.mark.parametrize("n", [1, 2]) +@pytest.mark.parametrize("include_reasoning", [True, False]) +@pytest.mark.parametrize("include_details", [True, False]) +@pytest.mark.parametrize("separate_finish", [True, False]) +@pytest.mark.parametrize( + "stream_options, force_usage, include_usage, continuous_usage", + [ + (None, False, False, False), + ({"include_usage": True}, False, True, False), + ({"include_usage": True, "continuous_usage_stats": True}, False, True, True), + ({"include_usage": False, "continuous_usage_stats": True}, False, False, False), + (None, True, True, True), + ], +) +async def test_streaming_reasoning_usage_counts_across_deltas( + n, + include_reasoning, + include_details, + separate_finish, + stream_options, + force_usage, + include_usage, + continuous_usage, +): + """Final usage needs one recount per choice, not per streamed delta.""" + serving = _build_minimal_metrics_serving_chat( + enable_per_request_metrics=False, enable_force_include_usage=force_usage + ) + serving._include_reasoning_tokens_details = include_details serving.model_config = None - parser = MagicMock() - parser.parse_delta.side_effect = [ - DeltaMessage(reasoning="reasoning"), - DeltaMessage(content="answer"), - ] - parser.count_reasoning_tokens.side_effect = lambda token_ids: sum( - token_id == 20 for token_id in token_ids - ) - serving.parser_cls = MagicMock(return_value=parser) - - first = _make_metrics_request_output(metrics=None, token_ids=(10, 20)) - first.outputs[0].text = "reasoning" - first.outputs[0].finish_reason = None - second = _make_metrics_request_output(metrics=None, token_ids=(11, 30)) - second.outputs[0].text = "answer" + parsers = [MagicMock() for _ in range(n)] + request_outputs = [] + num_steps = 3 if separate_finish else 2 + for parser in parsers: + parser.parse_delta.side_effect = [ + DeltaMessage(reasoning="reasoning") if include_reasoning else None, + DeltaMessage(content="answer"), + DeltaMessage(), + ] + parser.count_reasoning_tokens.side_effect = lambda token_ids: sum( + token_id == 20 for token_id in token_ids + ) + serving.parser_cls = MagicMock(side_effect=parsers) + for step in range(num_steps): + for i in reversed(range(n)): + tokens = ((10,) + (20,) * (i + 1), (11, 20, 30), ())[step] + result = _make_metrics_request_output(metrics=None, token_ids=tokens) + result.outputs[0].index = i + result.outputs[0].text = ("reasoning", "answer", "")[step] + finished = step == num_steps - 1 + result.outputs[0].finish_reason = "stop" if finished else None + result.finished = finished and i == 0 + request_outputs.append(result) request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "Test prompt"}], max_tokens=10, stream=True, - stream_options={"include_usage": True}, + n=n, + include_reasoning=include_reasoning, + return_token_ids=True, + stream_options=stream_options, + ) + metadata = RequestResponseMetadata(request_id="chatcmpl-test-id") + chunks: list[dict[str, Any]] = [] + async for line in serving.chat_completion_stream_generator( + request, + _stream_request_outputs(*request_outputs), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=metadata, + ): + payload = line.removeprefix("data: ").strip() + if payload != "[DONE]": + chunks.append(json.loads(payload)) + + assert all("error" not in chunk for chunk in chunks) + expected_count = sum(i + 2 for i in range(n)) + expected_details = {"reasoning_tokens": expected_count} if include_details else None + assert metadata.final_usage_info is not None + assert metadata.final_usage_info.model_dump()["completion_tokens_details"] == ( + expected_details + ) + final_chunks = [chunk for chunk in chunks if not chunk["choices"]] + assert len(final_chunks) == int(include_usage) + if include_usage: + assert final_chunks[0]["usage"].get("completion_tokens_details") == ( + expected_details + ) + for i, parser in enumerate(parsers): + expected_calls = ( + (num_steps if continuous_usage else 1) if include_details else 0 + ) + assert parser.count_reasoning_tokens.call_count == expected_calls + if include_details: + assert parser.count_reasoning_tokens.call_args.args == ( + (10,) + (20,) * (i + 1) + (11, 20, 30), + ) + choices = [ + chunk + for chunk in chunks + if chunk["choices"] and chunk["choices"][0]["index"] == i + ] + assert ( + "".join(c["choices"][0]["delta"].get("content") or "" for c in choices) + == "answer" + ) + if not include_reasoning: + assert all(c["choices"][0].get("token_ids") is None for c in choices) + if continuous_usage and include_details: + counts = [ + c["usage"]["completion_tokens_details"]["reasoning_tokens"] + for c in choices + ] + expected_counts = [0, i + 1] if include_reasoning else [0] + assert counts == expected_counts + [i + 2] * (num_steps - 1) + + +def _reasoning_lifecycle_output( + index: int = 0, + tokens: tuple[int, ...] = (20, 30), + text: str = "answer", + finish: str | None = None, +) -> RequestOutput: + result = _make_metrics_request_output(None, tokens) + result.outputs[0].index = index + result.outputs[0].text = text + result.outputs[0].finish_reason = finish + result.finished = finish is not None + return result + + +_EMPTY_OUTPUTS = _reasoning_lifecycle_output(tokens=(), text="") +_EMPTY_OUTPUTS.outputs = [] + +_REASONING_LIFECYCLE_CASES = [ + ("empty-generator", 1, [], 0, 0), + ("empty-outputs", 1, [_EMPTY_OUTPUTS], 0, 0), + ( + "empty-finish", + 1, + [_reasoning_lifecycle_output(tokens=(), text="", finish="stop")], + 0, + 0, + ), + ( + "empty-prefill", + 1, + [ + _reasoning_lifecycle_output(tokens=(), text=""), + _reasoning_lifecycle_output(finish="stop"), + ], + 2, + 1, + ), + ( + "one-empty-choice", + 2, + [_reasoning_lifecycle_output(index=1, finish="stop")], + 2, + 1, + ), + ( + "uneven-finish", + 2, + [ + _reasoning_lifecycle_output(index=1, finish="stop"), + _reasoning_lifecycle_output(), + _reasoning_lifecycle_output(tokens=(20,), finish="stop"), + ], + 5, + 3, + ), + ( + "empty-final-delta", + 1, + [ + _reasoning_lifecycle_output(), + _reasoning_lifecycle_output(tokens=(), text="", finish="stop"), + ], + 2, + 1, + ), + ( + "ignore-finished-choice", + 2, + [ + _reasoning_lifecycle_output(index=1, finish="stop"), + _reasoning_lifecycle_output(index=1, tokens=(20, 20, 20)), + _reasoning_lifecycle_output(finish="stop"), + ], + 4, + 2, + ), + ("exception-before", 1, [RuntimeError("fixture engine error")], None, None), + ( + "exception-after", + 1, + [_reasoning_lifecycle_output(), RuntimeError("fixture engine error")], + None, + None, + ), + ( + "generation-error", + 1, + [ + _reasoning_lifecycle_output(), + _reasoning_lifecycle_output(finish="error"), + ], + None, + None, + ), + ( + "cancel-after", + 1, + [_reasoning_lifecycle_output(), asyncio.CancelledError()], + None, + None, + ), + ( + "close-after", + 1, + [ + _reasoning_lifecycle_output(), + _reasoning_lifecycle_output(finish="stop"), + ], + None, + None, + ), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("usage_mode", ["none", "final", "continuous"]) +@pytest.mark.parametrize( + "name, n, outputs, expected_tokens, expected_reasoning", + _REASONING_LIFECYCLE_CASES, + ids=[case[0] for case in _REASONING_LIFECYCLE_CASES], +) +async def test_streaming_reasoning_usage_lifecycle_edges( + usage_mode, + name, + n, + outputs, + expected_tokens, + expected_reasoning, +): + """Cancel, errors, and empty choices still match baseline stream usage.""" + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) + serving.model_config = None + serving._include_reasoning_tokens_details = True + parsers = [MagicMock() for _ in range(n)] + for parser in parsers: + parser.parse_delta.side_effect = lambda **kwargs: DeltaMessage( + content=kwargs["delta_text"] + ) + parser.count_reasoning_tokens.side_effect = lambda ids: ids.count(20) + serving.parser_cls = MagicMock(side_effect=parsers) + stream_options = ( + None + if usage_mode == "none" + else { + "include_usage": True, + "continuous_usage_stats": usage_mode == "continuous", + } + ) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + stream=True, + n=n, + return_token_ids=True, + stream_options=stream_options, + ) + metadata = RequestResponseMetadata(request_id="chatcmpl-lifecycle") + + async def source(): + for result in outputs: + if isinstance(result, BaseException): + raise result + yield result + + iterator = source() + stream = serving.chat_completion_stream_generator( + request, + iterator, + "chatcmpl-lifecycle", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=metadata, + ) + chunks: list[object] = [] + outcome = "exhausted" + try: + async for line in stream: + raw = line.removeprefix("data: ").strip() + if raw == "[DONE]": + chunks.append(raw) + continue + item = json.loads(raw) + chunks.append(item) + if name == "close-after" and any( + choice["delta"].get("content") for choice in item.get("choices", []) + ): + await stream.aclose() + outcome = "closed" + break + except asyncio.CancelledError: + outcome = "cancelled" + finally: + await stream.aclose() + await iterator.aclose() + + error_chunks = [c for c in chunks if isinstance(c, dict) and "error" in c] + call_count = sum(parser.count_reasoning_tokens.call_count for parser in parsers) + if expected_tokens is not None: + assert outcome == "exhausted" and chunks[-1] == "[DONE]" and not error_chunks + assert metadata.final_usage_info is not None + assert metadata.final_usage_info.completion_tokens == expected_tokens + assert ( + metadata.final_usage_info.completion_tokens_details.reasoning_tokens + == expected_reasoning + ) + nonempty_choices = sum( + bool(parser.count_reasoning_tokens.call_args) for parser in parsers + ) + if usage_mode == "continuous": + assert call_count >= nonempty_choices + else: + assert call_count == nonempty_choices + return + + assert metadata.final_usage_info is None + if name in {"cancel-after", "close-after"}: + assert outcome == ("cancelled" if name == "cancel-after" else "closed") + assert "[DONE]" not in chunks and not error_chunks + else: + assert outcome == "exhausted" and chunks[-1] == "[DONE]" + assert len(error_chunks) == 1 + if usage_mode != "continuous" or name == "exception-before": + assert call_count == 0 + else: + assert call_count >= 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("include_reasoning", "expected_tokens"), + [(True, ["", "answer"]), (False, [])], +) +async def test_streaming_logprobs_kept_when_parser_suppresses_delta( + include_reasoning: bool, expected_tokens: list[str] +): + """Logprobs of tokens the parser swallows (e.g. ) must still + be streamed, unless per-token metadata is hidden with the reasoning.""" + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) + serving.model_config = None + serving.return_tokens_as_token_ids = False + + parser = MagicMock() + parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")] + parser.count_reasoning_tokens.return_value = 0 + serving.parser_cls = MagicMock(return_value=parser) + + def make_output(token_id: int, text: str, finish_reason: str | None): + request_output = _make_metrics_request_output( + metrics=None, token_ids=(token_id,) + ) + completion = request_output.outputs[0] + completion.text = text + completion.finish_reason = finish_reason + completion.logprobs = [ + {token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)} + ] + return request_output + + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + stream=True, + logprobs=True, + include_reasoning=include_reasoning, ) chunks: list[dict[str, Any]] = [] async for line in serving.chat_completion_stream_generator( request, - _stream_request_outputs(first, second), + _stream_request_outputs( + make_output(10, "", None), make_output(11, "answer", "stop") + ), "chatcmpl-test-id", "test-model", conversation=[{"role": "user", "content": "Test"}], @@ -839,10 +1203,14 @@ async def test_streaming_reasoning_usage_counts_across_deltas(): if payload != "[DONE]": chunks.append(json.loads(payload)) - usage_chunks = [chunk for chunk in chunks if chunk.get("usage")] - assert usage_chunks[-1]["usage"]["completion_tokens_details"] == { - "reasoning_tokens": 1 - } + streamed_tokens = [ + entry["token"] + for chunk in chunks + for choice in chunk["choices"] + if choice.get("logprobs") + for entry in choice["logprobs"]["content"] + ] + assert streamed_tokens == expected_tokens @dataclass @@ -2239,7 +2607,13 @@ async def test_tool_choice_validation_without_parser(): @pytest.mark.asyncio -async def test_streaming_n_gt1_independent_tool_parsers(): +@pytest.mark.parametrize( + ("engine_finish_reason", "expected_finish_reason"), + [("stop", "tool_calls"), ("length", "length")], +) +async def test_streaming_n_gt1_independent_tool_parsers( + engine_finish_reason: str, expected_finish_reason: str +): """n>1 streaming must use independent parser instances and token-id histories per choice. """ @@ -2344,7 +2718,7 @@ async def test_streaming_n_gt1_independent_tool_parsers(): token_ids=[], cumulative_logprob=0.0, logprobs=None, - finish_reason="stop", + finish_reason=engine_finish_reason, ) for choice_idx in range(num_choices) ], @@ -2408,8 +2782,8 @@ async def test_streaming_n_gt1_independent_tool_parsers(): assert len(reasons) == 1, ( f"Choice {choice_idx}: expected exactly 1 finish_reason, got {reasons}" ) - assert reasons[0] == "tool_calls", ( - f"Choice {choice_idx}: expected finish_reason='tool_calls', " + assert reasons[0] == expected_finish_reason, ( + f"Choice {choice_idx}: expected finish_reason={expected_finish_reason!r}, " f"got '{reasons[0]}'" ) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 28a499e129..05811555a9 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.render.serving import ServingRender +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import GenerationError, VLLMValidationError from vllm.outputs import CompletionOutput, RequestOutput +from vllm.renderers.deepseek_v4 import DeepseekV4Renderer from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats MODEL_NAME = "openai-community/gpt2" MODEL_NAME_SHORT = "gpt2" +DEEPSEEK_V4_FIM_BEGIN = "<|fim▁begin|>" +DEEPSEEK_V4_FIM_HOLE = "<|fim▁hole|>" +DEEPSEEK_V4_FIM_END = "<|fim▁end|>" _PER_REQUEST_STATS = RequestStateStats( queued_ts=1.0, scheduled_ts=1.5, @@ -51,6 +56,8 @@ class MockModelConfig: trust_remote_code = False tokenizer_mode = "auto" max_model_len = 100 + revision = None + code_revision = None tokenizer_revision = None multimodal_config = MultiModalConfig() hf_config = MockHFConfig() @@ -302,6 +309,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode(): assert response.metrics.speculative_decoding.num_spec_steps == 2 +def _build_online_renderer_for_completion( + model_config: MockModelConfig, + renderer: Any | None = None, +) -> OnlineRenderer: + if renderer is None: + renderer = MagicMock() + renderer.render_completion_suffix.return_value = None + + return OnlineRenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + +def _deepseek_v4_suffix_renderer( + model_config: MockModelConfig, +) -> DeepseekV4Renderer: + return DeepseekV4Renderer( + MockVllmConfig(model_config, parallel_config=MockParallelConfig()), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_uses_renderer_capability(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt="def fib(n):\n return ", + suffix="\n\nprint(fib(10))", + max_tokens=64, + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + online_renderer.preprocess_completion.assert_awaited_once() + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == ( + f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return " + f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))" + f"{DEEPSEEK_V4_FIM_END}" + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_supports_text_prompt_list(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt=["prefix A", "prefix B"], + suffix=" suffix", + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [ + f"{DEEPSEEK_V4_FIM_BEGIN}prefix A" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + f"{DEEPSEEK_V4_FIM_BEGIN}prefix B" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + ] + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_renderer_without_fim_support(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion(model_config) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "FIM completion rendering" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_echo(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + echo=True, + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "Echo is unsupported with suffix" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_prompt_embeds(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + prompt_embeds=b"embeds", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "suffix is not supported with prompt_embeds" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]])) +async def test_completion_suffix_rejects_token_prompt(prompt): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt=prompt, + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "requires text prompt input" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + @pytest.mark.asyncio async def test_completion_error_non_stream(): """Test finish_reason='error' returns 500 InternalServerError (non-streaming).""" diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 1a315902e2..429fcf1d0d 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -748,6 +748,7 @@ class TestInitializeToolSessions: class ToolCallingHarmonyContext(HarmonyContext): def __init__(self): self._messages = [] + self.request = None self._needs_tool_call = True def append_output(self, output) -> None: @@ -804,6 +805,53 @@ class TestInitializeToolSessions: assert followup_engine_input.get("cache_salt") == "request-salt" assert followup_engine_input["prompt_token_ids"] == [3, 4] + @pytest.mark.asyncio + async def test_harmony_tool_followup_respects_max_output_tokens( + self, serving_responses_instance + ): + class ToolCallingHarmonyContext(HarmonyContext): + def __init__(self, request): + self._messages = [] + self.request = request + self._needs_tool_call = True + + def append_output(self, output) -> None: + pass + + def need_builtin_tool_call(self) -> bool: + return self._needs_tool_call + + async def call_tool(self): + self._needs_tool_call = False + return [] + + def append_tool_output(self, output) -> None: + pass + + max_tokens_per_turn = [] + + async def generate_output(engine_input, sampling_params, *args, **kwargs): + max_tokens_per_turn.append(sampling_params.max_tokens) + yield MagicMock() + + serving_responses_instance.engine_client.generate.side_effect = generate_output + serving_responses_instance.online_renderer.render_responses_harmony_messages = ( + MagicMock(return_value=tokens_input([1, 2, 3])) + ) + context = ToolCallingHarmonyContext( + ResponsesRequest(input="test", max_output_tokens=10) + ) + + async for _ in serving_responses_instance._generate_with_builtin_tools( + request_id="req", + engine_input=tokens_input([1]), + sampling_params=SamplingParams(max_tokens=10), + context=context, + ): + pass + + assert max_tokens_per_turn == [10, 10] + @pytest.mark.asyncio async def test_initialize_tool_sessions( self, serving_responses_instance, mock_context, mock_exit_stack diff --git a/tests/entrypoints/openai/test_penalty_default_resolution.py b/tests/entrypoints/openai/test_penalty_default_resolution.py new file mode 100644 index 0000000000..2c39f5bef6 --- /dev/null +++ b/tests/entrypoints/openai/test_penalty_default_resolution.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Unit tests for presence_penalty / frequency_penalty resolution from +default_sampling_params in ChatCompletionRequest and CompletionRequest. + +Regression test for https://github.com/vllm-project/vllm/issues/50767: +these two penalties defaulted to 0.0 (not None) and to_sampling_params() +forwarded them straight through, so server-side defaults coming from +--override-generation-config / generation_config.json were silently +discarded on the /v1/chat/completions and /v1/completions endpoints. +""" + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, +) + +_DEFAULTS = {"presence_penalty": 1.5, "frequency_penalty": 0.5} + + +def _chat(**kwargs): + return ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + **kwargs, + ) + + +class TestChatCompletionPenaltyDefaults: + def test_defaults_applied_when_client_omits(self): + """Server-default penalties are applied when the client sends none.""" + sp = _chat().to_sampling_params(100, _DEFAULTS) + assert sp.presence_penalty == 1.5 + assert sp.frequency_penalty == 0.5 + + def test_client_value_overrides_default(self): + """An explicit client penalty wins over the server default.""" + sp = _chat(presence_penalty=0.2).to_sampling_params(100, _DEFAULTS) + assert sp.presence_penalty == 0.2 + assert sp.frequency_penalty == 0.5 + + def test_falls_back_to_zero_without_default(self): + """Without a server default the neutral 0.0 is used (no regression).""" + sp = _chat().to_sampling_params(100, {}) + assert sp.presence_penalty == 0.0 + assert sp.frequency_penalty == 0.0 + + +class TestCompletionPenaltyDefaults: + def test_defaults_applied_when_client_omits(self): + sp = CompletionRequest(model="test-model", prompt="hi").to_sampling_params( + 16, _DEFAULTS + ) + assert sp.presence_penalty == 1.5 + assert sp.frequency_penalty == 0.5 + + def test_client_value_overrides_default(self): + sp = CompletionRequest( + model="test-model", prompt="hi", frequency_penalty=0.9 + ).to_sampling_params(16, _DEFAULTS) + assert sp.presence_penalty == 1.5 + assert sp.frequency_penalty == 0.9 diff --git a/tests/entrypoints/openai/test_render_parity.py b/tests/entrypoints/openai/test_render_parity.py index 4fc44c5a71..2bf4345091 100644 --- a/tests/entrypoints/openai/test_render_parity.py +++ b/tests/entrypoints/openai/test_render_parity.py @@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer: enable_auto_tools=True, tool_parser="openai", exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, + trust_request_mm_kwargs=True, ) diff --git a/tests/entrypoints/pooling/embed/test_online_vision.py b/tests/entrypoints/pooling/embed/test_online_vision.py index 2b4bf57a13..8ca7961d42 100644 --- a/tests/entrypoints/pooling/embed/test_online_vision.py +++ b/tests/entrypoints/pooling/embed/test_online_vision.py @@ -46,6 +46,7 @@ def server(): json.dumps({"image": MAXIMUM_IMAGES}), "--chat-template", str(vlm2vec_jinja_path), + "--trust-request-mm-kwargs", ] with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: diff --git a/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py b/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py index dac7b6db6c..16392fd338 100644 --- a/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py +++ b/tests/entrypoints/pooling/scoring/test_bi_encoder_online.py @@ -126,35 +126,6 @@ async def test_score_api_queries_str_n_documents_str_n( assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) -@pytest.mark.asyncio -async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer): - text_pairs = [ - [TEXTS_1[0], TEXTS_2[0]], - [TEXTS_1[1], TEXTS_2[1]], - ] - - score_response = requests.post( - server.url_for("score"), - json={ - "model": MODEL_NAME, - "queries": TEXTS_1, - "documents": TEXTS_2, - }, - ) - score_response.raise_for_status() - score = ScoreResponse.model_validate(score_response.json()) - - assert score.id is not None - assert score.data is not None - assert len(score.data) == 2 - - vllm_outputs = [d.score for d in score.data] - hf_outputs = hf_model.predict(text_pairs).tolist() - - for i in range(len(vllm_outputs)): - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) - - @pytest.mark.asyncio async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer): text_pairs = [ diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py index 401bf4ff22..9228f96fe7 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_online.py @@ -143,35 +143,6 @@ async def test_score_api_queries_str_n_documents_str_n( assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) -@pytest.mark.asyncio -async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer): - text_pairs = [ - [TEXTS_1[0], TEXTS_2[0]], - [TEXTS_1[1], TEXTS_2[1]], - ] - - score_response = requests.post( - server.url_for("score"), - json={ - "model": MODEL_NAME, - "queries": TEXTS_1, - "documents": TEXTS_2, - }, - ) - score_response.raise_for_status() - score = ScoreResponse.model_validate(score_response.json()) - - assert score.id is not None - assert score.data is not None - assert len(score.data) == 2 - - vllm_outputs = [d.score for d in score.data] - hf_outputs = hf_model.predict(text_pairs).tolist() - - for i in range(len(vllm_outputs)): - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01) - - @pytest.mark.asyncio async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer): text_pairs = [ diff --git a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py index 8c2cf40f20..c83eaa544a 100644 --- a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py @@ -3,11 +3,15 @@ """Unit tests for ScoringIOProcessor post-tokenization helpers.""" from dataclasses import dataclass +from types import SimpleNamespace import pytest +from tokenizers import Tokenizer, models, pre_tokenizers, processors +from transformers import TokenizersBackend from vllm import TokensPrompt from vllm.entrypoints.pooling.scoring.io_processor import ( + CrossEncoderIOProcessor, _apply_post_tokenization_to_token_type_ids, ) from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids @@ -24,6 +28,53 @@ class _DummyTokenizer: pad_token_id: int = 99999 +@pytest.fixture +def llm_reranker_processor() -> CrossEncoderIOProcessor: + vocab = { + "[UNK]": 0, + "[CLS]": 1, + "[SEP]": 2, + "a": 3, + "bc": 4, + "##b": 5, + "##c": 6, + } + backend = Tokenizer(models.WordPiece(vocab, unk_token="[UNK]")) + backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + backend.post_processor = processors.TemplateProcessing( + single="[CLS] $A [SEP]", + special_tokens=[("[CLS]", 1), ("[SEP]", 2)], + ) + + processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor) + processor.model_config = SimpleNamespace(enable_prompt_embeds=False) + processor.tokenizer = TokenizersBackend( + tokenizer_object=backend, + unk_token="[UNK]", + cls_token="[CLS]", + sep_token="[SEP]", + ) + processor.supports_score_template = False + processor.use_sep_token = False + processor.model = None + return processor + + +def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit( + llm_reranker_processor: CrossEncoderIOProcessor, +): + encode_kwargs = {"add_special_tokens": True} + + _, uncapped = llm_reranker_processor.get_score_prompt("a", "bc", encode_kwargs) + _, nonbinding = llm_reranker_processor.get_score_prompt( + "a", "bc", encode_kwargs, max_tokens_per_doc=10 + ) + + expected = [1, 3, 5, 6, 2] + assert uncapped["prompt_token_ids"] == expected + assert nonbinding["prompt_token_ids"] == expected + + def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): """The cross-encoder segment boundary must survive truncate + pad. diff --git a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py index cb64524699..ab5a21c085 100644 --- a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest from tokenizers import Tokenizer, models, pre_tokenizers -from transformers import PreTrainedTokenizerFast +from transformers import TokenizersBackend from vllm import PoolingParams from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor @@ -70,7 +70,7 @@ def offline_processor_and_context(): backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]")) backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit() proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor) - proc.tokenizer = PreTrainedTokenizerFast(tokenizer_object=backend) + proc.tokenizer = TokenizersBackend(tokenizer_object=backend) proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False) proc.renderer = SimpleNamespace( default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024) diff --git a/tests/entrypoints/pooling/scoring/test_late_interaction_online.py b/tests/entrypoints/pooling/scoring/test_late_interaction_online.py index 7e4501fe85..51ef85c0d7 100644 --- a/tests/entrypoints/pooling/scoring/test_late_interaction_online.py +++ b/tests/entrypoints/pooling/scoring/test_late_interaction_online.py @@ -26,6 +26,36 @@ TEXTS_2 = [ ] +@pytest.mark.asyncio +async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default(): + from unittest.mock import MagicMock + + from vllm.config.multimodal import MultiModalConfig + from vllm.entrypoints.pooling.base.serving import PoolingBaseServing + from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest + from vllm.entrypoints.pooling.scoring.serving import ServingScores + from vllm.exceptions import VLLMValidationError + + serving = MagicMock() + serving.model_config = MagicMock(multimodal_config=MultiModalConfig()) + serving.trust_request_mm_kwargs = False + serving.io_processor = MagicMock() + serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving) + request = ScoreQueriesDocumentsRequest( + model=MODEL_NAME, + queries=TEXTS_1[0], + documents=TEXTS_2[0], + mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000}, + ) + + with pytest.raises( + VLLMValidationError, + match="Per-request mm_processor_kwargs are disabled", + ): + await ServingScores.flash_late_interaction(serving, request) + serving._preprocessing.assert_not_called() + + @pytest.fixture(scope="module", params=[True, False]) def server(request): args = [ diff --git a/tests/entrypoints/pooling/scoring/test_utils.py b/tests/entrypoints/pooling/scoring/test_utils.py index 35b205eebf..ddd3c082be 100644 --- a/tests/entrypoints/pooling/scoring/test_utils.py +++ b/tests/entrypoints/pooling/scoring/test_utils.py @@ -2,9 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import TokenizersBackend from vllm.entrypoints.pooling.scoring.typing import ScoreInput -from vllm.entrypoints.pooling.scoring.utils import validate_score_input +from vllm.entrypoints.pooling.scoring.utils import ( + truncate_text_to_tokens, + validate_score_input, +) from vllm.exceptions import VLLMValidationError @@ -63,3 +68,17 @@ def test_validate_score_input_rejects_invalid_inputs( assert str(exc_info.value) == message assert exc_info.value.parameter is None assert exc_info.value.value is None + + +def test_truncate_text_to_tokens_handles_shared_character_offsets(): + vocab = { + token: i for i, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet())) + } + backend = Tokenizer(models.BPE(vocab=vocab, merges=[])) + backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + tokenizer = TokenizersBackend(tokenizer_object=backend) + + truncated = truncate_text_to_tokens("aéx", tokenizer, max_tokens=2) + + assert truncated == "a" + assert len(tokenizer(truncated, add_special_tokens=False)["input_ids"]) == 1 diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 4e325db32c..8510d63c4b 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -18,6 +18,8 @@ Tests are split into two layers: """ import json +import threading +from concurrent.futures import ThreadPoolExecutor import pytest import pytest_asyncio @@ -214,7 +216,7 @@ def tokenizer(): @pytest.fixture(scope="module") -def derenderer(tokenizer): +def derenderer(tokenizer, request): """Construct a minimal OnlineDerenderer backed by a stub renderer.""" from unittest.mock import MagicMock @@ -223,6 +225,10 @@ def derenderer(tokenizer): renderer = MagicMock() renderer.get_tokenizer.return_value = tokenizer + executor = ThreadPoolExecutor(max_workers=1) + request.addfinalizer(executor.shutdown) + renderer._executor = executor + model_config = MagicMock() model_config.hf_config.model_type = "llama" model_config.model = MODEL_NAME @@ -244,13 +250,11 @@ def derenderer(tokenizer): def parsed_derenderer(tokenizer): """OnlineDerenderer with `_FakeParser` wired in as `self.parser`. - The parser configured path runs `_derender_chat_stream_parsed` via - `make_async(..., executor=renderer._executor)`, so unlike - `derenderer` above (whose streaming paths never touch the executor), - `renderer._executor` must be a real `ThreadPoolExecutor` here. + The parser-configured path runs `_derender_chat_stream_parsed` via + `make_async(..., executor=renderer._executor)`, so + `renderer._executor` must be a real `ThreadPoolExecutor`. `loop.run_in_executor` cannot submit work to a `MagicMock`. """ - from concurrent.futures import ThreadPoolExecutor from unittest.mock import MagicMock from vllm.renderers.online_derenderer import OnlineDerenderer @@ -390,6 +394,46 @@ class TestDetokenizeDelta: assert results[0] == self._one_shot(tokenizer, token_ids) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stream_method", + ["derender_chat_stream", "derender_completion_stream"], +) +async def test_streaming_detokenization_runs_off_event_loop( + derenderer, monkeypatch, stream_method +): + """Streaming detokenization runs on the renderer executor.""" + import vllm.renderers.online_derenderer as online_derenderer_module + + event_loop_thread_id = threading.get_ident() + executor_thread_id = derenderer.renderer._executor.submit( + threading.get_ident + ).result() + + detokenize_thread_id = None + original_detokenize = online_derenderer_module.detokenize_incrementally + + def record_detokenize_thread(*args, **kwargs): + nonlocal detokenize_thread_id + detokenize_thread_id = threading.get_ident() + return original_detokenize(*args, **kwargs) + + monkeypatch.setattr( + online_derenderer_module, + "detokenize_incrementally", + record_detokenize_thread, + ) + + await getattr(derenderer, stream_method)( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([10]), + ) + + assert detokenize_thread_id is not None + assert detokenize_thread_id == executor_thread_id + assert detokenize_thread_id != event_loop_thread_id + + class TestDerenderCompletionStream: """derender_completion_stream: streaming output parity with one shot.""" @@ -875,7 +919,6 @@ def harmony_derenderer(harmony_tokenizer): with a real tokenizer and parser, so the replay path is exercised against Harmony's actual channel grammar rather than a stub. """ - from concurrent.futures import ThreadPoolExecutor from unittest.mock import MagicMock from vllm.parser.harmony import HarmonyParser diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index e65a26a7d1..0daf4d405e 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client): async def test_messages_render_merges_inline_system(client): """Inline system messages merge into the leading system block. - Without a --chat-template arg the /v1/messages server path detects - merge_inline_system=True, so render must produce the same tokens as - the manually pre-merged request. + The model's chat template rejects non-leading system messages, so the + /v1/messages server path detects merge_inline_system=True and render must + produce the same tokens as the manually pre-merged request. """ inline = await client.post( "/v1/messages/render", diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py b/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py index 80f08078da..f42a406f41 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py @@ -26,3 +26,13 @@ def test_logprobs_zero_emits_sampled_token(): num_output_top_logprobs=0, ) assert len(result.content[0].top_logprobs) == 1 + + +def test_logprobs_minus_one_emits_all_tokens(): + result = ServingTokens._create_tokens_logprobs( + None, + token_ids=[7], + top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}], + num_output_top_logprobs=-1, + ) + assert len(result.content[0].top_logprobs) == 2 diff --git a/tests/entrypoints/serve/dev/rlhf/conftest.py b/tests/entrypoints/serve/dev/rlhf/conftest.py index c636b51572..dccc55803e 100644 --- a/tests/entrypoints/serve/dev/rlhf/conftest.py +++ b/tests/entrypoints/serve/dev/rlhf/conftest.py @@ -15,7 +15,6 @@ import subprocess import sys import threading import time -from collections.abc import Callable from contextlib import contextmanager from dataclasses import dataclass, field from typing import Any @@ -138,36 +137,6 @@ def server( proc.kill() -# --------------------------------------------------------------------------- -# Polling helper (200-lie workaround) -# --------------------------------------------------------------------------- - - -def poll_until( - predicate: Callable[[], bool], - timeout: float = 10.0, - interval: float = 0.5, -) -> bool: - """Poll predicate() until it returns True or timeout expires. - - Workaround for the vLLM sleep/wake "200-lie" — the HTTP endpoints may - return 200 before the underlying operation is complete, so callers that - need to verify state *after* an operation can use this helper instead of - assuming the 200 means completion. - - Returns True if predicate became true within timeout, False otherwise. - """ - deadline = time.time() + timeout - while time.time() < deadline: - try: - if predicate(): - return True - except Exception: - pass - time.sleep(interval) - return False - - # --------------------------------------------------------------------------- # HTTP helpers — generation # --------------------------------------------------------------------------- @@ -191,27 +160,6 @@ def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30): return None -def gen_with_logprobs( - url, prompt="The capital of France is", max_tokens=8, logprobs=5, timeout=30 -): - """Fire a /v1/completions request with logprobs; return JSON or None.""" - try: - r = requests.post( - f"{url}/v1/completions", - json={ - "model": "m", - "prompt": prompt, - "max_tokens": max_tokens, - "temperature": 0, - "logprobs": logprobs, - }, - timeout=timeout, - ) - return r.json() - except Exception: - return None - - def ok(resp) -> bool: """True iff resp is a successful completion (has choices, no error key).""" return ( @@ -399,37 +347,3 @@ def get_world_size(url, include_dp=True): params={"include_dp": include_dp}, timeout=5, ) - - -# --------------------------------------------------------------------------- -# GPU / metrics helpers -# --------------------------------------------------------------------------- - - -def gpu_free_bytes(device: int = 0) -> int: - """Read GPU free bytes via subprocess to avoid import-time torch init.""" - out = subprocess.check_output( - [ - sys.executable, - "-c", - f"import torch; f,_=torch.accelerator.get_memory_info({device}); print(f)", - ], - timeout=10, - ) - return int(out.strip()) - - -def sleep_metrics(url): - """Return (awake, weights_offloaded, discard_all) from /metrics.""" - try: - from prometheus_client.parser import text_string_to_metric_families - except ImportError: - return None, None, None - - r = requests.get(f"{url}/metrics", timeout=5) - vals: dict = {} - for family in text_string_to_metric_families(r.text): - if family.name == "vllm:engine_sleep_state": - for s in family.samples: - vals[s.labels.get("sleep_state", "")] = s.value - return vals.get("awake"), vals.get("weights_offloaded"), vals.get("discard_all") diff --git a/tests/entrypoints/serve/exception_handling/test_error_sanitization.py b/tests/entrypoints/serve/exception_handling/test_error_sanitization.py index 426d1ede74..98bf808965 100644 --- a/tests/entrypoints/serve/exception_handling/test_error_sanitization.py +++ b/tests/entrypoints/serve/exception_handling/test_error_sanitization.py @@ -18,6 +18,10 @@ def test_sanitize_message(): sanitize_message("<_io.BytesIO object at 0x7a95e299e750>") == "<_io.BytesIO object>" ) + assert ( + sanitize_message("") + == "" + ) class TestSanitizeMessageFilePaths: diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml new file mode 100644 index 0000000000..a3c6840fb3 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml @@ -0,0 +1,28 @@ +model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark" +accuracy_threshold: 0.92 +tolerance: 0.08 +min_acceptance_length: 1.2 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1800 +env: + VLLM_ROCM_USE_AITER: "1" +server_args: >- + --tokenizer-mode deepseek_v4 + --trust-remote-code + --max-model-len 8192 + --tensor-parallel-size 4 + --enable-expert-parallel + --all2all-backend allgather_reducescatter + --block-size 256 + --gpu-memory-utilization 0.5 + --kv-cache-dtype fp8 + --attention_config.indexer_kv_dtype=fp8 + --max-num-batched-tokens 16384 + --max-num-seqs 128 + --enforce-eager + --speculative-config '{"method":"dspark", + "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark", + "num_speculative_tokens":7, + "draft_sample_method":"probabilistic", + "enable_adaptive_verification":false}' diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml new file mode 100644 index 0000000000..879ad57c20 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.78 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_k3v4_nc --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml new file mode 100644 index 0000000000..aa66561223 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_k8v4 --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc-ROCm.yaml new file mode 100644 index 0000000000..103b37880a --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.75 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_3bit_nc --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml new file mode 100644 index 0000000000..1348661b47 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_4bit_nc --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml index b83d9e6a9e..c0bb8d3240 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml @@ -3,6 +3,5 @@ accuracy_threshold: 0.39 num_questions: 1319 num_fewshot: 5 server_args: >- - --enforce-eager --max-model-len 8192 --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml index 8ca4777ce5..357e7a0257 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml @@ -3,7 +3,6 @@ accuracy_threshold: 0.86 num_questions: 1319 num_fewshot: 5 server_args: >- - --enforce-eager --max-model-len 8192 --moe-backend humming --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml index 502ec79522..e41aa130ae 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml @@ -1,5 +1,5 @@ -model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM" -accuracy_threshold: 0.78 +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4" +accuracy_threshold: 0.70 num_questions: 1319 num_fewshot: 5 server_args: >- diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml new file mode 100644 index 0000000000..965a0fe54d --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml @@ -0,0 +1,8 @@ +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym" +accuracy_threshold: 0.85 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 4096 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt index 0537ed974a..0852d14633 100644 --- a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt +++ b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt @@ -1,3 +1,4 @@ gpt-oss-20b-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt index 68f5cd1c15..7a608c7c5a 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-int8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml Qwen3-30B-A3B-AWQ-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml -Qwen3-4B-mixed-quant-RTN-humming.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/tests/evals/gsm8k/configs/models-gfx950-small.txt b/tests/evals/gsm8k/configs/models-gfx950-small.txt new file mode 100644 index 0000000000..d818a067af --- /dev/null +++ b/tests/evals/gsm8k/configs/models-gfx950-small.txt @@ -0,0 +1,2 @@ +Qwen3-0.6B-FP8.yaml +Qwen3-1.7B-MXFP4.yaml diff --git a/tests/evals/gsm8k/configs/models-spec-decode-rocm.txt b/tests/evals/gsm8k/configs/models-spec-decode-rocm.txt index 177e336a3c..d96c405010 100644 --- a/tests/evals/gsm8k/configs/models-spec-decode-rocm.txt +++ b/tests/evals/gsm8k/configs/models-spec-decode-rocm.txt @@ -1,2 +1,3 @@ Kimi-K3-pruned75-DSpark-AITER-TP4.yaml DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml +DeepSeek-V4-Flash-DSpark-FP8-TP4-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt new file mode 100644 index 0000000000..9a1e252a4b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k3v4nc-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt new file mode 100644 index 0000000000..836993e0fb --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k3v4nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt new file mode 100644 index 0000000000..458fe69f00 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k8v4-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt b/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt new file mode 100644 index 0000000000..3c0f67d9bf --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k8v4.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt new file mode 100644 index 0000000000..cdc2462b8c --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t3nc-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt b/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt new file mode 100644 index 0000000000..49b31f737d --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t3nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt new file mode 100644 index 0000000000..cd599778b6 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t4nc-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt b/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt new file mode 100644 index 0000000000..8cf140fa69 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t4nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant.txt b/tests/evals/gsm8k/configs/models-turboquant.txt deleted file mode 100644 index 518aac780b..0000000000 --- a/tests/evals/gsm8k/configs/models-turboquant.txt +++ /dev/null @@ -1,4 +0,0 @@ -Qwen3-4B-TQ-k8v4.yaml -Qwen3-4B-TQ-t4nc.yaml -Qwen3-4B-TQ-k3v4nc.yaml -Qwen3-4B-TQ-t3nc.yaml diff --git a/tests/jit_monitor/test_no_runtime_jit_rocm.py b/tests/jit_monitor/test_no_runtime_jit_rocm.py new file mode 100644 index 0000000000..a99b8e1e14 --- /dev/null +++ b/tests/jit_monitor/test_no_runtime_jit_rocm.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Keep the MRV2 sampler warmup registry populated on ROCm.""" + +import pytest + +from vllm import LLM +from vllm.platforms import current_platform + +from ..models.utils import dummy_hf_overrides +from ..utils import create_new_process_for_each_test +from .test_no_runtime_jit import _run_shape_battery + +pytestmark = pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm") + + +def _worker_monitor_state(worker): + from vllm.utils import jit_monitor + + return ( + jit_monitor.is_active(), + worker.observability_config.jit_monitor_mode, + ) + + +@create_new_process_for_each_test("spawn") +def _run_rocm_shape_battery() -> None: + llm = LLM( + "Qwen/Qwen3-0.6B", + max_model_len=2048, + max_num_seqs=8, + gpu_memory_utilization=0.03, + kv_cache_memory_bytes=256 * 1024 * 1024, + load_format="dummy", + hf_overrides=dummy_hf_overrides, + enforce_eager=False, + jit_monitor_mode="error", + ) + try: + states = llm.collective_rpc(_worker_monitor_state, timeout=30) + assert states and all(state == (True, "error") for state in states) + _run_shape_battery(llm) + finally: + llm.llm_engine.engine_core.shutdown(timeout=30) + + +def test_v2_sampler_warmup_rocm(monkeypatch, tmp_path): + """Warmup must cover the existing prefill, decode, and sampler battery. + + A fresh cache and child process prevent previous model executions from + hiding a missing warmup specialization. + """ + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "0") + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + monkeypatch.setenv("TRITON_CACHE_DIR", str(tmp_path / "triton")) + monkeypatch.setenv("VLLM_CACHE_ROOT", str(tmp_path / "vllm")) + monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path / "inductor")) + _run_rocm_shape_battery() diff --git a/tests/kernels/attention/test_attention.py b/tests/kernels/attention/test_attention.py index 560eea641d..e78ede88f0 100644 --- a/tests/kernels/attention/test_attention.py +++ b/tests/kernels/attention/test_attention.py @@ -309,46 +309,6 @@ def test_paged_attention( torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol) -def ref_multi_query_kv_attention( - cu_seq_lens: list[int], - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - scale: float, - alibi_bias: list[torch.Tensor] | None, - dtype: torch.dtype, -) -> torch.Tensor: - num_seqs = len(cu_seq_lens) - 1 - ref_outputs: list[torch.Tensor] = [] - if alibi_bias: - assert len(alibi_bias) == num_seqs - for i in range(num_seqs): - start_idx = cu_seq_lens[i] - end_idx = cu_seq_lens[i + 1] - seq_len = end_idx - start_idx - - # Create attention mask. ALiBi already includes a tril causal mask. - if alibi_bias: - attn_mask = alibi_bias[i] - else: - attn_mask = torch.triu( - torch.ones(seq_len, seq_len, dtype=dtype), diagonal=1 - ) - attn_mask = attn_mask * torch.finfo(dtype).min - attn_mask = attn_mask.to(dtype=dtype) - - ref_output = ref_masked_attention( - query[start_idx:end_idx], - key[start_idx:end_idx], - value[start_idx:end_idx], - scale, - attn_mask=attn_mask, - ) - ref_outputs.append(ref_output) - - return torch.cat(ref_outputs, dim=0) - - @pytest.mark.parametrize("attention_cls", [Attention, MMEncoderAttention]) def test_num_heads_not_divisible_by_num_kv_heads(attention_cls: type) -> None: head_size = 64 diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index 53867a5edf..cb72c27904 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -341,61 +341,6 @@ def test_fp32_fallback(device: str): get_attn_backend(16, torch.float32, None) -def test_flash_attn(monkeypatch: pytest.MonkeyPatch): - """Test FlashAttn validation.""" - pytest.skip( - "Skipping as current backend selector does not " - "handle fallbacks when a backend is explicitly set." - ) - - attention_config = AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN) - cache_config = CacheConfig(block_size=16) - vllm_config = VllmConfig( - attention_config=attention_config, cache_config=cache_config - ) - - with set_current_vllm_config(vllm_config): - # Unsupported CUDA arch - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _=None: (7, 5)) - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # Reset the monkeypatch for subsequent tests - monkeypatch.undo() - - # Unsupported data type - backend = get_attn_backend(16, torch.float8_e4m3fn, None) - assert backend.get_name() != "FLASH_ATTN" - - # Unsupported kv cache data type - backend = get_attn_backend(16, torch.float16, "fp8") - assert backend.get_name() != "FLASH_ATTN" - - # Unsupported block size - vllm_config.cache_config.block_size = 8 - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # flash-attn is not installed - import sys - - vllm_config.cache_config.block_size = 16 - original_module = sys.modules.get("vllm_flash_attn") - monkeypatch.setitem(sys.modules, "vllm_flash_attn", None) - backend = get_attn_backend(16, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - # Restore the original module if it existed - if original_module is not None: - monkeypatch.setitem(sys.modules, "vllm_flash_attn", original_module) - else: - monkeypatch.delitem(sys.modules, "vllm_flash_attn", raising=False) - - # Unsupported head size - backend = get_attn_backend(17, torch.float16, None) - assert backend.get_name() != "FLASH_ATTN" - - def test_invalid_backend(): """Test that invalid attention backend names raise ValueError.""" with ( @@ -848,7 +793,7 @@ def hopper_selection(): def test_hopper_mm_prefix_selects_triton_flash_attn( use_mm_prefix, flash_attn_version, hopper_selection ): - """Hopper selects the composite only when its causal route resolves FA4.""" + """Hopper selects the composite when its causal route resolves FA3 or FA4.""" from vllm.engine.arg_utils import EngineArgs config = EngineArgs( @@ -860,11 +805,7 @@ def test_hopper_mm_prefix_selects_triton_flash_attn( backend = get_attn_backend( 256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix ) - fa4_resolved = config.attention_config.flash_attn_version == 4 - if use_mm_prefix: - expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN" - else: - expected = "FLASH_ATTN" + expected = "TRITON_FLASH_ATTN" if use_mm_prefix else "FLASH_ATTN" assert backend.get_name() == expected diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 41f0a1138b..4bc0ce1a67 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -1012,6 +1012,50 @@ def test_concat_and_cache_ds_mla( torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1) +@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("block_size", [64, 256]) +@torch.inference_mode() +def test_concat_and_cache_ds_mla_nope(device: str, block_size: int) -> None: + """NoPE matches zero RoPE, clears valid tails, and preserves unused slots.""" + dtype = torch.bfloat16 + if current_platform.is_rocm(): + pytest.skip("concat_and_cache_mla doesn't support fp8_ds_mla on ROCm") + num_tokens, num_blocks = 3, 2 + set_random_seed(0) + torch.set_default_device(device) + torch.accelerator.set_device_index(device) + + slot_mapping = torch.tensor( + [block_size - 1, block_size, -1], dtype=torch.long, device=device + ) + kv_c = torch.randn(num_tokens, 512, dtype=dtype, device=device) + k_pe = torch.empty(num_tokens, 0, dtype=dtype, device=device) + zero_k_pe = torch.zeros(num_tokens, 64, dtype=dtype, device=device) + scale = torch.tensor(1.0, dtype=torch.float32, device=device) + kv_cache = torch.full( + (num_blocks, block_size, 656), 0xFF, dtype=torch.uint8, device=device + ) + ref_cache = kv_cache.clone() + + opcheck( + torch.ops._C_cache_ops.concat_and_cache_mla, + (kv_c, k_pe, kv_cache, slot_mapping, "fp8_ds_mla", scale), + test_utils=DEFAULT_OPCHECK_TEST_UTILS, + ) + kv_cache.fill_(0xFF) + ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, "fp8_ds_mla", scale) + ops.concat_and_cache_mla( + kv_c, zero_k_pe, ref_cache, slot_mapping, "fp8_ds_mla", scale + ) + + torch.testing.assert_close(kv_cache, ref_cache, atol=0, rtol=0) + rows = kv_cache.view(num_blocks * block_size, 656) + assert (rows[slot_mapping[:2], 528:] == 0).all() + untouched = torch.ones(num_blocks * block_size, dtype=torch.bool, device=device) + untouched[slot_mapping[:2]] = False + assert (rows[untouched] == 0xFF).all() + + # Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py). NVFP4_DS_MLA_ENTRY_SIZE = 352 diff --git a/tests/kernels/attention/test_cpu_attn.py b/tests/kernels/attention/test_cpu_attn.py index 7c4b7811c7..f79b1b15d2 100644 --- a/tests/kernels/attention/test_cpu_attn.py +++ b/tests/kernels/attention/test_cpu_attn.py @@ -3,13 +3,20 @@ import functools import math +from types import SimpleNamespace import pytest import torch +from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import set_random_seed +from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.cpu_attn import _get_attn_isa +from vllm.v1.attention.backends.zentorch_sdpa import ( + should_use_zentorch_sdpa, + zentorch_sdpa_attn, +) if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) @@ -47,6 +54,24 @@ _FP8_RTOL = 0.1 ENCODER_SEQ_LENS = [ [1, 678, 2367, 145, 4162, 36, 7812], ] +# Uniform (dense zentorch_sdpa call) and ragged (per-sequence loop). +ZEN_ENCODER_SEQ_LENS = [ + [128, 128, 128], + [1, 67, 233, 5], +] +_skip_no_zentorch_sdpa = pytest.mark.skipif( + not current_platform.is_zen_cpu() or not has_zentorch_op(["zentorch_sdpa_attn"]), + reason="zentorch_sdpa requires a Zen CPU with the op registered", +) +# should_use_zentorch_sdpa additionally gates each dtype on its ISA. +_skip_no_avx512_bf16 = pytest.mark.skipif( + not torch.cpu._is_avx512_bf16_supported(), + reason="zentorch_sdpa bfloat16 requires AVX512-BF16", +) +_skip_no_avx512 = pytest.mark.skipif( + not torch.cpu._is_avx512_supported(), + reason="zentorch_sdpa float32 requires AVX512", +) def get_attn_isa( @@ -205,6 +230,7 @@ def ref_varlen_encoder_attn( seq_lens: list[int], scale: float, sliding_window: int | None = None, + alibi_slopes: torch.Tensor | None = None, ) -> torch.Tensor: num_seqs = len(seq_lens) dtype = query.dtype @@ -232,6 +258,11 @@ def ref_varlen_encoder_attn( else: mask = empty_mask.logical_not() + if alibi_slopes is not None: + q_pos = torch.arange(0, seq_len)[None, :, None] + kv_pos = torch.arange(0, seq_len)[None, None, :] + attn += -alibi_slopes[:, None, None].float() * (q_pos - kv_pos) + attn.masked_fill_(mask, float("-inf")) attn = torch.softmax(attn, dim=-1) out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype) @@ -428,6 +459,7 @@ def varlen_with_paged_kv( k_scale: float = 1.0, v_scale: float = 1.0, dynamic_causal: list[bool] | None = None, + s_aux_dtype: torch.dtype = torch.bfloat16, ) -> None: set_random_seed(0) num_seqs = len(seq_lens) @@ -449,9 +481,7 @@ def varlen_with_paged_kv( # 2^(-8/n) alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None - s_aux = ( - 15 * torch.rand((num_query_heads,), dtype=torch.bfloat16) if use_sink else None - ) + s_aux = 15 * torch.rand((num_query_heads,), dtype=s_aux_dtype) if use_sink else None is_fp8 = kv_cache_dtype != "auto" if is_fp8 and current_platform.get_cpu_architecture() != CpuArchEnum.X86: @@ -538,6 +568,8 @@ def varlen_with_paged_kv( ) out_without_split = torch.empty_like(query) + if s_aux is not None and s_aux.dtype != torch.bfloat16: + s_aux = s_aux.to(torch.float32) cpu_attention_with_kv_cache( query=query, key_cache=packed_key_cache, @@ -771,6 +803,108 @@ def test_varlen_encoder_attention_amx( ) +@torch.inference_mode() +def varlen_encoder_zentorch_sdpa( + seq_lens: list[int], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + use_alibi: bool, +) -> None: + set_random_seed(0) + num_query_heads, num_kv_heads = num_heads + assert num_query_heads % num_kv_heads == 0 + scale = head_size**-0.5 + token_num = sum(seq_lens) + + query = tensor_cache( + token_num * num_query_heads * head_size, dtype, tag="query" + ).view(token_num, num_query_heads, head_size) + key_value = tensor_cache( + 2 * token_num * num_kv_heads * head_size, dtype, tag="kv" + ).view(2, token_num, num_kv_heads, head_size) + key, value = key_value.unbind(0) + + query_start_loc = torch.zeros(len(seq_lens) + 1, dtype=torch.int32) + torch.cumsum(torch.tensor(seq_lens, dtype=torch.int32), 0, out=query_start_loc[1:]) + alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None + output = torch.empty_like(query) + + zentorch_sdpa_attn( + query, + key, + value, + output, + SimpleNamespace(query_start_loc=query_start_loc, causal=False), + scale, + sliding_window if sliding_window is not None else -1, + alibi_slopes, + ) + ref_output = ref_varlen_encoder_attn( + query=query, + key=key, + value=value, + seq_lens=seq_lens, + scale=scale, + sliding_window=sliding_window, + alibi_slopes=alibi_slopes, + ) + torch.testing.assert_close(output, ref_output, atol=1.5e-2, rtol=1e-2) + + +_ZENTORCH_SDPA_DTYPES = [ + pytest.param(torch.bfloat16, marks=_skip_no_avx512_bf16), + pytest.param(torch.float32, marks=_skip_no_avx512), +] + + +@_skip_no_zentorch_sdpa +@pytest.mark.parametrize("seq_lens", ZEN_ENCODER_SEQ_LENS) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", [128]) +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS) +@pytest.mark.parametrize("dtype", _ZENTORCH_SDPA_DTYPES) +@pytest.mark.parametrize("use_alibi", [False, True]) +def test_zentorch_sdpa_attn( + seq_lens: list[int], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + use_alibi: bool, +) -> None: + varlen_encoder_zentorch_sdpa( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + sliding_window=sliding_window, + dtype=dtype, + use_alibi=use_alibi, + ) + + +@_skip_no_zentorch_sdpa +@pytest.mark.parametrize( + "attn_type", [AttentionType.ENCODER_ONLY, AttentionType.ENCODER] +) +@pytest.mark.parametrize("dtype", _ZENTORCH_SDPA_DTYPES) +def test_should_use_zentorch_sdpa(attn_type: str, dtype: torch.dtype) -> None: + assert should_use_zentorch_sdpa(attn_type, dtype) + + +@_skip_no_zentorch_sdpa +@pytest.mark.parametrize( + "attn_type,dtype", + [ + (AttentionType.DECODER, torch.bfloat16), + (AttentionType.ENCODER_ONLY, torch.float16), + ], +) +def test_should_not_use_zentorch_sdpa(attn_type: str, dtype: torch.dtype) -> None: + assert not should_use_zentorch_sdpa(attn_type, dtype) + + @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"]) @pytest.mark.parametrize("seq_lens", SEQ_LENS) @pytest.mark.parametrize("num_heads", NUM_HEADS) @@ -1128,6 +1262,51 @@ def test_varlen_with_paged_kv_sink( ) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3"]) +@pytest.mark.parametrize("seq_lens", SEQ_LENS) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", [96]) +@pytest.mark.parametrize("block_size", [128]) +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("soft_cap", [None]) +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS) +@pytest.mark.parametrize("use_alibi", [False]) +@pytest.mark.parametrize("use_sink", [True]) +@pytest.mark.parametrize("isa", [get_attn_isa()]) +@pytest.mark.parametrize("s_aux_dtype", [torch.float16]) +def test_varlen_with_paged_kv_sink_fp16( + seq_lens: list[tuple[int, int]], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + block_size: int, + soft_cap: float | None, + num_blocks: int, + use_alibi: bool, + use_sink: bool, + isa: str, + kv_cache_dtype: str, + s_aux_dtype: torch.dtype, +) -> None: + varlen_with_paged_kv( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + sliding_window=sliding_window, + dtype=dtype, + block_size=block_size, + soft_cap=soft_cap, + num_blocks=num_blocks, + use_alibi=use_alibi, + use_sink=use_sink, + isa=isa, + kv_cache_dtype=kv_cache_dtype, + s_aux_dtype=s_aux_dtype, + ) + + @pytest.mark.parametrize( "kv_cache_dtype", [ diff --git a/tests/kernels/attention/test_flashmla_sparse.py b/tests/kernels/attention/test_flashmla_sparse.py index bfcd6ae10a..8fcf3f6a58 100644 --- a/tests/kernels/attention/test_flashmla_sparse.py +++ b/tests/kernels/attention/test_flashmla_sparse.py @@ -509,3 +509,334 @@ def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width(): assert sparse_indices.shape == (1, padded_width) assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1] assert sparse_lens.cpu().tolist() == [padded_width] + + +def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps(): + """A DSpark non-causal row keeps -1 out of the kernel's active ranges. + + Each draft token's causal window holds its first visible entries, the rest + of the block starts at column `window`, and the compressed entries follow + directly; `sparse_topk_lens` ends right after them. + """ + from vllm.models.deepseek_v4.common.ops.cache_utils import ( + build_flashinfer_mixed_sparse_indices, + ) + + device = torch.device("cuda") + window, width = 4, 8 + # One request: 1 context token, a 2-token draft block, slots 0..2 visible. + decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device) + decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device) + compressed = torch.tensor( + [[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device + ) + sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( + decode_swa_indices=decode_swa, + decode_compressed_indices=compressed, + decode_compressed_topk_lens=torch.tensor( + [2, 2], dtype=torch.int32, device=device + ), + prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device), + query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device), + seq_lens=torch.tensor([3], dtype=torch.int32, device=device), + token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device), + swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), + swa_block_size=64, + compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), + compressed_block_size=64, + window_size=window, + compress_ratio=128, + topk=0, + ) + + assert sparse_indices.cpu().tolist() == [ + # pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it. + [0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1], + # pos 2: window [0, 1, 2]; nothing spills; compressed at column 4. + [0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1], + ] + assert sparse_lens.cpu().tolist() == [7, 6] + + +def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length(): + """A CUDA-graph pad token of the DSpark graph (a zero-length request) gets + a causal pad row's shape instead of negative spill arithmetic.""" + from vllm.models.deepseek_v4.common.ops.cache_utils import ( + build_flashinfer_mixed_sparse_indices, + ) + + device = torch.device("cuda") + window, width = 128, 192 + decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device) + decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device) + sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( + decode_swa_indices=decode_swa, + decode_compressed_indices=None, + decode_compressed_topk_lens=None, + prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device), + # Request 1 is the padding: no query tokens, seq_len 0. + query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device), + seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device), + token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device), + swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device), + swa_block_size=64, + compressed_block_table=None, + compressed_block_size=64, + window_size=window, + compress_ratio=1, + topk=0, + ) + + assert sparse_lens.cpu().tolist() == [129, 128, 128] + assert (sparse_indices[2] == -1).all() + + +@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"]) +@pytest.mark.parametrize("page_padding", [0, 64]) +def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding): + """Decode and prefill must read storage rows after vLLM remaps packed pages.""" + import importlib + + from vllm.platforms import current_platform + from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103") + mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse") + device = "cuda" + block_size = 64 + storage = torch.full( + (3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device + ) + cache = storage[:, :block_size] + cache[1].fill_(2) + cache[2].fill_(3) + attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention) + attn.compress_ratio = 1 + attn.window_size = 128 + attn.scale = 512**-0.5 + attn.kv_cache_torch_dtype = torch.bfloat16 + attn.attn_sink = torch.full((64,), -float("inf"), device=device) + attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device) + decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device) + decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device) + query_start = torch.tensor([0, 1, 3], dtype=torch.int32) + metadata = DeepseekSparseSWAMetadata( + block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device), + slot_mapping=torch.tensor([67, 130, 131], device=device), + block_size=block_size, + seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device), + query_start_loc=query_start.to(device), + query_start_loc_cpu=query_start, + token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device), + decode_swa_indices=decode_indices, + decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device), + decode_swa_width=128, + is_valid_token=torch.ones(3, dtype=torch.bool, device=device), + replay_start=torch.zeros(2, dtype=torch.int32, device=device), + num_decodes=1, + num_prefills=1, + num_decode_tokens=1, + num_prefill_tokens=2, + max_decode_query_len=1, + ) + query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device) + output = torch.empty_like(query) + attn._forward(query, None, cache, metadata, None, True, output) + expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device) + torch.testing.assert_close(output, expected[:, None, None].expand_as(output)) + + +def _make_rope_quant_attn(mod, num_rows: int, device: str): + """A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV.""" + from types import SimpleNamespace + + heads, groups, rank = 128, 16, 256 + angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05 + angles = angles + torch.arange(32, device=device)[None] * 0.1 + attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention) + attn.compress_ratio = 1 + attn.window_size = 128 + attn.scale = 512**-0.5 + attn.kv_cache_torch_dtype = torch.float8_e4m3fn + attn._flashinfer_fp8_bmm1_scale = attn.scale + attn._flashinfer_fp8_bmm2_scale = 1.0 + attn.attn_sink = torch.linspace(-1, 1, heads, device=device) + attn.padded_heads = heads + attn.topk_indices_buffer = torch.empty( + (num_rows, 0), dtype=torch.int32, device=device + ) + attn.rotary_emb = SimpleNamespace( + cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous() + ) + attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank + attn.nope_head_dim, attn.rope_head_dim = 448, 64 + attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True + attn.wo_a = SimpleNamespace( + weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to( + torch.float8_e4m3fn + ), + weight_scale=torch.ones((groups, rank, 32), device=device), + ) + attn.wo_b = lambda z: z + return attn + + +def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens): + """Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused).""" + from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out + + heads, groups = attn.n_local_heads, attn.n_local_groups + unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device) + attn._forward(q, None, cache, metadata, None, True, unfused) + fused = rope_quant_attn_out(q.shape[0], q.device) + metadata.flashinfer_sparse_index_cache.clear() + attn._forward(q, None, cache, metadata, None, True, fused) + z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions) + z_fused = attn._o_proj(fused, positions)[:num_tokens] + return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1) + + +def _assert_projects_alike(z_unfused, z_fused): + z_unfused = z_unfused.float().view_as(z_fused) + rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm() + assert rel < 2e-2, rel + + +def _skip_unless_rope_quant(): + from vllm.platforms import current_platform + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103") + + +def test_flashinfer_rope_quant_matches_unfused_o_proj(): + """RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path. + + One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows, + so both calls run over the full buffer with un-rebased cum_seq_lens_q. + """ + from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod + from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata + + _skip_unless_rope_quant() + torch.manual_seed(0) + device = "cuda" + block_size = 64 + q_lens, seq_lens = [1, 1, 3], [5, 9, 70] + num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3 + cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn) + attn = _make_rope_quant_attn(mod, num_rows, device) + + decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device) + for i, seq_len in enumerate(seq_lens[:2]): + decode_indices[i, :seq_len] = torch.arange( + i * block_size, i * block_size + seq_len, device=device + ) + query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32) + metadata = DeepseekSparseSWAMetadata( + block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device), + slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device), + block_size=block_size, + seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device), + query_start_loc=query_start.to(device), + query_start_loc_cpu=query_start, + token_to_req_indices=torch.tensor( + [0, 1, 2, 2, 2], dtype=torch.int32, device=device + ), + decode_swa_indices=decode_indices, + decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device), + decode_swa_width=128, + is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device), + num_decodes=2, + num_prefills=1, + num_decode_tokens=2, + num_prefill_tokens=3, + max_decode_query_len=1, + ) + positions = torch.tensor([4, 8, 67, 68, 69], device=device) + q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn) + _, z_unfused, z_fused = _rope_quant_vs_unfused( + attn, q, cache, metadata, positions, num_tokens + ) + _assert_projects_alike(z_unfused, z_fused) + + +@pytest.mark.parametrize("context_len", [0, 20, 127, 900]) +def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len): + """Every DSpark draft token attends to the whole block, fused or not. + + The kernel counts every entry of its active index ranges as a key, so the + builder must place the block around each query's causal window without -1 + gaps, down to an empty context. + """ + from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod + from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, + ) + from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata + + _skip_unless_rope_quant() + torch.manual_seed(0) + device = "cuda" + block_size, window, draft = 64, 128, 4 + width = get_dspark_swa_index_width(window, draft) + contexts = [context_len, context_len + 37] + num_reqs, num_tokens = len(contexts), len(contexts) * draft + num_rows = num_tokens + 2 + pages = -(-(max(contexts) + draft) // block_size) + cache = torch.randn((num_reqs * pages, block_size, 512), device=device) + cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn) + attn = _make_rope_quant_attn(mod, num_rows, device) + attn.attn_sink = torch.full((128,), -float("inf"), device=device) + + # As the DSpark SWA builder lays them out: the trailing window of context + # and the whole block, contiguous from column 0. + decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32) + visible, positions = [], [] + for r, ctx in enumerate(contexts): + slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size + for k in range(draft): + decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32) + visible.append(slots) + positions.append(ctx + k) + query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32) + seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32) + metadata = DeepseekSparseSWAMetadata( + block_table=torch.arange( + num_reqs * pages, dtype=torch.int32, device=device + ).view(num_reqs, pages), + slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device), + block_size=block_size, + seq_lens=seq_lens.to(device), + query_start_loc=query_start.to(device), + query_start_loc_cpu=query_start, + token_to_req_indices=torch.arange( + num_reqs, dtype=torch.int32, device=device + ).repeat_interleave(draft), + decode_swa_indices=decode_indices.to(device), + decode_swa_lens=torch.tensor( + [v.numel() for v in visible], dtype=torch.int32, device=device + ), + decode_swa_width=width, + is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device), + num_decodes=num_reqs, + num_prefills=0, + num_decode_tokens=num_tokens, + num_prefill_tokens=0, + max_decode_query_len=draft, + ) + q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1) + q = q.to(torch.float8_e4m3fn) + attention, z_unfused, z_fused = _rope_quant_vs_unfused( + attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens + ) + + keys_all = cache.flatten(0, 1).float() + for token, slots in enumerate(visible): + keys = keys_all[slots.to(device)] + weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1) + torch.testing.assert_close( + attention[token].float(), weights @ keys, atol=0.05, rtol=0.05 + ) + _assert_projects_alike(z_unfused, z_fused) diff --git a/tests/kernels/attention/test_rocm_attention_selector.py b/tests/kernels/attention/test_rocm_attention_selector.py deleted file mode 100644 index 2a684ed70d..0000000000 --- a/tests/kernels/attention/test_rocm_attention_selector.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest -import torch - -from vllm.config import AttentionConfig, VllmConfig, set_current_vllm_config -from vllm.platforms.rocm import RocmPlatform -from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.attention.selector import _cached_get_attn_backend, get_attn_backend - - -@pytest.fixture(autouse=True) -def clear_cache(): - """Clear lru cache to ensure each test case runs without caching.""" - _cached_get_attn_backend.cache_clear() - - -@pytest.mark.skip(reason="Skipped for now. Should be revisited.") -def test_selector(monkeypatch: pytest.MonkeyPatch): - # Set the current platform to ROCm using monkeypatch - monkeypatch.setattr("vllm.v1.attention.selector.current_platform", RocmPlatform()) - - # Test standard ROCm attention - attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_ATTN) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(16, torch.float16, torch.float16, 16, False) - assert backend.get_name() == "ROCM_FLASH" or backend.get_name() == "TRITON_ATTN" - - # MLA test for deepseek related - # Change the attention backend to triton MLA - attention_config = AttentionConfig(backend=AttentionBackendEnum.TRITON_MLA) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True) - assert backend.get_name() == "TRITON_MLA" - - # If attention backend is None - # If use_mla is true - # The selected backend is triton MLA - attention_config = AttentionConfig(backend=None) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 16, False, use_mla=True) - assert backend.get_name() == "TRITON_MLA" - - # Change the attention backend to AITER MLA - attention_config = AttentionConfig(backend=AttentionBackendEnum.ROCM_AITER_MLA) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend(576, torch.bfloat16, "auto", 1, False, use_mla=True) - assert backend.get_name() == "ROCM_AITER_MLA" - - # If attention backend is None - # If use_mla is true - # If VLLM_ROCM_USE_AITER is enabled - # The selected backend is ROCM_AITER_MLA - with monkeypatch.context() as m: - m.setenv("VLLM_ROCM_USE_AITER", "1") - - attention_config = AttentionConfig(backend=None) - vllm_config = VllmConfig(attention_config=attention_config) - - with set_current_vllm_config(vllm_config): - backend = get_attn_backend( - 576, torch.bfloat16, "auto", 1, False, use_mla=True - ) - assert backend.get_name() == "ROCM_AITER_MLA" diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index 5b68732f7a..bd41048d9b 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools from types import SimpleNamespace import pytest @@ -51,6 +52,65 @@ ROPE_HEAD_DIM = 64 HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM +@pytest.fixture +def enable_aiter_mqa(monkeypatch): + from vllm._aiter_ops import rocm_aiter_ops + + with monkeypatch.context() as context: + context.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + yield + rocm_aiter_ops.refresh_env_variables() + + +@requires_gfx950 +@pytest.mark.parametrize("column_scales", [False, True], ids=["vector", "column"]) +@torch.inference_mode() +def test_unpaged_mqa_logits_preserves_intervals_and_scale_layout( + column_scales, monkeypatch, enable_aiter_mqa +) -> None: + """The vLLM AITER dispatch preserves ragged masking and per-token scales.""" + from vllm._aiter_ops import rocm_aiter_ops + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + + assert rocm_aiter_ops.is_enabled(), "AITER is required on gfx950" + assert mod.mqa_logits_module() is not None, "AITER MQA logits are required" + torch.manual_seed(42) + device = torch.device("cuda") + dtype = current_platform.fp8_dtype() + q = torch.randn(7, 32, 128, device=device).to(dtype) + k = torch.randn(259, 128, device=device).to(dtype) + scales = torch.linspace(0.25, 1.25, k.shape[0], device=device) + if column_scales: + scales = scales[:, None] + weights = torch.rand(q.shape[:2], device=device) / q.shape[1] + starts = torch.tensor([0, 3, 128, 5, 0, 257, 259], device=device, dtype=torch.int32) + ends = torch.tensor( + [1, 129, 259, 5, 259, 259, 259], device=device, dtype=torch.int32 + ) + + module = mod.mqa_logits_module() + implementation = module.fp8_mqa_logits + calls = 0 + + def traced_implementation(*args, **kwargs): + nonlocal calls + calls += 1 + return implementation(*args, **kwargs) + + monkeypatch.setattr(module, "fp8_mqa_logits", traced_implementation) + actual = mod.rocm_fp8_mqa_logits(q, (k, scales), weights, starts, ends) + assert calls == 1 + score = torch.einsum("mhd,nd->mhn", q.float(), k.float()) * scales.reshape(-1) + expected = (score.relu() * weights[..., None]).sum(dim=1) + positions = torch.arange(k.shape[0], device=device) + valid = (positions >= starts[:, None]) & (positions < ends[:, None]) + assert actual.shape == expected.shape + assert actual.dtype == torch.float32 + assert torch.equal(torch.isneginf(actual), ~valid) + torch.testing.assert_close(actual[valid], expected[valid], rtol=1e-3, atol=1e-3) + + def _ref_global_topk_ragged( topk_indices: torch.Tensor, token_to_req_indices: torch.Tensor, @@ -259,13 +319,21 @@ def _launch_sparse_decode_reduce( part_l: torch.Tensor, part_acc: torch.Tensor, adaptive_splits: bool, + positions: torch.Tensor | None = None, + cos_sin_cache: torch.Tensor | None = None, + out_dtype: torch.dtype = torch.bfloat16, + out_scale: torch.Tensor | None = None, ) -> torch.Tensor: + """Run the reduce kernel, with the inverse-RoPE epilogue if given positions. + + Passing ``out_scale`` selects the MXFP8 epilogue (``out_dtype`` e4m3). + """ from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod num_queries, num_splits, num_heads = part_m.shape out = torch.empty( (num_queries, num_heads, HEAD_DIM), - dtype=torch.bfloat16, + dtype=out_dtype, device=part_m.device, ) attn_sink = torch.empty(1, dtype=torch.float32, device=part_m.device) @@ -275,13 +343,19 @@ def _launch_sparse_decode_reduce( part_acc, attn_sink, out, + positions, + cos_sin_cache, + out_scale, out.stride(0), out.stride(1), + out_scale.stride(0) if out_scale is not None else 0, + HEAD_DIM // 32, part_m.stride(0), part_m.stride(1), part_acc.stride(0), part_acc.stride(1), part_acc.stride(2), + cos_sin_cache.stride(0) if cos_sin_cache is not None else 0, num_heads, HAS_ATTN_SINK=False, ADAPTIVE_SPLITS=adaptive_splits, @@ -289,6 +363,10 @@ def _launch_sparse_decode_reduce( BLOCK_H=1, NUM_SPLITS=num_splits, SPLITS_PAD=1 << (num_splits - 1).bit_length(), + FUSE_INV_ROPE=positions is not None, + NOPE=NOPE_HEAD_DIM, + HALF=ROPE_HEAD_DIM // 2, + QUANT_OUT=out_scale is not None, num_warps=4, ) return out @@ -1008,6 +1086,224 @@ def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> No assert torch.equal(actual, torch.full_like(actual, 2)) +@requires_gfx950 +@pytest.mark.parametrize("adaptive_splits", [False, True]) +@torch.inference_mode() +def test_sparse_attn_decode_reduce_inverse_rope_epilogue(adaptive_splits: bool) -> None: + """FUSE_INV_ROPE must land where the standalone rotation would. + + The epilogue rotates a whole [H, nope + rope] row with one expression by + feeding cos=1/sin=0 to the NoPE lanes, which only holds if the pair split + lands on ``nope_head_dim // 2``; the NoPE lanes therefore have to come + back untouched, not merely close. + """ + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _fused_inverse_rope_gptj + + device = torch.device("cuda") + torch.manual_seed(7) + num_queries, num_splits, num_heads, max_pos = 5, 4, 8, 64 + shape = (num_queries, num_splits, num_heads) + part_m = torch.randn(shape, dtype=torch.float32, device=device) + part_l = torch.rand(shape, dtype=torch.float32, device=device) + 0.5 + part_acc = torch.randn((*shape, HEAD_DIM), dtype=torch.float32, device=device) + positions = torch.randint( + 0, max_pos, (num_queries,), dtype=torch.int64, device=device + ) + angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device) + cos_sin_cache = torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous() + + unfused = _launch_sparse_decode_reduce(part_m, part_l, part_acc, adaptive_splits) + expected = _fused_inverse_rope_gptj( + unfused, positions, cos_sin_cache, ROPE_HEAD_DIM + ) + actual = _launch_sparse_decode_reduce( + part_m, part_l, part_acc, adaptive_splits, positions, cos_sin_cache + ) + + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + assert torch.equal(actual[..., :NOPE_HEAD_DIM], unfused[..., :NOPE_HEAD_DIM]) + # An epilogue that quietly did nothing would satisfy everything above. + assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:]) + + +def _mxfp8_dequant(data: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + blocks = data.float().view(*data.shape[:-1], -1, 32) + return (blocks * torch.exp2(scale.float() - 127.0)[..., None]).view(data.shape) + + +def _random_cos_sin_cache(max_pos: int, device: torch.device) -> torch.Tensor: + angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device) + return torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous() + + +@requires_gfx950 +@torch.inference_mode() +def test_sparse_attn_decode_reduce_mxfp8_epilogue() -> None: + """QUANT_OUT must equal MXFP8-quantizing the reduce's fp32 output.""" + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + _mxfp8_e4m3_quantize_torch, + ) + + device = torch.device("cuda") + torch.manual_seed(7) + num_queries, num_splits, num_heads = 5, 4, 8 + shape = (num_queries, num_splits, num_heads) + part_m = torch.randn(shape, device=device) + part_l = torch.rand(shape, device=device) + 0.5 + part_acc = torch.randn((*shape, HEAD_DIM), device=device) + positions = torch.randint(0, 64, (num_queries,), device=device) + cos_sin_cache = _random_cos_sin_cache(64, device) + args = (part_m, part_l, part_acc, False, positions, cos_sin_cache) + + rotated = _launch_sparse_decode_reduce(*args, out_dtype=torch.float32) + expected_data, expected_scale = _mxfp8_e4m3_quantize_torch( + rotated.view(num_queries, -1) + ) + scale = torch.empty_like(expected_scale) + data = _launch_sparse_decode_reduce( + *args, out_dtype=torch.float8_e4m3fn, out_scale=scale + ) + + assert torch.equal(scale, expected_scale) + assert torch.equal( + data.view(num_queries, -1).view(torch.uint8), expected_data.view(torch.uint8) + ) + + +@requires_gfx950 +@torch.inference_mode() +def test_inverse_rope_mxfp8_rows() -> None: + """Prefill rows get the same inverse RoPE + MXFP8 as the decode epilogue.""" + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + _mxfp8_e4m3_quantize_torch, + ) + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + rocm_inverse_rope_mxfp8_rows, + ) + + device = torch.device("cuda") + torch.manual_seed(5) + num_tokens, num_heads = 7, 4 + o = torch.randn(num_tokens, num_heads, HEAD_DIM, device=device).bfloat16() + positions = torch.randint(0, 64, (num_tokens,), device=device) + cos_sin_cache = _random_cos_sin_cache(64, device) + + ref = o.float() + cos, sin = cos_sin_cache[positions, None].chunk(2, dim=-1) + even, odd = ref[..., NOPE_HEAD_DIM::2].clone(), ref[..., NOPE_HEAD_DIM + 1 :: 2] + ref[..., NOPE_HEAD_DIM::2] = even * cos + odd * sin + ref[..., NOPE_HEAD_DIM + 1 :: 2] = odd * cos - even * sin + ref = ref.view(num_tokens, -1) + _, expected_scale = _mxfp8_e4m3_quantize_torch(ref) + + data = torch.empty_like(ref, dtype=torch.float8_e4m3fn) + scale = torch.empty_like(expected_scale) + rocm_inverse_rope_mxfp8_rows( + o, positions, cos_sin_cache, ROPE_HEAD_DIM, data, scale + ) + + assert torch.equal(scale, expected_scale) + # Within half an e4m3 step at the top of each block's range. + half_step = 16.0 * torch.exp2(scale.float() - 127.0).repeat_interleave(32, -1) + assert bool(((_mxfp8_dequant(data, scale) - ref).abs() <= half_step).all()) + + +@requires_gfx950 +@torch.inference_mode() +def test_sparse_attn_decode_mxfp8_output() -> None: + """``out_mxfp8`` routes the real decode through the MXFP8 epilogue.""" + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _rocm_sparse_attn_decode_ragged_triton, + ) + + device = torch.device("cuda") + torch.manual_seed(1) + num_heads = 16 + q = torch.randn(2, num_heads, HEAD_DIM, device=device).bfloat16() * 0.125 + kv = torch.randn(6, HEAD_DIM, device=device).bfloat16() * 0.125 + decode = functools.partial( + _rocm_sparse_attn_decode_ragged_triton, + q=q, + main_cache=_pack_fp8_ds_mla_cache(kv, 4, current_platform.is_fp8_fnuz()), + main_indices=torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device), + main_indptr=torch.tensor([0, 2, 4], dtype=torch.int32, device=device), + scale=HEAD_DIM**-0.5, + attn_sink=None, + nope_head_dim=NOPE_HEAD_DIM, + rope_head_dim=ROPE_HEAD_DIM, + inv_rope_positions=torch.tensor([3, 9], device=device), + inv_rope_cos_sin_cache=_random_cos_sin_cache(16, device), + ) + + expected = decode().float().view(2, -1) + data = torch.empty( + 2, num_heads * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device + ) + scale = torch.empty(2, num_heads * HEAD_DIM // 32, dtype=torch.uint8, device=device) + decode(out_mxfp8=(data, scale)) + + torch.testing.assert_close( + _mxfp8_dequant(data, scale), expected, atol=1e-3, rtol=2**-4 + ) + + +@requires_gfx950 +@pytest.mark.parametrize( + "num_tokens, n_groups", + # One case per tile tier of _mxfp8_wo_a_bmm_config, with partial M tiles. + [ + (1, 4), + (20, 4), + (48, 4), + (77, 4), + (130, 4), + (300, 4), + (700, 4), + (1000, 4), + (1100, 8), + ], +) +@pytest.mark.parametrize("block_scales", [False, True]) +@torch.inference_mode() +def test_rocm_mxfp8_wo_a_bmm( + num_tokens: int, n_groups: int, block_scales: bool +) -> None: + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + _as_block32_scale, + ) + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + _mxfp8_e4m3_quantize_torch, + ) + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import rocm_mxfp8_wo_a_bmm + + device = torch.device("cuda") + torch.manual_seed(3) + o_lora_rank, group_dim = 128, 2048 + a, a_scale = _mxfp8_e4m3_quantize_torch( + torch.randn(num_tokens, n_groups * group_dim, device=device) + ) + w, w_scale = _mxfp8_e4m3_quantize_torch( + torch.randn(n_groups * o_lora_rank, group_dim, device=device) + ) + if block_scales: + # A 32x32 checkpoint loads as per-row scales that repeat every 32 rows; + # RocmDotScaledMxfp8LinearKernel compacts them before wo_a sees them. + w_scale = _as_block32_scale(w_scale[::32].repeat_interleave(32, dim=0)) + assert w_scale is not None + wo_a = SimpleNamespace(weight=w, weight_scale=w_scale) + + out = rocm_mxfp8_wo_a_bmm(a, a_scale, wo_a, n_groups, o_lora_rank) + + expected = torch.einsum( + "tgd,grd->tgr", + _mxfp8_dequant(a, a_scale).view(num_tokens, n_groups, group_dim), + _mxfp8_dequant( + w, w_scale.repeat_interleave(w.shape[0] // w_scale.shape[0], 0) + ).view(n_groups, o_lora_rank, group_dim), + ) + torch.testing.assert_close(out.float(), expected.flatten(1), atol=5e-2, rtol=1e-2) + + @requires_gfx950 @pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65]) @torch.inference_mode() diff --git a/tests/kernels/attention/test_triton_prefill_attention.py b/tests/kernels/attention/test_triton_prefill_attention.py index 6316a926ae..e9b90251a4 100644 --- a/tests/kernels/attention/test_triton_prefill_attention.py +++ b/tests/kernels/attention/test_triton_prefill_attention.py @@ -5,6 +5,7 @@ import pytest import torch import torch.nn.functional as F +import vllm.v1.attention.ops.triton_prefill_attention as prefill_ops from vllm.platforms import current_platform from vllm.v1.attention.ops.triton_prefill_attention import context_attention_fwd @@ -230,3 +231,87 @@ def test_context_attention_sliding_window( # Compare outputs torch.testing.assert_close(o, o_ref, rtol=2e-2, atol=2e-2) + + +class _LaunchCapture: + """Records the launch configuration in place of ``_fwd_kernel``.""" + + grid: tuple + kwargs: dict + + def __getitem__(self, grid): + self.grid = grid + return self._record + + def _record(self, *args, **kwargs) -> None: + self.kwargs = kwargs + + +def _capture_tile_config( + monkeypatch, *, is_rocm: bool, on_gfx1x: bool, dtype=torch.bfloat16 +) -> _LaunchCapture: + """Capture the tile configuration with the platform predicates mocked. + + Both tile widths are numerically correct, so the tests above pass whichever + one is selected. This needs no RDNA part and allocates no device memory, so + it covers the RDNA branch on the CDNA and NVIDIA agents CI actually runs. + """ + platform = prefill_ops.current_platform + monkeypatch.setattr(platform, "is_rocm", lambda: is_rocm) + # Every device this kernel targets is cuda-alike at capability 80 or better, + # so the stock tile is 128 for 16-bit dtypes. + monkeypatch.setattr(platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(platform, "has_device_capability", lambda *a, **k: True) + if is_rocm: + import vllm.platforms.rocm as rocm_platform + + monkeypatch.setattr(rocm_platform, "on_gfx1x", lambda: on_gfx1x) + + capture = _LaunchCapture() + monkeypatch.setattr(prefill_ops, "_fwd_kernel", capture) + + def meta(*shape): + return torch.empty(shape, dtype=dtype, device="meta") + + seq_lens = torch.empty(2, dtype=torch.int32, device="meta") + context_attention_fwd( + meta(256, 8, 128), + meta(256, 2, 128), + meta(256, 2, 128), + meta(256, 8, 128), + seq_lens, + seq_lens, + 128, + ) + return capture + + +@pytest.mark.parametrize( + ("is_rocm", "on_gfx1x", "dtype", "expected_block_n"), + [ + pytest.param(True, True, torch.bfloat16, 32, id="rdna"), + # on_gfx1x() is what excludes gfx10xx, CDNA and gfx1250. + pytest.param(True, False, torch.bfloat16, 128, id="rocm-not-rdna"), + pytest.param(False, False, torch.bfloat16, 128, id="not-rocm"), + # get_block_size already returns 32 here, so min() must not widen it. + pytest.param(True, True, torch.float32, 32, id="rdna-float32"), + ], +) +def test_kv_tile_width_is_gated_by_platform( + monkeypatch, is_rocm: bool, on_gfx1x: bool, dtype, expected_block_n: int +) -> None: + capture = _capture_tile_config( + monkeypatch, is_rocm=is_rocm, on_gfx1x=on_gfx1x, dtype=dtype + ) + assert capture.kwargs["BLOCK_N"] == expected_block_n + + +def test_rdna_narrows_the_kv_tile_and_nothing_else(monkeypatch) -> None: + with monkeypatch.context() as m: + tuned = _capture_tile_config(m, is_rocm=True, on_gfx1x=True) + with monkeypatch.context() as m: + stock = _capture_tile_config(m, is_rocm=True, on_gfx1x=False) + + assert tuned.kwargs.pop("BLOCK_N") != stock.kwargs.pop("BLOCK_N") + assert tuned.kwargs == stock.kwargs + assert tuned.grid == stock.grid diff --git a/tests/kernels/attention/test_triton_unified_attention.py b/tests/kernels/attention/test_triton_unified_attention.py index c3a2919fac..e30d9225ea 100644 --- a/tests/kernels/attention/test_triton_unified_attention.py +++ b/tests/kernels/attention/test_triton_unified_attention.py @@ -10,6 +10,7 @@ from vllm.triton_utils import tl, triton from vllm.utils.math_utils import next_power_of_2 from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.ops.triton_attention_helpers import ( + apply_softcap, compute_tile_loop_bounds, ) from vllm.v1.attention.ops.triton_unified_attention import unified_attention @@ -899,3 +900,40 @@ def test_triton_unified_attn_use_td_tile_clamp( soft_cap=None, seq_threshold_3D=0, ) + + +@triton.jit +def _softcap_probe_kernel( + scores_ptr, + out_ptr, + numel, + soft_cap, + BLOCK: tl.constexpr, +): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = offs < numel + scores = tl.load(scores_ptr + offs, mask=mask, other=0.0) + tl.store(out_ptr + offs, apply_softcap(scores, soft_cap), mask=mask) + + +@torch.inference_mode() +def test_softcap_does_not_overflow_on_large_scores() -> None: + """Scores above ~88 * soft_cap must not overflow to inf/NaN. + + The exp-based softcap computes ``(exp(y) - exp(-y)) / (exp(y) + exp(-y))`` + with ``y = S / soft_cap``. For ``|y| > ~88`` the exponentials overflow to + ``inf`` and the ratio becomes ``inf / inf = NaN``, poisoning the whole + attention row. Gemma-2 style models use ``attn_logit_softcapping = 50``, + so scores above 4400 are in range for their large attention logits. + """ + soft_cap = 50.0 + scores = torch.tensor( + [1.0, 100.0, 1000.0, 3000.0, 5000.0, 10000.0, -1.0, -10000.0, 4400.0], + device=DEVICE_TYPE, + dtype=torch.float32, + ) + out = torch.empty_like(scores) + _softcap_probe_kernel[(1,)](scores, out, scores.numel(), soft_cap, BLOCK=16) + ref = soft_cap * torch.tanh(scores / soft_cap) + assert torch.isfinite(out).all(), out + torch.testing.assert_close(out, ref, atol=1e-3, rtol=1e-3) diff --git a/tests/kernels/core/test_activation.py b/tests/kernels/core/test_activation.py index c1ba96a792..85ec4903ee 100644 --- a/tests/kernels/core/test_activation.py +++ b/tests/kernels/core/test_activation.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import random +from unittest.mock import patch import pytest import torch @@ -236,6 +237,10 @@ def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None: torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2) +@pytest.mark.parametrize( + ("alpha", "beta"), + [(1.0, 0.0), (1.702, 0.0), (1.0, 1.0)], +) @pytest.mark.parametrize("swiglu_limit", SWIGLU_LIMITS) @pytest.mark.parametrize("num_tokens", NUM_TOKENS) @pytest.mark.parametrize("d", D) @@ -245,6 +250,8 @@ def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None: @torch.inference_mode() def test_silu_and_mul_with_clamp( default_vllm_config, + alpha: float, + beta: float, swiglu_limit: float, num_tokens: int, d: int, @@ -258,8 +265,38 @@ def test_silu_and_mul_with_clamp( # Use large values to ensure clamping is exercised. x = torch.randn(num_tokens, 2 * d, dtype=dtype) * swiglu_limit * 2 - layer = SiluAndMulWithClamp(swiglu_limit, compile_native=False) - out = layer(x) + default_vllm_config.compilation_config.custom_ops = [ + "none", + "+silu_and_mul_with_clamp", + ] + layer = SiluAndMulWithClamp( + swiglu_limit, + alpha=alpha, + beta=beta, + compile_native=False, + ) + if current_platform.is_rocm(): + # forward_hip is always dispatched; the alpha/beta gate is checked + # inside it at call time rather than picked at construction time, so + # verify the actual routing by spying on the two candidate methods. + assert layer._forward_method == layer.forward_hip + with ( + patch.object(layer, "forward_cuda", wraps=layer.forward_cuda) as cuda_spy, + patch.object( + layer, "forward_native", wraps=layer.forward_native + ) as native_spy, + ): + out = layer(x) + if alpha == 1.0 and beta == 0.0: + cuda_spy.assert_called_once() + native_spy.assert_not_called() + else: + native_spy.assert_called_once() + cuda_spy.assert_not_called() + else: + assert layer._forward_method == layer.forward_cuda + out = layer(x) + ref_out = layer.forward_native(x) rtol = { @@ -273,10 +310,11 @@ def test_silu_and_mul_with_clamp( # Verify clamping is actually being applied: the clamped output should # differ from the unclamped SiluAndMul output when inputs are large. - unclamped_out = SiluAndMul.forward_native(x) - assert not torch.equal(ref_out.float(), unclamped_out.float()), ( - "Input was not large enough to exercise the clamp; increase scale" - ) + if alpha == 1.0 and beta == 0.0: + unclamped_out = SiluAndMul.forward_native(x) + assert not torch.equal(ref_out.float(), unclamped_out.float()), ( + "Input was not large enough to exercise the clamp; increase scale" + ) # Verify gate clamping semantics with a controlled scalar case. # gate=large_val is clamped to limit first, then silu(limit) * 1.0. @@ -309,7 +347,10 @@ def test_silu_and_mul_with_clamp( # opcheck out_buf = torch.empty(x.shape[:-1] + (d,), dtype=dtype, device=device) - opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit)) + opcheck( + torch.ops._C.silu_and_mul_with_clamp, + (out_buf, x, swiglu_limit, layer.alpha, layer.beta), + ) @pytest.mark.parametrize("linear_beta", [-1.0, 2.0]) diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index fd92de16bb..a576199663 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -184,3 +184,44 @@ def test_rotary_embedding_dispatch( - forward_cuda/forward methods should call ApplyRotaryEmb.forward() """ run_dispatch_test(test_case, device) + + +@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU only test.") +@pytest.mark.parametrize( + "num_tokens,num_heads,head_size,rot_dim,is_neox_style", + [ + (128, 32, 128, 128, True), # typical text shape + (4096, 16, 80, 80, False), # GPT-J style (interleaved) + (4096, 32, 128, 64, True), # partial rotary (rot_dim < head_size) + ], + ids=["text", "gptj_interleaved", "partial_rotary"], +) +def test_apply_rotary_emb_xpu_matches_native( + num_tokens: int, + num_heads: int, + head_size: int, + rot_dim: int, + is_neox_style: bool, +): + """ApplyRotaryEmb.forward_xpu (vllm_xpu_kernels SYCL op) must match + forward_native (the CustomOp default fallback) exactly.""" + from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb + + vllm_config = VllmConfig( + compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"]) + ) + get_cached_compilation_config.cache_clear() + + with set_current_vllm_config(vllm_config): + op = ApplyRotaryEmb(enforce_enable=True, is_neox_style=is_neox_style) + + x = torch.randn( + num_tokens, num_heads, head_size, device="xpu", dtype=torch.bfloat16 + ) + x = x[..., :rot_dim].contiguous() + cos = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16) + sin = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16) + + expected = op.forward_native(x, cos, sin) + actual = op.forward_xpu(x, cos, sin) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) diff --git a/tests/kernels/core/test_fused_q_kv_rmsnorm.py b/tests/kernels/core/test_fused_q_kv_rmsnorm.py index 3d0e2ac11e..485a35348e 100644 --- a/tests/kernels/core/test_fused_q_kv_rmsnorm.py +++ b/tests/kernels/core/test_fused_q_kv_rmsnorm.py @@ -223,7 +223,6 @@ def test_shared_query_quant_preserves_projection( with set_current_vllm_config( VllmConfig(compilation_config=CompilationConfig(mode=CompilationMode.NONE)) ): - kernel = cls(Mxfp8LinearLayerConfig()) # Match the checkpoint's TP4 attention and replicated indexer widths. if projection == "attention": linear = ColumnParallelLinear.__new__(ColumnParallelLinear) @@ -247,6 +246,7 @@ def test_shared_query_quant_preserves_projection( ) linear.weight = torch.nn.Parameter(w, requires_grad=False) linear.weight_scale = torch.nn.Parameter(ws, requires_grad=False) + kernel = cls(Mxfp8LinearLayerConfig(weight_shape=w.shape)) quant_config = DeepseekV4FP8Config.from_config( { "quant_method": "fp8", diff --git a/tests/kernels/core/test_fused_quant_layernorm.py b/tests/kernels/core/test_fused_quant_layernorm.py index 201830fe2f..7a6e53314f 100644 --- a/tests/kernels/core/test_fused_quant_layernorm.py +++ b/tests/kernels/core/test_fused_quant_layernorm.py @@ -318,10 +318,13 @@ def test_rms_norm( # Per-block bf16 scales: allow a small relative tolerance for a few groups # whose abs-max flips by one ULP between the fused and reference paths. The # per-token and fp32 paths stay strict. - relax_block_rocm = ( + # The same one-ULP group-scale flip also occurs on CUDA (H200), so extend + # the block relaxation there too — the fused groupwise reduction rounds + # differently than the reference for isolated groups. + relax_block = ( group_size is not None and dtype == torch.bfloat16 - and current_platform.is_rocm() + and (current_platform.is_rocm() or current_platform.is_cuda()) ) use_gfx950_fp8_allclose = ( current_platform.is_rocm() @@ -340,7 +343,7 @@ def test_rms_norm( def scales_close(rtol: float, atol: float) -> bool: if torch.allclose(ref_scales, ops_scales, rtol=rtol, atol=atol): return True - return relax_block_rocm and torch.allclose( + return relax_block and torch.allclose( ref_scales, ops_scales, rtol=1e-2, atol=atol ) @@ -354,7 +357,7 @@ def test_rms_norm( b = ops_out.to(dtype=torch.float32) ok = torch.allclose(a, b, atol=1e-6) if not ok: - if relax_block_rocm: + if relax_block: # ULP-flipped group scale can cross an E4M3 tie; tolerate a # bounded count of isolated fp8 outliers. ulp = fp8_ulp_distance(ref_out, ops_out) diff --git a/tests/kernels/core/test_mrope.py b/tests/kernels/core/test_mrope.py index 0e355d7cd3..e590d8264c 100644 --- a/tests/kernels/core/test_mrope.py +++ b/tests/kernels/core/test_mrope.py @@ -6,7 +6,10 @@ import pytest import torch from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.rotary_embedding.mrope import apply_interleaved_rope +from vllm.model_executor.layers.rotary_embedding.mrope import ( + apply_interleaved_rope, + triton_mrope, +) from vllm.platforms import current_platform from vllm.transformers_utils.config import get_config from vllm.utils.torch_utils import set_random_seed @@ -174,6 +177,75 @@ def test_mrope( torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol) torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol) + # Strided q/k views of a packed buffer (the MiniMax M3 ViT usage): + # bitwise-identical to the contiguous run, neighboring columns untouched. + pad = 16 + packed = torch.zeros( + num_tokens, + query.shape[1] + key.shape[1] + pad, + dtype=dtype, + device=device, + ) + packed[:, : query.shape[1]] = query + packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key + sentinel = packed[:, -pad:].clone() + q_strided = packed[:, : query.shape[1]] + k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]] + mrope_helper_class.forward_cuda(positions, q_strided, k_strided) + torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0) + torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0) + torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests." +) +def test_mrope_row_offset_int64(): + # MiniMax M3 ViT worst case: a max-size video item packs ~768k patches, + # and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241 + # rows. Row-base math must be int64 or those rows wrap to negative + # offsets (illegal memory access at startup profiling). + set_random_seed(0) + n_qh, n_kh, hd, rd = 16, 16, 80, 78 + mrope_section = [13, 13, 13] + row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row + num_tokens = 2**31 // row_stride + 8 + free, _ = current_platform.mem_get_info() + if free < 2 * num_tokens * row_stride * 2: + pytest.skip("not enough free device memory") + + packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device) + q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device) + k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device) + packed[:, : n_qh * hd] = q_fill + packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill + half_rd = rd // 2 + cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device) + sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device) + q = packed[:, : n_qh * hd] + k = packed[:, n_qh * hd : (n_qh + n_kh) * hd] + triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True) + + t_end = mrope_section[0] + h_end = t_end + mrope_section[1] + for r in (0, num_tokens - 8, num_tokens - 1): + cos_row = torch.cat( + [cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]] + ) + sin_row = torch.cat( + [sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]] + ) + for fill, out in ((q_fill, q[r]), (k_fill, k[r])): + row = fill.view(-1, hd).float() + r1, r2 = row[:, :half_rd], row[:, half_rd:rd] + exp = torch.cat( + [r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]], + dim=1, + ) + torch.testing.assert_close( + out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2 + ) + @pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests." diff --git a/tests/kernels/fla/__init__.py b/tests/kernels/fla/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/kernels/test_fla_layernorm_guard.py b/tests/kernels/fla/test_fla_layernorm_guard.py similarity index 100% rename from tests/kernels/test_fla_layernorm_guard.py rename to tests/kernels/fla/test_fla_layernorm_guard.py diff --git a/tests/kernels/test_fused_gdn_post_conv.py b/tests/kernels/fla/test_fused_gdn_post_conv.py similarity index 100% rename from tests/kernels/test_fused_gdn_post_conv.py rename to tests/kernels/fla/test_fused_gdn_post_conv.py diff --git a/tests/kernels/test_fused_recurrent_packed_decode.py b/tests/kernels/fla/test_fused_recurrent_packed_decode.py similarity index 100% rename from tests/kernels/test_fused_recurrent_packed_decode.py rename to tests/kernels/fla/test_fused_recurrent_packed_decode.py diff --git a/tests/kernels/test_fused_sigmoid_gating_delta_rule.py b/tests/kernels/fla/test_fused_sigmoid_gating_delta_rule.py similarity index 100% rename from tests/kernels/test_fused_sigmoid_gating_delta_rule.py rename to tests/kernels/fla/test_fused_sigmoid_gating_delta_rule.py diff --git a/tests/kernels/helion/test_benchmark_script.py b/tests/kernels/helion/test_benchmark_script.py index a42eea00c4..c593412843 100644 --- a/tests/kernels/helion/test_benchmark_script.py +++ b/tests/kernels/helion/test_benchmark_script.py @@ -9,7 +9,7 @@ from vllm.utils.import_utils import has_helion if not has_helion(): pytest.skip("Helion is not installed", allow_module_level=True) -from scripts import benchmark_helion_kernels +from tools import benchmark_helion_kernels check_correctness = benchmark_helion_kernels.check_correctness diff --git a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py index c239f831c6..3b662479b1 100644 --- a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py +++ b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py @@ -473,6 +473,20 @@ def _conv_inputs(total_tokens: int): return x, weight, bias +def _conv_states( + num_slots: int, + state_len: int, + dim: int = CONV_DIM, + layout: str = "SD", +) -> torch.Tensor: + if layout == "SD": + return torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16).transpose( + 1, 2 + ) + assert layout == "DS" + return torch.zeros(num_slots, dim, state_len, dtype=torch.bfloat16) + + def _sd_conv_states( num_slots: int, state_len: int, dim: int = CONV_DIM ) -> torch.Tensor: @@ -484,6 +498,39 @@ def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight +@torch.inference_mode() +def test_causal_conv1d_update_cpu_preserves_sd_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """SD state and indices reach the native operator unchanged.""" + num_slots = 2 + state_len = CONV_KERNEL - 1 + conv_state = _sd_conv_states(num_slots, state_len) + state_indices = torch.tensor([1], dtype=torch.int32) + x = torch.zeros(1, CONV_DIM, dtype=torch.bfloat16) + recorded = {} + + def record_update(**kwargs): + recorded.update(kwargs) + return x + + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False) + monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_update_cpu", record_update) + out = gdn_attention._causal_conv1d_update_cpu( + x=x, + conv_states=conv_state, + weight=torch.zeros(CONV_DIM, CONV_KERNEL, dtype=torch.bfloat16), + bias=None, + silu_activation=False, + conv_state_indices=state_indices, + is_vnni=False, + ) + + assert out is x + assert recorded["conv_states"] is conv_state + assert recorded["conv_state_indices"] is state_indices + + @torch.inference_mode() def test_spec_aware_mixed_routing_preserves_token_order( monkeypatch: pytest.MonkeyPatch, @@ -1024,7 +1071,18 @@ def test_batch_memcpy_cpu_fallback() -> None: copy each src into its dst, validating the (src_ptrs, dst_ptrs, sizes) argument order against ctypes.memmove(dst, src, size). """ - from vllm.utils.cpu_triton_utils import batch_memcpy_kernel + from vllm.triton_utils.dispatcher import ( + KernelOverride, + register_kernels, + ) + from vllm.utils.cpu_triton_utils import _batch_memcpy_impl + + register_kernels( + {"vllm.v1.worker.mamba_utils.batch_memcpy_kernel": _batch_memcpy_impl} + ) + from vllm.v1.worker.mamba_utils import batch_memcpy_kernel + + assert isinstance(batch_memcpy_kernel, KernelOverride) # Varied byte sizes, including a non-power-of-two run. sizes_bytes = [256, 1024, 17 * 4, 4096] @@ -1123,6 +1181,145 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False): return out.transpose(0, 1).contiguous(), conv_state +def _python_native_fwd(**kwargs): + from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( + causal_conv1d_fn_cpu, + ) + + return causal_conv1d_fn_cpu( + x=kwargs["x"], + weight=kwargs["weight"], + bias=kwargs["bias"], + conv_states=kwargs["conv_states"], + query_start_loc=kwargs["query_start_loc"], + cache_indices=kwargs["cache_indices"], + has_initial_state=kwargs["has_initial_state"], + activation=kwargs["silu_activation"], + ) + + +@torch.inference_mode() +def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + dim = 8 + seq_lens = [2, 3] + state_len = _STATE_LEN + state_indices = torch.tensor([2, 0], dtype=torch.int32) + query_start_loc = torch.tensor([0, 2, 5], dtype=torch.int32) + has_initial_state = torch.tensor([False, True]) + x_tokens, weight, bias = _conv_inputs(sum(seq_lens)) + x_tokens = x_tokens[:, :dim] + weight = weight[:dim] + bias = bias[:dim] + x = x_tokens.transpose(0, 1).contiguous() + logical_state = ( + torch.arange(3 * dim * state_len, dtype=torch.float32) + .view(3, dim, state_len) + .to(torch.bfloat16) + ) + state_sd = _conv_states(3, state_len, dim, "SD") + state_ds = _conv_states(3, state_len, dim, "DS") + state_sd.copy_(logical_state) + state_ds.copy_(logical_state) + recorded = [] + + def record_fwd(**kwargs): + recorded.append( + ( + kwargs["conv_states"].shape, + kwargs["conv_states"].stride(), + kwargs["cache_indices"].clone(), + ) + ) + return _python_native_fwd(**kwargs) + + monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_fwd_cpu", record_fwd) + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False) + out_sd = gdn_attention._causal_conv1d_fwd_cpu( + x=x, + weight=weight, + bias=bias, + conv_states=state_sd, + query_start_loc=query_start_loc, + cache_indices=state_indices, + has_initial_state=has_initial_state, + silu_activation=True, + is_vnni=False, + ) + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True) + out_ds = gdn_attention._causal_conv1d_fwd_cpu( + x=x, + weight=weight, + bias=bias, + conv_states=state_ds, + query_start_loc=query_start_loc, + cache_indices=state_indices, + has_initial_state=has_initial_state, + silu_activation=True, + is_vnni=False, + ) + + torch.testing.assert_close(out_ds, out_sd) + torch.testing.assert_close(state_ds, state_sd) + assert recorded[0][2].equal(state_indices) + assert recorded[1][0] == (2, dim, state_len) + assert recorded[1][1] == (dim * state_len, 1, dim) + assert recorded[1][2].equal(torch.arange(2, dtype=torch.int32)) + + +@torch.inference_mode() +def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + dim = 8 + seq_len = 4 + state_len = seq_len + 5 + state = _conv_states(4, state_len, dim, "DS") + state.copy_( + torch.arange(state.numel(), dtype=torch.float32).view_as(state).to(state.dtype) + ) + original_state = state.clone() + state_indices = torch.tensor([3, 1, 0], dtype=torch.int32) + num_accepted_tokens = torch.tensor([1, 4, 2], dtype=torch.int32) + mutated_scratch = torch.arange( + state_indices.numel() * dim * state_len, dtype=torch.float32 + ).view(state_indices.numel(), dim, state_len) + mutated_scratch = (mutated_scratch + 200).to(state.dtype) + recorded = {} + + def mutate_update(**kwargs): + recorded.update(kwargs) + kwargs["conv_states"].copy_(mutated_scratch) + return torch.zeros_like(kwargs["x"]) + + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True) + monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_update_cpu", mutate_update) + gdn_attention._causal_conv1d_update_cpu( + x=torch.zeros(state_indices.numel(), seq_len, dim, dtype=state.dtype), + conv_states=state, + weight=torch.zeros(dim, CONV_KERNEL, dtype=state.dtype), + bias=None, + silu_activation=False, + conv_state_indices=state_indices, + is_vnni=False, + num_accepted_tokens=num_accepted_tokens, + ) + + expected_state = original_state.clone() + expected_state.index_copy_(0, state_indices.to(torch.long), mutated_scratch) + torch.testing.assert_close(state, expected_state) + torch.testing.assert_close( + state.index_select(0, state_indices)[:, :, _STATE_LEN:], + mutated_scratch[:, :, _STATE_LEN:], + ) + torch.testing.assert_close( + recorded["conv_state_indices"], + torch.arange(state_indices.numel(), dtype=torch.int32), + ) + torch.testing.assert_close(recorded["num_accepted_tokens"], num_accepted_tokens) + + @pytest.mark.skipif( not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" ) diff --git a/tests/kernels/mamba/test_causal_conv1d.py b/tests/kernels/mamba/test_causal_conv1d.py index d4f5fac59c..ddd532ad13 100644 --- a/tests/kernels/mamba/test_causal_conv1d.py +++ b/tests/kernels/mamba/test_causal_conv1d.py @@ -120,37 +120,6 @@ def causal_conv1d_update_ref( return (out if activation is None else F.silu(out)).to(dtype=dtype_in) -@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float]) -@pytest.mark.parametrize("silu_activation", [True]) -@pytest.mark.parametrize("has_bias", [True]) -def causal_conv1d_opcheck_fn( - x: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor | None = None, - cu_seq_len: torch.Tensor | None = None, - cache_indices: torch.Tensor | None = None, - has_initial_state: torch.Tensor | None = None, - conv_states: torch.Tensor | None = None, - activation: str | None = "silu", - null_block_id: int = NULL_BLOCK_ID, -): - """x: (batch, dim, seqlen) - weight: (dim, width) - bias: (dim,) - seq_idx: (batch, seqlen) - initial_states: (batch, dim, width - 1) - final_states_out: (batch, dim, width - 1), to be written to - activation: either None or "silu" or "swish" - - out: (batch, dim, seqlen) - """ - if activation not in [None, "silu", "swish"]: - raise NotImplementedError("activation must be None, silu, or swish") - if x.stride(-1) != 1: - x = x.contiguous() - bias = bias.contiguous() if bias is not None else None - - @pytest.mark.parametrize("itype", [torch.bfloat16]) @pytest.mark.parametrize("silu_activation", [False, True]) @pytest.mark.parametrize("has_bias", [False, True]) diff --git a/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py index ae34ffcb80..851d6fb1d3 100644 --- a/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py +++ b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py @@ -6,9 +6,9 @@ The AITER fused reshape+conv kernel learned Qwen3.5's flat ``[q|k|v|z]`` packing in https://github.com/ROCm/aiter/pull/3251, so flat-layout models take the decode fast path too instead of falling back to the generic path. -These tests run host-side on CPU: ``_forward_core_rocm`` is bound to a stub -layer whose ``_forward_core_decode_aiter``/``_forward_core`` record which -branch ran, so no GPU or AITER install is needed. +The dispatch tests run host-side with recording stubs. The prefill numerical +tests exercise the production ChunkGatedDeltaRule wrapper on ROCm against a +token-wise Torch reference, including ragged state and output-buffer layouts. """ from __future__ import annotations @@ -122,3 +122,130 @@ def test_non_pure_decode_batches_use_the_generic_path( ) -> None: layer = _make_layer(gqa_interleaved_layout) assert _run(layer, _make_metadata(**meta_kwargs)) == "generic" + + +def _prefill_reference(q, k, v, g, beta, initial_state, lengths): + """Independent token-wise recurrence in the layer's [value, key] layout.""" + q, k, v, g, beta, initial_state = ( + x.cpu().float() for x in (q, k, v, g, beta, initial_state) + ) + q = q.repeat_interleave(v.shape[2] // q.shape[2], dim=2) + k = k.repeat_interleave(v.shape[2] // k.shape[2], dim=2) + q = q * q.shape[-1] ** -0.5 + output = torch.empty_like(v) + final_state = torch.empty_like(initial_state) + previous_threads = torch.get_num_threads() + torch.set_num_threads(1) + try: + start = 0 + for sequence, length in enumerate(lengths): + state = initial_state[sequence].clone() + for token in range(start, start + length): + state = state * g[0, token].exp()[:, None, None] + key = k[0, token] + prediction = (state * key[:, None, :]).sum(dim=-1) + delta = (v[0, token] - prediction) * beta[0, token, :, None] + state = state + delta[:, :, None] * key[:, None, :] + output[0, token] = (state * q[0, token, :, None, :]).sum(dim=-1) + final_state[sequence] = state + start += length + finally: + torch.set_num_threads(previous_threads) + return output, final_state + + +@pytest.mark.parametrize( + "state_dtype,correlated", + [(torch.bfloat16, False), (torch.float32, False), (torch.float32, True)], + ids=["ragged-bf16-state", "ragged-fp32-state", "correlated-keys"], +) +@torch.inference_mode() +def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated): + from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + ChunkGatedDeltaRule, + ) + from vllm.third_party.flash_linear_attention.ops.index import ( + prepare_chunk_indices, + prepare_chunk_offsets, + ) + + if not torch.cuda.is_available(): + pytest.skip("ROCm prefill numerics require a GPU") + # Only the config is needed for production backend selection; no weights. + config = VllmConfig( + model_config=ModelConfig( + model="Qwen/Qwen3.5-0.8B", + revision="2fc06364715b967f1860aea9cf38778875588b17", + max_model_len=1024, + ) + ) + with set_current_vllm_config(config): + prefill = ChunkGatedDeltaRule() + assert prefill.gdn_prefill_backend == "triton" + assert prefill._forward_method == prefill.forward_native + + torch.manual_seed(1234) + lengths = [64] if correlated else [1, 63, 64, 65, 129] + tokens, key_heads, value_heads, dim = sum(lengths), 4, 8, 128 + q = torch.randn(1, tokens, key_heads, dim, device="cuda") + q = torch.nn.functional.normalize(q, dim=-1).to(torch.bfloat16) + k = torch.randn_like(q) + k = torch.nn.functional.normalize(k.float(), dim=-1).to(q.dtype) + v = torch.randn(1, tokens, value_heads, dim, device="cuda", dtype=q.dtype) + g = -torch.rand(1, tokens, value_heads, device="cuda") * 0.1 + beta = torch.rand_like(g) + state = (torch.randn(len(lengths), value_heads, dim, dim, device="cuda") * 0.05).to( + state_dtype + ) + if correlated: + q.zero_() + k.zero_() + q[..., 0] = 1 + k[..., 0] = 1 + g.zero_() + beta.fill_(0.9) + state.zero_() + expected, expected_state = _prefill_reference(q, k, v, g, beta, state, lengths) + initial_state_copy = state.clone() + cu_seqlens = torch.tensor( + [0, *torch.tensor(lengths).cumsum(0).tolist()], + device="cuda", + dtype=torch.int32, + ) + kwargs = dict( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=state, + output_final_state=True, + cu_seqlens=cu_seqlens, + chunk_indices=prepare_chunk_indices(cu_seqlens, 64), + chunk_offsets=prepare_chunk_offsets(cu_seqlens, 64), + use_qk_l2norm_in_kernel=False, + ) + storage = torch.full( + (tokens + 3, value_heads, dim), float("nan"), device="cuda", dtype=q.dtype + ) + output, final_state = prefill(**kwargs, core_attn_out=storage[:tokens]) + assert output.data_ptr() == storage.data_ptr() + assert output.shape == v.shape + assert final_state.shape == state.shape + assert torch.isnan(storage[tokens:]).all() + torch.testing.assert_close(state, initial_state_copy, atol=0, rtol=0) + + # Preserve the upstream prefill suite's numerical budgets. The reference + # here is a token-wise recurrence, independent of the chunk implementation. + output_error = (output.cpu().float() - expected).abs() + state_error = (final_state.cpu().float() - expected_state).abs() + assert torch.isfinite(output).all() and torch.isfinite(final_state).all() + assert output_error.max() < (1e-1 if correlated else 2e-3) + assert output_error.mean() < (2e-3 if correlated else 6e-5) + assert state_error.max() < (2e-1 if correlated else 2e-2) + assert state_error.mean() < 6e-4 + + no_buffer_output, no_buffer_state = prefill(**kwargs) + torch.testing.assert_close(no_buffer_output, output, atol=0, rtol=0) + torch.testing.assert_close(no_buffer_state, final_state, atol=0, rtol=0) diff --git a/tests/kernels/mhc/__init__.py b/tests/kernels/mhc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/kernels/test_mhc_jit_warmup.py b/tests/kernels/mhc/test_mhc_jit_warmup.py similarity index 100% rename from tests/kernels/test_mhc_jit_warmup.py rename to tests/kernels/mhc/test_mhc_jit_warmup.py diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/mhc/test_mhc_kernels.py similarity index 100% rename from tests/kernels/test_mhc_kernels.py rename to tests/kernels/mhc/test_mhc_kernels.py diff --git a/tests/kernels/test_mhc_tilelang_jit.py b/tests/kernels/mhc/test_mhc_tilelang_jit.py similarity index 97% rename from tests/kernels/test_mhc_tilelang_jit.py rename to tests/kernels/mhc/test_mhc_tilelang_jit.py index 1c430e7ae3..4292513d25 100644 --- a/tests/kernels/test_mhc_tilelang_jit.py +++ b/tests/kernels/mhc/test_mhc_tilelang_jit.py @@ -108,7 +108,7 @@ def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols() -> None: # Both claims are about process-global state, `sys.modules` and the symbol # table, and a sibling test legitimately imports TileLang to exercise those # kernels, so the checks only mean something in an interpreter of their own. - script = Path(__file__).parent / "scripts" / "check_no_tilelang_hijack.py" + script = Path(__file__).parents[1] / "scripts" / "check_no_tilelang_hijack.py" result = subprocess.run( [sys.executable, str(script)], capture_output=True, diff --git a/tests/kernels/moe/modular_kernel_tools/common.py b/tests/kernels/moe/modular_kernel_tools/common.py index 3ec522c8ec..1944aaac57 100644 --- a/tests/kernels/moe/modular_kernel_tools/common.py +++ b/tests/kernels/moe/modular_kernel_tools/common.py @@ -84,6 +84,8 @@ class Config: world_size: int + activation: MoEActivation = MoEActivation.SILU + torch_trace_dir_path: str | None = None def __post_init__(self): @@ -311,6 +313,16 @@ class Config: f"block={self.quant_block_shape})" ) + # Check activation support; NotImplementedError means no opinion. + try: + if not self.fused_experts_type._supports_activation(self.activation): + return False, ( + f"FE {self.fused_experts_type.__name__} does not support " + f"activation {self.activation}" + ) + except NotImplementedError: + pass + # Check block quantization support is_block_quantized = self.quant_block_shape is not None if is_block_quantized and self.quant_dtype is None: @@ -605,6 +617,7 @@ def reference_moe_impl( topk_ids=rank_tensors.topk_ids, global_num_experts=config.E, expert_map=None, + activation=config.activation, w1_scale=w1_scale, w2_scale=w2_scale, a1_scale=a_scale, @@ -648,7 +661,7 @@ def make_modular_kernel( moe_parallel_config=moe_parallel_config, in_dtype=config.dtype, max_num_tokens=next_power_of_2(config.M), - activation=MoEActivation.SILU, + activation=config.activation, device=vllm_config.device_config.device, routing_method=RoutingMethodType.DeepSeekV3, ) @@ -774,7 +787,7 @@ def run_modular_kernel( "w2": rank_weights.w2, "topk_weights": rank_tensors.topk_weights, "topk_ids": topk_ids, - "activation": MoEActivation.SILU, + "activation": config.activation, "expert_map": rank_tensors.expert_map, "global_num_experts": config.E, "apply_router_weight_on_input": config.topk == 1 diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index e0f37f5436..6ca644c5fc 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -448,24 +448,6 @@ if cutlass_fp4_supported() or has_flashinfer_cutlass_fused_moe(): ] -def _slice(rank: int, num_local_experts: int, t: torch.Tensor) -> torch.Tensor: - s = rank * num_local_experts - e = s + num_local_experts - return t[s:e] - - -def make_cutlass_strides( - e: int, - n: int, - k: int, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - ab_strides1 = torch.full((e,), k, device="cuda", dtype=torch.int64) - ab_strides2 = torch.full((e,), n, device="cuda", dtype=torch.int64) - c_strides1 = torch.full((e,), 2 * n, device="cuda", dtype=torch.int64) - c_strides2 = torch.full((e,), k, device="cuda", dtype=torch.int64) - return ab_strides1, ab_strides2, c_strides1, c_strides2 - - def make_fused_experts( fused_experts_type: mk.FusedMoEExpertsModular, moe: FusedMoEConfig, diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 63f64c735a..68f6b3b0b3 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -107,7 +107,16 @@ SEEDS = [0] def torch_w8a8_block_fp8_moe( - a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False + a, + w1, + w2, + w1_s, + w2_s, + topk_weight, + topk_ids, + block_shape, + silu_fp32=False, + expert_map=None, ): """Fused MoE with block-wise fp8 quantization using native torch. @@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe( topk_weight = topk_weight.view(-1) topk_ids = topk_ids.view(-1) + if expert_map is not None: + # w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing. + topk_ids = expert_map[topk_ids] _, block_k = block_shape[0], block_shape[1] # Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the @@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant( ) +@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)]) +@pytest.mark.parametrize("topk", [2, 6]) +@pytest.mark.parametrize("ep_size", [2, 4]) +@torch.inference_mode() +def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init): + """One EP rank: local expert shard plus an expert_map with -1 entries.""" + E, block_size, dtype = 16, [128, 128], torch.bfloat16 + torch.manual_seed(0) + + a = torch.randn((M, K), dtype=dtype) / 10 + score = torch.randn((M, E), dtype=dtype) + w1, w2, quant_config = make_test_quant_config( + E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size + ) + + local_e = E // ep_size + e_ids = torch.randperm(E, dtype=torch.int32)[:local_e] + e_map = torch.full((E,), -1, dtype=torch.int32) + e_map[e_ids] = torch.arange(local_e, dtype=torch.int32) + w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous() + w1_s = quant_config.w1_scale[e_ids].contiguous() + w2_s = quant_config.w2_scale[e_ids].contiguous() + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size + ) + + m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config) + topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False) + + with set_current_vllm_config(vllm_config): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map + ) + m_out = m_fused_moe.apply( + a, + w1, + w2, + topk_weights, + topk_ids, + activation=MoEActivation.SILU, + apply_router_weight_on_input=False, + expert_map=e_map, + global_num_experts=E, + ) + + torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035) + + @pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG) @pytest.mark.parametrize("E", E) @pytest.mark.parametrize("topk", TOP_KS) diff --git a/tests/kernels/moe/test_block_int8.py b/tests/kernels/moe/test_block_int8.py index 415faafb3a..5e853834bb 100644 --- a/tests/kernels/moe/test_block_int8.py +++ b/tests/kernels/moe/test_block_int8.py @@ -12,6 +12,9 @@ from tests.kernels.quant_utils import ( from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk +from vllm.model_executor.layers.quantization.utils.int8_utils import ( + per_token_group_quant_int8, +) from vllm.platforms import current_platform if current_platform.get_device_capability() < (7, 0): @@ -64,7 +67,7 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): topk_ids = topk_ids.view(-1) _, block_k = block_shape[0], block_shape[1] - a_q, a_s = native_per_token_group_quant_int8(a, block_k) + a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False) for i in range(w1.shape[0]): mask = topk_ids == i if mask.sum(): @@ -72,7 +75,9 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype ) act_out = SiluAndMul().forward_native(inter_out) - act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k) + act_out_q, act_out_s = native_per_token_group_quant_int8( + act_out, block_k, round_to_nearest=False + ) act_out = act_out.to(torch.float32) out[mask] = native_w8a8_block_matmul( act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype @@ -132,3 +137,27 @@ def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed): # Check results torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +def test_block_int8_quantization_truncates_fractional_values(dtype): + """Block INT8 quantization truncates, unlike per-token round-to-nearest.""" + device = current_platform.device_type + x = torch.tensor( + [[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122], + dtype=dtype, + device=device, + ) + expected = torch.tensor( + [[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device + ) + + out_q, scale = per_token_group_quant_int8(x, group_size=128) + ref_q, ref_s = native_per_token_group_quant_int8( + x, group_size=128, round_to_nearest=False + ) + + torch.testing.assert_close(out_q, expected, atol=0, rtol=0) + torch.testing.assert_close(ref_q, expected, atol=0, rtol=0) + torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0) + torch.testing.assert_close(ref_s, scale, atol=0, rtol=0) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 9189c5c641..5d55876e81 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -1,14 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing -from dataclasses import dataclass +from dataclasses import dataclass, replace +from types import SimpleNamespace from unittest.mock import MagicMock import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from tests.kernels.moe.utils import make_dummy_moe_config +from tests.kernels.moe.utils import ( + check_deferred_moe_finalize, + make_dummy_moe_config, + make_test_weights, +) from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.all2all_utils import ( @@ -32,11 +37,20 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( TrtLlmFp8ExpertsMonolithic, ) from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts +from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( + Fp8MoeBackend, + convert_to_fp8_moe_kernel_format, + make_fp8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( rotate_weights_for_fi_trtllm_fp8_per_tensor_moe, swap_w13_to_w31, ) from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8 +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + mxfp8_e4m3_quantize, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kFp8Dynamic128Sym, @@ -623,6 +637,9 @@ def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed( def _check_flashinfer_ipc_weights(entries, expected, mode, device_index): from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader + from vllm.model_executor.model_loader.weight_cache.protocol import ( + WeightCacheState, + ) from vllm.model_executor.utils import weights_already_processed torch.accelerator.set_device_index(device_index) @@ -632,7 +649,7 @@ def _check_flashinfer_ipc_weights(entries, expected, mode, device_index): ) loader = object.__new__(IpcModelLoader) loader.mode = mode - loader._apply_entries(layer, entries, {}, device_index) + loader._apply_entries(layer, WeightCacheState(entries, {}, {}), device_index) pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr()) # Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel. @@ -764,3 +781,109 @@ def test_trtllm_fp8_swiglu_clamp_support( assert supported == expected, reason if not expected: assert "SwiGLU" in reason + + +def _make_mxfp8_moe_weights( + e: int, n: int, k: int +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + q, s = zip( + *(mxfp8_e4m3_quantize(w[i], is_sf_swizzled_layout=False) for i in range(e)) + ) + return torch.stack(q), torch.stack(s) + + w1, w1_scale = quantize( + torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10 + ) + w2, w2_scale = quantize( + torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10 + ) + return w1, w2, w1_scale, w2_scale + + +@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"]) +@pytest.mark.parametrize("m", [1, 16]) +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), + reason="Requires TRTLLM-Gen FP8 MoE (SM100)", +) +def test_trtllm_fp8_block_moe_deferred_finalize( + m: int, block_shape: list[int], workspace_init +): + """TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k + finalize to the caller.""" + e, topk, n, k = 32, 4, 1024, 1024 + set_random_seed(7) + with set_current_vllm_config(vllm_config): + if block_shape == [1, 32]: + w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k) + else: + (_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights( + e, n, k, quant_dtype=torch.float8_e4m3fn, block_shape=block_shape + ) + w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format( + Fp8MoeBackend.FLASHINFER_TRTLLM, + SimpleNamespace( + weight_block_size=block_shape, + moe_config=SimpleNamespace( + is_act_and_mul=True, intermediate_size_per_partition=n + ), + activation=MoEActivation.SILU, + ), + w1, + w2, + w1_scale, + w2_scale, + w13_input_scale=None, + w2_input_scale=None, + ) + quant_config = make_fp8_moe_quant_config( + Fp8MoeBackend.FLASHINFER_TRTLLM, + w1_scale, + w2_scale, + a1_scale=None, + a2_scale=None, + block_shape=block_shape, + ) + # One rank of a TP group, which deferral needs. + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=2 * n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=replace( + FusedMoEParallelConfig.make_no_parallel(), tp_size=2 + ), + in_dtype=torch.bfloat16, + routing_method=RoutingMethodType.TopK, + max_num_tokens=next_power_of_2(m), + ) + kernel = mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, quant_config=quant_config, allow_new_interface=True + ), + TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config), + ) + + a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10 + score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False) + check_deferred_moe_finalize( + moe_config, + lambda: kernel.apply( + hidden_states=a, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ), + router_weights=topk_weights, + ) diff --git a/tests/kernels/moe/test_modular_kernel_combinations.py b/tests/kernels/moe/test_modular_kernel_combinations.py index fc1d8ac07b..253abc5c38 100644 --- a/tests/kernels/moe/test_modular_kernel_combinations.py +++ b/tests/kernels/moe/test_modular_kernel_combinations.py @@ -2,19 +2,24 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +import inspect import textwrap import traceback from itertools import product from typing import Any +from unittest import mock import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe -from vllm.utils.import_utils import has_deep_ep, has_deep_gemm +from vllm.utils.import_utils import has_aiter, has_deep_ep, has_deep_gemm from vllm.utils.torch_utils import set_random_seed from vllm.v1.worker.workspace import init_workspace_manager @@ -71,7 +76,7 @@ def assert_aiter_quant_scheme_case(config: Config) -> None: indirectly through the general quant-config sweep. See https://github.com/vllm-project/vllm/issues/54966.""" fe_cls = config.fused_experts_type - if getattr(fe_cls, "__name__", "") != "AiterExperts": + if fe_cls is not AiterExperts: return if config.quant_config is None: @@ -87,6 +92,22 @@ def assert_aiter_quant_scheme_case(config: Config) -> None: print(f"[AITER case] weight_key={w_key}, activation_key={a_key}") +def assert_aiter_activation_case(config: Config) -> None: + """Make the AITER activation this config exercises explicit, instead of + AiterExperts being reached only indirectly through the general + activation sweep. See https://github.com/vllm-project/vllm/issues/54966.""" + fe_cls = config.fused_experts_type + if fe_cls is not AiterExperts: + return + + assert fe_cls._supports_activation(config.activation), ( + f"AITER case activation={config.activation} reached the " + "modular-kernel harness, but AiterExperts._supports_activation " + "does not declare it supported." + ) + print(f"[AITER case] activation={config.activation}") + + def rank_worker( pgi: ProcessGroupInfo, vllm_config: VllmConfig, @@ -131,7 +152,7 @@ def rank_worker( if ( topk == 1 and config.supports_apply_weight_on_input() - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and config.quant_block_shape is not None ): print( @@ -147,7 +168,7 @@ def rank_worker( # https://github.com/vllm-project/vllm/issues/57029 if ( config.world_size > 1 - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and getattr(config.prepare_finalize_type, "__name__", "") in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize") ): @@ -160,6 +181,7 @@ def rank_worker( continue assert_aiter_quant_scheme_case(config) + assert_aiter_activation_case(config) # modular kernel out mk_out = run_modular_kernel(pgi, vllm_config, config, weights, rank_tensors) @@ -184,7 +206,7 @@ def rank_worker( is_aiter_fp8 = ( _cp.is_rocm() - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and config.quant_config is not None ) if is_aiter_fp8: @@ -225,6 +247,10 @@ Ns = [1024] TOPKs = [4, 1] Es = [32] DTYPEs = [torch.bfloat16] +MK_ACTIVATIONS = [ + MoEActivation.SILU, + MoEActivation.GELU, +] def is_nyi_config(config: Config) -> bool: @@ -236,7 +262,14 @@ def is_nyi_config(config: Config) -> bool: unsupported_quant_config = ( config.is_per_act_token_quant + config.is_per_out_ch_quant ) == 1 - return unsupported_quant_config + if unsupported_quant_config: + return True + + if config.activation != MoEActivation.SILU: + if config.fused_experts_type is not AiterExperts: + return True # AITER-only for this axis, for now + if config.quant_dtype is not None: + return True # unquantized-only for this axis, for now return False @@ -247,12 +280,13 @@ def generate_valid_test_cases( cases = [] total = 0 - for k, n, e, dtype, quant_config, combination in product( + for k, n, e, dtype, quant_config, activation, combination in product( Ks, Ns, Es, DTYPEs, MK_QUANT_CONFIGS, + MK_ACTIVATIONS, product(prepare_finalize_types, MK_FUSED_EXPERT_TYPES), ): total = total + 1 @@ -265,6 +299,7 @@ def generate_valid_test_cases( topks=TOPKs, dtype=dtype, quant_config=quant_config, + activation=activation, prepare_finalize_type=combination[0], fused_experts_type=combination[1], world_size=world_size, @@ -292,6 +327,7 @@ def generate_valid_test_cases( e, dtype, quant_config, + activation, combination[0], combination[1], world_size, @@ -304,7 +340,8 @@ def generate_valid_test_cases( @pytest.mark.parametrize( - "k,n,e,dtype,quant_config,prepare_finalize_type,fused_experts_type,world_size", + "k,n,e,dtype,quant_config,activation," + "prepare_finalize_type,fused_experts_type,world_size", generate_valid_test_cases( world_size=2, prepare_finalize_types=MK_MULTI_GPU_PREPARE_FINALIZE_TYPES ), @@ -316,6 +353,7 @@ def test_modular_kernel_combinations_multigpu( e: int, dtype: torch.dtype, quant_config: TestMoEQuantConfig | None, + activation: MoEActivation, prepare_finalize_type: mk.FusedMoEPrepareAndFinalize, fused_experts_type: mk.FusedMoEExperts, world_size: int, @@ -336,6 +374,7 @@ def test_modular_kernel_combinations_multigpu( topks=TOPKs, dtype=dtype, quant_config=quant_config, + activation=activation, prepare_finalize_type=prepare_finalize_type, fused_experts_type=fused_experts_type, world_size=world_size, @@ -345,7 +384,8 @@ def test_modular_kernel_combinations_multigpu( @pytest.mark.parametrize( - "k,n,e,dtype,quant_config,prepare_finalize_type,fused_experts_type,world_size", + "k,n,e,dtype,quant_config,activation," + "prepare_finalize_type,fused_experts_type,world_size", generate_valid_test_cases( world_size=1, prepare_finalize_types=MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES ), @@ -356,6 +396,7 @@ def test_modular_kernel_combinations_singlegpu( e: int, dtype: torch.dtype, quant_config: TestMoEQuantConfig | None, + activation: MoEActivation, prepare_finalize_type: mk.FusedMoEPrepareAndFinalize, fused_experts_type: mk.FusedMoEExperts, world_size: int, @@ -372,6 +413,7 @@ def test_modular_kernel_combinations_singlegpu( topks=TOPKs, dtype=dtype, quant_config=quant_config, + activation=activation, prepare_finalize_type=prepare_finalize_type, fused_experts_type=fused_experts_type, world_size=world_size, @@ -387,6 +429,268 @@ def test_modular_kernel_combinations_singlegpu( run(config, verbosity > 0) +# AITER sorting-backend dispatch env-var matrix (issue #54966 step 3) ------- +# +# AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING are aiter globals +# read once at import time, so each combo needs a fresh child process (see +# parallel_launch_with_config) rather than in-process monkeypatching. +# +# AITER_MOE_SORT_BACKEND is not covered: it only matters when output_aux=True, +# which vLLM only sets on the MXFP4 path -- not yet wired into AiterExperts. + + +def _aiter_sort_backend_hooks_available() -> bool: + """Whether the installed aiter build exposes the sort-backend hooks that + AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING select between. + """ + try: + import aiter.fused_moe as aiter_fused_moe + except ImportError: + return False + + required = ( + "_moe_sorting_impl", + "_flydsl_moe_sorting", + "_USE_CK_MOE_SORTING", + "_USE_FLYDSL_MOE_SORTING", + ) + return all(hasattr(aiter_fused_moe, name) for name in required) + + +# Deliberately does NOT call _aiter_sort_backend_hooks_available() here: this +# marker is evaluated at module-import time, which also happens inside each +# spawned child (to unpickle its worker) -- before that case's env vars are +# applied, permanently freezing aiter's sort-backend globals. The hooks check +# instead runs inside the test body below (parent process only). +require_aiter_moe = pytest.mark.skipif( + not ( + current_platform.is_rocm() + and has_aiter() + and rocm_aiter_ops.is_fused_moe_enabled() + ), + reason=( + "AITER MoE sorting-backend dispatch needs ROCm + AITER, with " + "VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1 set before " + "the test process starts." + ), +) + +# (AITER_USE_CK_MOE_SORTING, AITER_USE_FLYDSL_MOE_SORTING, expected_backend) +AITER_SORTING_BACKEND_ENV_CASES = [ + (1, 0, "ck"), + (1, 1, "ck"), # CK wins over FlyDSL even when both are requested. + (0, 1, "flydsl"), + (0, 0, "opus"), # default when neither flag is set. +] + + +def _aiter_sorting_backend_worker( + pgi: ProcessGroupInfo, + vllm_config: VllmConfig, + cpu_group, + config: Config, + weights: WeightTensors, + verbose: bool, + ck: int, + flydsl: int, + expected_backend: str, +): + # Imported lazily in the spawned child, after env vars are applied, so + # aiter's import-time globals pick up this case's env. + import aiter.fused_moe as aiter_fused_moe + + # Assert the env vars actually stuck in aiter's import-time globals, + # rather than inferring it indirectly from which sorting function fires. + assert bool(ck) == aiter_fused_moe._USE_CK_MOE_SORTING, ( + f"aiter.fused_moe._USE_CK_MOE_SORTING={aiter_fused_moe._USE_CK_MOE_SORTING} " + f"but AITER_USE_CK_MOE_SORTING={ck} was set before this process started -- " + "the env var never reached aiter's import-time globals." + ) + assert bool(flydsl) == aiter_fused_moe._USE_FLYDSL_MOE_SORTING, ( + "aiter.fused_moe._USE_FLYDSL_MOE_SORTING=" + f"{aiter_fused_moe._USE_FLYDSL_MOE_SORTING} but AITER_USE_FLYDSL_MOE_SORTING=" + f"{flydsl} was set before this process started -- the env var never " + "reached aiter's import-time globals." + ) + + # Captured before patching, since the patched attribute would resolve to + # the mock's own (*args, **kwargs) signature instead of the real one. + original_moe_sorting_impl = aiter_fused_moe._moe_sorting_impl + + with ( + mock.patch.object( + aiter_fused_moe, + "_moe_sorting_impl", + wraps=original_moe_sorting_impl, + ) as sorting_impl_mock, + mock.patch.object( + aiter_fused_moe, + "_flydsl_moe_sorting", + wraps=aiter_fused_moe._flydsl_moe_sorting, + ) as flydsl_mock, + ): + # Reuses the real correctness/accuracy checking rank_worker already + # does for every other AiterExperts combo, instead of duplicating it. + rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose) + + if expected_backend == "flydsl": + assert flydsl_mock.call_count > 0, "Expected FlyDSL sorting to fire." + assert sorting_impl_mock.call_count == 0, ( + "FlyDSL was requested and eligible, but the opus/CK sorting " + "path fired instead -- a silent fallback." + ) + else: + assert flydsl_mock.call_count == 0, ( + f"Expected the '{expected_backend}' sorting path, but FlyDSL " + "fired instead." + ) + assert sorting_impl_mock.call_count > 0, ( + f"Expected the '{expected_backend}' sorting path to fire." + ) + # Bind by signature so this works regardless of whether use_opus + # is passed positionally or by keyword. + call = sorting_impl_mock.call_args + bound = inspect.signature(original_moe_sorting_impl).bind( + *call.args, **call.kwargs + ) + use_opus = bound.arguments["use_opus"] + assert use_opus == (expected_backend == "opus"), ( + f"Expected use_opus={expected_backend == 'opus'} for the " + f"'{expected_backend}' sorting path, got use_opus={use_opus}." + ) + + +@require_aiter_moe +@pytest.mark.parametrize("ck,flydsl,expected_backend", AITER_SORTING_BACKEND_ENV_CASES) +def test_aiter_moe_sorting_backend_dispatch_env_matrix( + ck: int, flydsl: int, expected_backend: str +): + """See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER + backend and dispatch settings").""" + if not _aiter_sort_backend_hooks_available(): + pytest.skip( + "Installed aiter build predates the sort-backend hooks " + "(_moe_sorting_impl, _flydsl_moe_sorting, _USE_CK_MOE_SORTING, " + "_USE_FLYDSL_MOE_SORTING)." + ) + + from vllm.model_executor.layers.fused_moe.prepare_finalize import ( + MoEPrepareAndFinalizeNoDPEPModular, + ) + + config = Config( + Ms=[32], + K=2048, + N=1024, + E=32, + topks=[4], + dtype=torch.bfloat16, + quant_config=None, + prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular, + fused_experts_type=AiterExperts, + world_size=1, + ) + assert config.is_valid()[0] + + weights = WeightTensors.make(config) + vllm_config, env_dict = config.make_env_data() + env_dict = { + **env_dict, + "AITER_USE_CK_MOE_SORTING": str(ck), + "AITER_USE_FLYDSL_MOE_SORTING": str(flydsl), + } + + parallel_launch_with_config( + config.world_size, + _aiter_sorting_backend_worker, + vllm_config, + env_dict, + config, + weights, + False, + ck, + flydsl, + expected_backend, + ) + + +def _aiter_dispatch_policy_worker( + pgi: ProcessGroupInfo, + vllm_config: VllmConfig, + cpu_group, + config: Config, + weights: WeightTensors, + verbose: bool, + dispatch_policy: int, +): + # Drives the real AiterExperts.apply() -> rocm_aiter_fused_experts() call + # through the modular kernel, so it also catches a regression in apply()'s + # own forwarding (unlike test_rocm_aiter_moe.py's version, which supplies + # the policy value directly to rocm_aiter_fused_experts()). + with mock.patch.object( + rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe + ) as fused_moe_mock: + rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose) + + assert fused_moe_mock.call_count > 0, ( + "Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe." + ) + for call in fused_moe_mock.call_args_list: + forwarded = call.kwargs["moe_sorting_dispatch_policy"] + assert forwarded == dispatch_policy, ( + f"AiterExperts.apply() forwarded moe_sorting_dispatch_policy=" + f"{forwarded}, but VLLM_ROCM_AITER_MOE_DISPATCH_POLICY=" + f"{dispatch_policy} was set before this process started." + ) + + +@require_aiter_moe +@pytest.mark.parametrize("dispatch_policy", [0, 1, 2]) +def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int): + """See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER + backend and dispatch settings"). + + Complements test_aiter_moe_dispatch_policy_forwarded_to_fused_moe in + test_rocm_aiter_moe.py by driving the real AiterExperts.apply() call + instead of supplying moe_sorting_dispatch_policy directly. + """ + from vllm.model_executor.layers.fused_moe.prepare_finalize import ( + MoEPrepareAndFinalizeNoDPEPModular, + ) + + config = Config( + Ms=[32], + K=2048, + N=1024, + E=32, + topks=[4], + dtype=torch.bfloat16, + quant_config=None, + prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular, + fused_experts_type=AiterExperts, + world_size=1, + ) + assert config.is_valid()[0] + + weights = WeightTensors.make(config) + vllm_config, env_dict = config.make_env_data() + env_dict = { + **env_dict, + "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": str(dispatch_policy), + } + + parallel_launch_with_config( + config.world_size, + _aiter_dispatch_policy_worker, + vllm_config, + env_dict, + config, + weights, + False, + dispatch_policy, + ) + + if __name__ == "__main__": # Ability to test individual PrepareAndFinalize and FusedExperts combination from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 5b1af59731..ba79cc5c34 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -275,11 +275,6 @@ def maybe_roundup_layer_hidden_size( return hidden_size -def rank_chunk(num: int, r: int, w: int) -> int: - rem = num % w - return (num // w) + (1 if r < rem else 0) - - def chunk_by_rank( t: torch.Tensor, r: int, @@ -294,19 +289,6 @@ def chunk_by_rank( return t -def maybe_chunk_by_rank( - t: torch.Tensor | None, - r: int, - w: int, - dim: int = 0, - device: torch.device | None = None, -) -> torch.Tensor | None: - if t is not None: - return chunk_by_rank(t, r, w, dim, device) - else: - return t - - def tp_chunk_gate_up( w: torch.Tensor, tp_rank: int, @@ -676,26 +658,6 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]: return True, None -def chunk_scales_by_rank( - t: torch.Tensor | None, - r: int, - w: int, - device: torch.device | None = None, -) -> torch.Tensor | None: - if t is not None and t.numel() > 1: - # Calculate start index by summing chunk sizes for all previous ranks - # start = sum(rank_chunk(t.shape[0], i, w) for i in range(r)) - # chunk = rank_chunk(t.shape[0], r, w) - # t = t[start:(start + chunk)] - chunk = rank_chunk(t.shape[0], r, w) - t = t[(r * chunk) : max(t.shape[0], (r + 1) * chunk)] - - if t is not None and device is not None: - t = t.to(device) - - return t - - def chunk_scales( t: torch.Tensor | None, start: int, @@ -771,19 +733,6 @@ def _quantize_fp8_halves( ) -def quantization_to_quant_dtype( - quantization: str | None, -) -> torch.dtype | str | None: - if quantization is None: - return None - elif quantization in ["fp8", "fp8_blocked", "modelopt_fp8"]: - return fp8_dtype - elif quantization in ["modelopt_fp4"]: - return "nvfp4" - else: - raise NotImplementedError(f"Unsupported quantization: {quantization}") - - def make_quant_config( quantization: str | None, w1: torch.Tensor, @@ -796,11 +745,11 @@ def make_quant_config( return None, QuantizedWeights(w13_weight=w1, w2_weight=w2) if quantization == "fp8": - return Fp8Config(True), _quantize_fp8_halves(w1, w2) + return Fp8Config(), _quantize_fp8_halves(w1, w2) if quantization == "fp8_blocked": block_shape = [128, 128] - return Fp8Config(True, weight_block_size=block_shape), _quantize_fp8_halves( + return Fp8Config(weight_block_size=block_shape), _quantize_fp8_halves( w1, w2, block_shape ) @@ -1484,6 +1433,7 @@ def _run_one_config( use_shared_experts: bool, use_gate: bool, use_routed_input_transform: bool, + input_scale: float = 1.0, **kwargs, ) -> None: """Generic test loop that sets up environment and delegates to test_body_fn. @@ -1530,6 +1480,8 @@ def _run_one_config( # Extract data from test_data hidden_states = test_data.hidden_states + if input_scale != 1.0: + hidden_states = hidden_states * input_scale router_logits = test_data.router_logits w1 = test_data.w1 w2 = test_data.w2 @@ -1982,7 +1934,140 @@ def _parallel_worker_rocm_deepep( os._exit(exit_code) -# TODO: add cudagraphs/torch.compile tests +def _test_body_mori_graph( + moe_layer: MoERunner, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + vllm_config: VllmConfig, + num_tokens: int, + num_tokens_across_dp: torch.Tensor, + baseline_output: torch.Tensor, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + AiterExperts, + ) + from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import ( + MoriPrepareAndFinalize, + ) + + kernel = moe_layer._quant_method.moe_kernel + assert kernel is not None + assert isinstance(kernel.prepare_finalize, MoriPrepareAndFinalize) + assert isinstance(kernel.fused_experts, AiterExperts) + assert not kernel.prepare_finalize.use_fp8_dispatch + + def check(actual: torch.Tensor, expected: torch.Tensor) -> None: + torch.testing.assert_close(actual, expected, atol=3.5e-2, rtol=3.5e-2) + # These inputs produce small outputs; an absolute-only budget can + # accept all-zero output. Also bound error relative to the reference. + ref_norm = expected.float().norm() + assert ref_norm > 0 + assert (actual.float() - expected.float()).norm() < 3.5e-2 * ref_norm + + # Forward-context metadata belongs on CPU, outside graph capture. + with set_forward_context( + None, + vllm_config, + num_tokens=num_tokens, + num_tokens_across_dp=num_tokens_across_dp.cpu(), + ): + initial_hidden = hidden_states.clone() + initial_router = router_logits.clone() + for _ in range(2): + eager_output = moe_layer(hidden_states, router_logits) + saved_eager_output = eager_output.clone() + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured_output = moe_layer(hidden_states, router_logits) + graph.replay() + original_output = captured_output.clone() + check(original_output, baseline_output) + + for scale, shift in [(-0.5, 1), (1.75, 3)]: + hidden_states.copy_(initial_hidden * scale) + router_logits.copy_(initial_router.roll(shift, dims=-1)) + expected = moe_layer(hidden_states, router_logits).clone() + graph.replay() + check(captured_output, expected) + assert (expected.float() - original_output.float()).norm() > ( + 0.1 * original_output.float().norm() + ) + + # Later calls must not overwrite an earlier eager return tensor. + check(eager_output, saved_eager_output) + check(original_output, baseline_output) + return baseline_output, original_output + + +def _mori_graph_worker( + pgi: ProcessGroupInfo, + vllm_config: VllmConfig, + cpu_group, + m: int, + n: int, +) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + rocm_aiter_ops.refresh_env_variables() + _run_one_config( + vllm_config=vllm_config, + ep_size=2, + dp_size=2, + tp_size=1, + dp_rank=pgi.rank, + tp_rank=0, + is_sequence_parallel=False, + m=m, + n=n, + k=512, + num_experts=8, + top_k=2, + quantization=None, + backend="mori_high_throughput", + test_body_fn=_test_body_mori_graph, + use_shared_experts=False, + use_gate=False, + use_routed_input_transform=False, + input_scale=1.0 + pgi.rank, + ) + + +@pytest.mark.parametrize("m,n", [(1, 256), (32, 512), (45, 256)]) +def test_moe_layer_mori_graph(m: int, n: int, monkeypatch): + """BF16 EP dispatch/combine preserves routing and outputs across replays.""" + from vllm._aiter_ops import is_aiter_found_and_supported + + if not current_platform.is_rocm() or not has_mori(): + pytest.skip("Requires ROCm and MoRI") + if not is_aiter_found_and_supported(): + pytest.skip("Requires supported AITER kernels") + if current_platform.device_count() < 2: + pytest.skip("Requires two GPUs") + + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "0") + compilation_config = CompilationConfig() + compilation_config.pass_config.fuse_allreduce_rms = False + vllm_config = VllmConfig( + parallel_config=ParallelConfig( + data_parallel_size=2, + tensor_parallel_size=1, + enable_expert_parallel=True, + all2all_backend="mori_high_throughput", + ), + compilation_config=compilation_config, + scheduler_config=SchedulerConfig.default_factory( + max_num_batched_tokens=64, max_num_seqs=64 + ), + ) + parallel_launch_with_config(2, _mori_graph_worker, vllm_config, None, m, n) + + +# TODO: add torch.compile tests and graph coverage for other backends. @pytest.mark.parametrize("dp_size, tp_size, use_ep", PARALLEL_COMBOS) @pytest.mark.parametrize("backend", BACKENDS) @pytest.mark.parametrize("enable_eplb", [False, True]) diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index ef31de06d1..f7da37dc92 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -544,6 +544,198 @@ class TestLoadWeightsExpertBias: list(RoutedExperts.load_weights(experts, weights)) +class TestLoadWeightsExpertMapping: + @staticmethod + def _mapping(projs=("gate_proj", "down_proj", "up_proj"), **kwargs): + return RoutedExperts.build_expert_params_mapping( + *projs, + num_experts=16, + routed_experts_prefix="", + include_fused=True, + **kwargs, + ) + + @staticmethod + def _load( + mapping, + weights, + suffix="weight", + *, + transposed=False, + quant_method="tensor", + layer_name="model.layers.0.mlp.experts", + ): + experts = torch.nn.Module() + experts.layer_name = layer_name + experts.get_expert_mapping = lambda **_: mapping + experts.is_fused_checkpoint_transposed = transposed + experts._orient_fused_weight = RoutedExperts._orient_fused_weight + calls = [] + + def weight_loader(param, loaded_weight, shard_id, expert_id, **_): + calls.append((param.test_name, shard_id, expert_id, loaded_weight.clone())) + return True + + for prefix in ("w13", "w2"): + param_name = f"{prefix}_{suffix}" + param = torch.nn.Parameter(torch.empty(0), requires_grad=False) + param.test_name = param_name + param.quant_method = quant_method + param.weight_loader = weight_loader + setattr(experts, param_name, param) + loaded = list(RoutedExperts.load_weights(experts, iter(weights))) + assert loaded == [call[0] for call in calls] + return calls + + @pytest.mark.parametrize( + "projs,lora_prefix", + [ + (("gate_proj", "down_proj", "up_proj"), ""), + (("w1", "w2", "w3"), ""), + (("up_proj", "down_proj", "up_proj"), ""), + (("up_proj", "down_proj", ""), ""), + (("gate_proj", "down_proj", "up_proj"), "base_layer."), + ], + ) + @pytest.mark.parametrize( + "suffix,shape,dtype", + [ + ("weight", (2, 3), torch.bfloat16), + ("input_scale", (), torch.float32), + ("g_idx", (2,), torch.int32), + ], + ) + def test_per_expert_weights_keep_all_physical_destinations( + self, projs, lora_prefix, suffix, shape, dtype + ): + mapping = self._mapping( + projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix + ) + for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]: + for proj in dict.fromkeys(projs): + if not proj: + continue + weight = torch.full(shape, logical_id + 1, dtype=dtype) + name = f"{logical_id}.{proj}.{lora_prefix}{suffix}" + calls = self._load(mapping, [(name, weight)], suffix) + shards = [s for p, s in zip(projs, ("w1", "w2", "w3")) if p == proj] + expected = [ + (f"{'w2' if shard == 'w2' else 'w13'}_{suffix}", shard, physical) + for physical in physical_ids + for shard in shards + ] + assert [call[:3] for call in calls] == expected + for call in calls: + torch.testing.assert_close(call[3], weight, rtol=0, atol=0) + + @pytest.mark.parametrize( + "projs,fused_name,lora_prefix", + [ + (("gate_proj", "down_proj", "up_proj"), "gate_up_proj", ""), + (("w1", "w2", "w3"), "w13", "base_layer."), + ], + ) + def test_per_expert_fused_gate_up_keeps_both_halves( + self, projs, fused_name, lora_prefix + ): + mapping = self._mapping( + projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix + ) + weight = torch.arange(12).reshape(4, 3) + calls = self._load(mapping, [(f"1.{fused_name}.{lora_prefix}weight", weight)]) + assert [call[:3] for call in calls] == [ + ("w13_weight", shard, expert) + for expert in (1, 17) + for shard in ("w1", "w3") + ] + for call, expected in zip(calls, weight.chunk(2) * 2): + torch.testing.assert_close(call[3], expected, rtol=0, atol=0) + + @pytest.mark.parametrize( + "suffix,quant_method,transposed", + [ + ("weight", "tensor", False), + ("weight", "tensor", True), + ("weight_scale", "channel", True), + ("weight_scale", "block", True), + ], + ) + def test_mixed_weights_reset_mapping_and_keep_fused_layout( + self, transposed, suffix, quant_method + ): + weight = torch.arange(192).reshape(16, 4, 3) + checkpoint = ( + weight.transpose(-1, -2) + if (transposed and (suffix == "weight" or quant_method == "block")) + else weight + ) + calls = self._load( + self._mapping(), + [ + (f"1.gate_proj.{suffix}", weight[1, :2]), + ("gate_up_proj" + suffix[6:], checkpoint), + (f"10.down_proj.{suffix}", weight[10, :2]), + ], + suffix, + transposed=transposed, + quant_method=quant_method, + ) + fused_calls = [ + (f"w13_{suffix}", shard, expert) + for shard in ("w1", "w3") + for expert in range(16) + ] + assert [call[:3] for call in calls] == [ + (f"w13_{suffix}", "w1", 1), + *fused_calls, + (f"w2_{suffix}", "w2", 10), + ] + expected = [weight[1, :2]] + expected += [expert for half in weight.chunk(2, dim=1) for expert in half] + expected += [weight[10, :2]] + for call, tensor in zip(calls, expected): + torch.testing.assert_close(call[3], tensor, rtol=0, atol=0) + + def test_fused_weights_stop_after_first_matching_run(self): + mapping = self._mapping() + calls = self._load( + [mapping[0], mapping[2], mapping[1]], + [("gate_up_proj", torch.ones(2, 4, 3))], + ) + assert [call[:3] for call in calls] == [ + ("w13_weight", "w1", 0), + ("w13_weight", "w1", 1), + ] + + def test_repeated_experts_prefix_does_not_hide_later_matches(self): + calls = self._load( + self._mapping(), + [("1.gate_proj.weight", torch.ones(2, 3))], + layer_name="model.experts.0.child.experts", + ) + assert [call[:3] for call in calls] == [("w13_weight", "w1", 1)] + + @pytest.mark.parametrize( + "name", + [ + "01.gate_proj.weight", + "16.gate_proj.weight", + "gate.weight", + "non_expert.weight", + ], + ) + def test_unmatched_names_do_not_load_experts(self, name): + assert self._load(self._mapping(), [(name, torch.ones(2, 3))]) == [] + + @pytest.mark.parametrize( + "weight_name", ["1.gate_proj.", "experts.", "experts.1", ""] + ) + def test_unusual_matching_entries_are_not_silently_skipped(self, weight_name): + mapping = self._mapping() + [("missing", weight_name, 1, "w3")] + with pytest.raises(AttributeError, match="has no parameter"): + self._load(mapping, [("1.gate_proj.weight", torch.ones(2, 3))]) + + class TestPerTensorScaleCoercion: """Regression test for shape-(1,) per-tensor scales (issue #43297). diff --git a/tests/kernels/moe/test_mori_moe.py b/tests/kernels/moe/test_mori_moe.py new file mode 100644 index 0000000000..9869b6bc54 --- /dev/null +++ b/tests/kernels/moe/test_mori_moe.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix.""" + +from dataclasses import replace + +import pytest +import torch + +from tests.kernels.moe.modular_kernel_tools.parallel_utils import ( + parallel_launch_with_config, +) +from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights +from tests.kernels.utils import torch_experts +from tests.utils import multi_gpu_test +from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_mori +from vllm.utils.torch_utils import set_random_seed +from vllm.v1.worker.workspace import init_workspace_manager + +MNKS = [ + (1, 256, 256), + (2, 256, 512), + (3, 1024, 2048), + (32, 256, 1024), + (45, 512, 2048), + (64, 1024, 1024), + (222, 1024, 2048), +] + + +def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts + from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import ( + MoriPrepareAndFinalize, + ) + + rocm_aiter_ops.refresh_env_variables() + init_workspace_manager(pgi.device) + set_random_seed(47) + dtype = current_platform.fp8_dtype() if fp8 else None + (_, w1, s1, _), (_, w2, s2, _) = make_test_weights( + 32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8 + ) + # Share global expert weights, but give each rank different token inputs. + set_random_seed(101 + pgi.rank) + x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10 + routing = torch.randn(m, 32, device=pgi.device) + # Guarantee a route to both ranks, including the one-token case. + routing[:, 0] = routing[:, 16] = float("inf") + ids = routing.topk(6, dim=-1).indices.to(torch.int32) + weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32) + parallel = replace( + FusedMoEParallelConfig.make_no_parallel(), + dp_size=2, + dp_rank=pgi.rank, + ep_size=2, + ep_rank=pgi.rank, + use_ep=True, + all2all_backend="mori_high_throughput", + ) + config = replace( + make_dummy_moe_config( + num_experts=32, + num_local_experts=16, + experts_per_token=6, + hidden_dim=k, + intermediate_size=n, + max_num_tokens=256, + ), + moe_parallel_config=parallel, + ) + start, end = pgi.rank * 16, (pgi.rank + 1) * 16 + quant = FusedMoEQuantConfig.make( + dtype, + per_act_token_quant=fp8, + per_out_ch_quant=fp8, + w1_scale=None if s1 is None else s1[start:end].contiguous(), + w2_scale=None if s2 is None else s2[start:end].contiguous(), + ) + prepare = maybe_make_prepare_finalize(config, quant) + assert isinstance(prepare, MoriPrepareAndFinalize) + assert prepare.use_fp8_dispatch == fp8 + assert prepare.num_dispatchers() == 2 + experts = AiterExperts(config, quant) + assert not experts.expects_unquantized_inputs + kernel = FusedMoEKernel(prepare, experts) + local_w1, local_w2 = rocm_aiter_ops.shuffle_weights( + w1[start:end].contiguous(), w2[start:end].contiguous() + ) + local_w1.is_shuffled = local_w2.is_shuffled = True + expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device) + expert_mask[start:end] = 1 + + def reference(route_weights): + return torch_experts( + x, + w1, + w2, + route_weights, + ids, + w1_scale=s1, + w2_scale=s2, + quant_dtype=dtype, + per_act_token_quant=fp8, + ) + + def run(): + with set_forward_context(None, vllm_config): + return kernel.apply( + x, + local_w1, + local_w2, + weights, + ids, + activation=MoEActivation.SILU, + global_num_experts=32, + expert_map=expert_mask, + apply_router_weight_on_input=False, + ) + + def check(actual): + expected = reference(weights) + assert actual.shape == expected.shape + assert actual.dtype == expected.dtype + local_weights = weights * ((ids >= start) & (ids < end)) + missing_remote = reference(local_weights) + norm = expected.float().norm() + assert norm > 0 + norm_budget = 0.2 if fp8 else 0.06 + assert (actual.float() - expected.float()).norm() / norm < norm_budget + # The norm check also rejects low-signal zero or rank-local results. + for incorrect in (torch.zeros_like(expected), missing_remote): + assert (incorrect.float() - expected.float()).norm() / norm > norm_budget + if fp8: + assert ( + torch.isclose(actual, expected, atol=0.2, rtol=0.2) + .float() + .mean() + .item() + > 0.99 + ) + else: + torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06) + assert torch.isfinite(actual).all() + + actual = run() + check(actual) + if graph: + saved = actual.clone() + run() + capture = torch.cuda.CUDAGraph() + with torch.cuda.graph(capture): + captured = run() + for factor in (-0.5, 1.5): + x.mul_(factor) + weights.mul_(-0.75) + capture.replay() + torch.accelerator.synchronize() + check(captured) + torch.testing.assert_close(actual, saved, atol=0, rtol=0) + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI" +) +@pytest.mark.parametrize("m,n,k", MNKS) +@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"]) +@multi_gpu_test(num_gpus=2) +def test_mori_moe(m, n, k, fp8): + config = VllmConfig( + parallel_config=ParallelConfig( + data_parallel_size=2, + enable_expert_parallel=True, + all2all_backend="mori_high_throughput", + ), + scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256), + ) + env = { + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_USE_AITER_MOE": "1", + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", + } + parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 1cd204d4b0..e4b11c9548 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, replace import pytest import torch -from tests.kernels.moe.utils import check_accuracy +from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize from vllm._aiter_ops import ( is_aiter_found, is_aiter_found_and_supported, @@ -663,6 +663,154 @@ def test_trtllm_gen_mxfp4_fused_moe( check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8) +@pytest.mark.parametrize( + "monolithic,num_tokens,chunk_size", + [(False, 1, None), (False, 16, None), (False, 16, 5), (True, 16, None)], +) +@pytest.mark.skipif( + not TRTLLM_GEN_MXFP4_AVAILABLE, + reason="nvidia gpu and compute capability sm100 is required for this test", +) +def test_trtllm_mxfp4_deferred_finalize( + monolithic: bool, + num_tokens: int, + chunk_size: int | None, + monkeypatch: pytest.MonkeyPatch, + workspace_init, +): + """TRTLLM-Gen MXFP4 experts can leave the top-k finalize to the caller.""" + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, + ) + from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import ( + TrtLlmMxfp4ExpertsModular, + TrtLlmMxfp4ExpertsMonolithic, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + convert_weight_to_mxfp4_moe_kernel_format, + make_mxfp4_moe_kernel, + make_mxfp4_moe_quant_config, + ) + + num_experts, topk, hidden_size, intermediate_size = 32, 4, 1024, 512 + device = torch.device("cuda:0") + torch.manual_seed(0) + if chunk_size is not None: + monkeypatch.setattr( + TrtLlmMxfp4ExpertsModular, + "_max_supported_tokens", + lambda self, top_k, global_num_experts: chunk_size, + ) + + def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + w_q, w_scale = fp4_quantize( + w, + torch.tensor(1.0, device=device), + 32, + sf_use_ue8m0=True, + is_sf_swizzled_layout=False, + ) + return ( + w_q.reshape(*w.shape[:-1], -1), + w_scale.view(torch.uint8).reshape(*w.shape[:-1], -1), + ) + + w13, w13_scale = quantize( + torch.randn( + num_experts, + 2 * intermediate_size, + hidden_size, + device=device, + dtype=torch.bfloat16, + ) + ) + w2, w2_scale = quantize( + torch.randn( + num_experts, + hidden_size, + intermediate_size, + device=device, + dtype=torch.bfloat16, + ) + ) + backend = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8 + w13, w2, w13_scale, w2_scale, _, _ = convert_weight_to_mxfp4_moe_kernel_format( + backend, + torch.nn.Module(), + w13, + w2, + w13_scale, + w2_scale, + _cache_permute_indices={}, + ) + # One rank of a TP group, which deferral needs. + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=topk, + hidden_dim=hidden_size, + intermediate_size=2 * intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + activation=MoEActivation.SILU, + device=device, + routing_method=RoutingMethodType.Renormalize, + moe_parallel_config=replace( + FusedMoEParallelConfig.make_no_parallel(), tp_size=2 + ), + in_dtype=torch.bfloat16, + ) + kernel = make_mxfp4_moe_kernel( + make_mxfp4_moe_quant_config(backend, w1_scale=w13_scale, w2_scale=w2_scale), + moe_config, + TrtLlmMxfp4ExpertsMonolithic if monolithic else TrtLlmMxfp4ExpertsModular, + backend, + ) + + hidden_states = torch.randn( + num_tokens, hidden_size, device=device, dtype=torch.bfloat16 + ) + if monolithic: + router_logits = torch.randn(num_tokens, num_experts, device=device) + check_deferred_moe_finalize( + moe_config, + lambda: kernel.apply_monolithic( + hidden_states, + w13, + w2, + router_logits, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + apply_router_weight_on_input=False, + ), + ) + return + + topk_ids = torch.rand(num_tokens, num_experts, device=device).argsort(dim=-1) + topk_ids = topk_ids[:, :topk].to(torch.int32) + topk_weights = torch.rand(num_tokens, topk, device=device).softmax(dim=-1) + check_deferred_moe_finalize( + moe_config, + lambda: kernel.apply( + hidden_states=hidden_states, + w1=w13, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + apply_router_weight_on_input=False, + ), + router_weights=topk_weights, + chunked=chunk_size is not None, + ) + + @pytest.mark.parametrize("topk", [1, 4]) @pytest.mark.parametrize("num_experts", [32]) @pytest.mark.parametrize("num_tokens", [1, 128]) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 0967f9499e..fbc8586fd3 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -22,6 +22,7 @@ import importlib import math import warnings from typing import Any, NamedTuple +from unittest import mock import pytest import torch @@ -565,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env( assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected -def test_aiter_moe_situv2_syncs_aiter_a4w4_env( - monkeypatch: pytest.MonkeyPatch, -): - """SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4( - monkeypatch: pytest.MonkeyPatch, -): - """Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_clears_aiter_env_when_disabled( +@pytest.mark.parametrize( + "value,expected_act,expected_env", + [ + # Unset / auto / legacy "1" all mean the a4w4 default. + (None, "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("0", "a16w4", {}), + ("a16w4", "a16w4", {}), + ("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}), + ], +) +def test_aiter_moe_situv2_activation_syncs_aiter_env( + value: str | None, + expected_act: str, + expected_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, ): + """Each SiTUv2 activation choice sets exactly its AITER dispatch env.""" + import os + from vllm._aiter_ops import rocm_aiter_ops _assert_aiter_supported() with monkeypatch.context() as mp: + # Stale AITER overrides must be replaced by the vLLM selection. + mp.setenv("AITER_SITUV2_A8W4", "1") mp.setenv("AITER_SITUV2_A4W4", "1") mp.setenv("VLLM_ROCM_USE_AITER", "1") mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0") + if value is None: + mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) + else: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value) _reload_envs() rocm_aiter_ops.refresh_env_variables() - import os + assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act + assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is ( + expected_act == "a8w4" + ) + assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4") + for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"): + assert os.environ.get(name) == expected_env.get(name) - assert "AITER_SITUV2_A4W4" not in os.environ + +def test_aiter_moe_situv2_rejects_unknown_activation( + monkeypatch: pytest.MonkeyPatch, +): + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8") + _reload_envs() + import vllm.envs as envs + + with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"): + _ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @pytest.mark.parametrize("moe_padding", [True, False]) @@ -645,6 +638,76 @@ def test_aiter_moe_padding_env_var( assert envs.VLLM_ROCM_MOE_PADDING is moe_padding +# Dispatch-policy forwarding test ------------------------------------------ + + +@pytest.mark.parametrize("dispatch_policy", [0, 1, 2]) +def test_aiter_moe_dispatch_policy_forwarded_to_fused_moe( + dispatch_policy: int, + monkeypatch: pytest.MonkeyPatch, +): + """VLLM_ROCM_AITER_MOE_DISPATCH_POLICY should reach rocm_aiter_ops.fused_moe + unchanged, the same forwarding AiterExperts.apply does via + rocm_aiter_fused_experts, for every documented policy value (0=auto, + 1=always single-pass, 2=always multi-pass). See vllm-project/vllm#54966. + """ + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + rocm_aiter_fused_experts, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=8, + hidden_dim=256, + intermediate_dim=512, + num_experts=4, + topk=2, + seed=5, + ) + w1_shuffled, w2_shuffled = _shuffle_moe_weights(case["w1"], case["w2"]) + moe_config = make_dummy_moe_config( + num_experts=4, + experts_per_token=2, + hidden_dim=256, + intermediate_size=512, + in_dtype=torch.bfloat16, + ) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", str(dispatch_policy)) + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + assert rocm_aiter_ops.get_moe_dispatch_policy() == dispatch_policy, ( + "rocm_aiter_ops cached a stale dispatch policy after the env var changed." + ) + + with mock.patch.object( + rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe + ) as fused_moe_mock: + rocm_aiter_fused_experts( + hidden_states=case["hidden_states"], + w1=w1_shuffled, + w2=w2_shuffled, + topk_weights=case["topk_weights"], + topk_ids=case["topk_ids"], + moe_config=moe_config, + activation=MoEActivation.SILU, + moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(), + ) + + assert ( + fused_moe_mock.call_args.kwargs["moe_sorting_dispatch_policy"] + == dispatch_policy + ), ( + "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY was not forwarded to " + "rocm_aiter_ops.fused_moe." + ) + + # Enum tests -------------------------------------------------------------- diff --git a/tests/kernels/moe/test_trtllm_nvfp4_moe.py b/tests/kernels/moe/test_trtllm_nvfp4_moe.py index b9684396b2..292e4549bb 100644 --- a/tests/kernels/moe/test_trtllm_nvfp4_moe.py +++ b/tests/kernels/moe/test_trtllm_nvfp4_moe.py @@ -8,11 +8,13 @@ and GELU — including a Gemma4-shaped case (128 experts, top-k 8, intermediate_size 704) that exercises the non-256-aligned padding path. """ +from dataclasses import replace + import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from tests.kernels.moe.utils import make_test_quant_config +from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config from tests.kernels.quantization.nvfp4_utils import ( FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, @@ -31,6 +33,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, + FusedMoEQuantConfig, RoutingMethodType, ) from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import ( @@ -92,6 +95,36 @@ SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME] SITU = op_registry[_SITU_OP_NAME] +def _make_trtllm_fp4_moe_kernel( + moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig +) -> mk.FusedMoEKernel: + experts = TrtLlmNvFp4ExpertsModular( + moe_config=moe_config, quant_config=quant_config + ) + # Mimic the production weight-loader path so per-expert tensors that + # are normally precomputed in process_weights_after_loading (g1_scale_c + # and the rescaled gemm1_clamp_limit) get materialized. The test's + # synthetic quant_config has g1_alphas/g2_alphas already at their + # post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas + # (same tensor) and use input_scale=1 to make the in-place + # weight_scale_2 *= input_scale step a no-op. + fake_layer = torch.nn.Module() + fake_layer.w13_weight_scale_2 = quant_config.g1_alphas + fake_layer.w2_weight_scale_2 = quant_config.g2_alphas + fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas) + fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas) + experts.process_weights_after_loading(fake_layer) + return mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + experts, + ) + + ACTIVATION_CASES = [ pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"), pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"), @@ -191,34 +224,11 @@ def test_trtllm_fp4_moe_no_graph( ), ) - trtllm_inner = TrtLlmNvFp4ExpertsModular( - moe_config=moe_config, quant_config=quant_config - ) - # Mimic the production weight-loader path so per-expert tensors that - # are normally precomputed in process_weights_after_loading (g1_scale_c - # and the rescaled gemm1_clamp_limit) get materialized. The test's - # synthetic quant_config has g1_alphas/g2_alphas already at their - # post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas - # (same tensor) and use input_scale=1 to make the in-place - # weight_scale_2 *= input_scale step a no-op. - fake_layer = torch.nn.Module() - fake_layer.w13_weight_scale_2 = quant_config.g1_alphas - fake_layer.w2_weight_scale_2 = quant_config.g2_alphas - fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas) - fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas) - trtllm_inner.process_weights_after_loading(fake_layer) + trtllm_experts = _make_trtllm_fp4_moe_kernel(moe_config, quant_config) if activation == MoEActivation.SITU: - torch.testing.assert_close(trtllm_inner.g1_scale_c, quant_config.a2_gscale) - - trtllm_experts = mk.FusedMoEKernel( - maybe_make_prepare_finalize( - moe=moe_config, - quant_config=quant_config, - allow_new_interface=True, - use_monolithic=False, - ), - trtllm_inner, - ) + torch.testing.assert_close( + trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale + ) trtllm_output = trtllm_experts.apply( hidden_states=a, @@ -277,6 +287,124 @@ def test_trtllm_fp4_moe_no_graph( torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1) +@torch.inference_mode() +def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params(): + """Weight reloads rerun post-processing on the same experts object; the + g1_alphas fold of clamp/beta must start from the unfolded values.""" + e, n, k = 8, 256, 256 + clamp, beta = 7.0, 1.0 + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1)) + ): + _, _, quant_config = make_test_quant_config( + e, + n, + k, + in_dtype=torch.bfloat16, + quant_dtype="nvfp4", + block_shape=None, + per_act_token_quant=False, + is_scale_swizzled=False, + ) + quant_config.gemm1_clamp_limit = clamp + quant_config.gemm1_beta = beta + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=2, + hidden_dim=k, + intermediate_size=n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + routing_method=RoutingMethodType.TopK, + max_num_tokens=16, + ) + experts = TrtLlmNvFp4ExpertsModular( + moe_config=moe_config, quant_config=quant_config + ) + g1_alphas = quant_config.g1_alphas + assert g1_alphas is not None + layer = torch.nn.Module() + layer.w13_weight_scale_2 = g1_alphas + layer.w2_weight_scale_2 = quant_config.g2_alphas + layer.w13_input_scale = torch.ones_like(g1_alphas) + layer.w2_input_scale = torch.ones_like(g1_alphas) + + for new_alphas in (0.5, 0.25): + g1_alphas.fill_(new_alphas) + experts.process_weights_after_loading(layer) + expected = torch.full_like(g1_alphas, 1.0 / new_alphas) + torch.testing.assert_close(experts.gemm1_clamp_limit, clamp * expected) + torch.testing.assert_close(experts.gemm1_beta, beta * expected) + + +@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)]) +@torch.inference_mode() +def test_trtllm_fp4_moe_deferred_finalize( + m: int, + chunk_size: int | None, + monkeypatch: pytest.MonkeyPatch, + workspace_init, +): + """TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller.""" + e, topk, n, k = 128, 8, 1024, 1024 + dtype = torch.bfloat16 + if chunk_size is not None: + monkeypatch.setattr( + TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size + ) + + set_random_seed(7) + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1)) + ): + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1_q, w2_q, quant_config = make_test_quant_config( + e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False + ) + score = torch.randn((m, e), device="cuda", dtype=dtype) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False) + + # One rank of a TP group, which deferral needs. + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=2 * n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=replace( + FusedMoEParallelConfig.make_no_parallel(), tp_size=2 + ), + in_dtype=dtype, + routing_method=RoutingMethodType.TopK, + max_num_tokens=next_power_of_2(m), + ) + kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config) + + check_deferred_moe_finalize( + moe_config, + lambda: kernel.apply( + hidden_states=a, + w1=w1_q, + w2=w2_q, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ), + router_weights=topk_weights, + chunked=chunk_size is not None, + ) + + if __name__ == "__main__": test_trtllm_fp4_moe_no_graph( 64, diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index d92e4d6129..71b28a9375 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable import torch @@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import ( fused_experts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput +from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import ( BatchedPrepareAndFinalize, ) @@ -549,16 +552,6 @@ class TestMLP(torch.nn.Module): return x -def make_naive_shared_experts( - N: int, - K: int, - in_dtype: torch.dtype = torch.bfloat16, -) -> torch.nn.Module: - w1 = torch.randn((K, N * 2), device=DEVICE, dtype=in_dtype) / 15 - w2 = torch.randn((N, K), device=DEVICE, dtype=in_dtype) / 15 - return TestMLP(w1, w2, out_dtype=in_dtype) - - class RealMLP(torch.nn.Module): def __init__( self, @@ -641,7 +634,7 @@ def make_shared_experts_with_weights( if quant_dtype == torch.float8_e4m3fn: from vllm.model_executor.layers.quantization.fp8 import Fp8Config - quant_config = Fp8Config(True) + quant_config = Fp8Config() else: quant_config = None @@ -665,25 +658,6 @@ def modular_triton_fused_moe( ) -def make_shared_experts( - N: int, - K: int, - in_dtype: torch.dtype = torch.bfloat16, - quant_dtype: torch.dtype | str | None = None, -) -> torch.nn.Module: - (_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights( - 1, - N, - K, - in_dtype=in_dtype, - quant_dtype=quant_dtype, - ) - - return make_shared_experts_with_weights( - N, K, in_dtype, w1, w2, w1_s=w1_s, w2_s=w2_s, quant_dtype=quant_dtype - ) - - def check_accuracy(a, b, atol, rtol, percent): """Allow a mismatch percentage of 1 - percent.""" if torch.any(torch.isnan(a)): @@ -707,6 +681,51 @@ def check_accuracy(a, b, atol, rtol, percent): ) +def check_deferred_moe_finalize( + moe_config: FusedMoEConfig, + run: Callable[[], torch.Tensor | UnfinalizedMoEOutput], + router_weights: torch.Tensor | None = None, + chunked: bool = False, +) -> None: + """Check a kernel that defers its finalize against the finalize it skips. + + ``run`` calls the kernel on fixed inputs, first as built and then with + ``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must + report truthfully. A deferred output reduced by ``moe_unpermute``, the + TRT-LLM finalize kernel, must give the kernel's own finalized output bit for + bit, and modular experts hand the router's weights back as-is. A call the + experts split across kernel launches finalizes instead. + """ + # The deferred output views the router's buffer, so compare with a copy. + expected_weights = None if router_weights is None else router_weights.clone() + finalized = run() + assert isinstance(finalized, torch.Tensor) + moe_config.defer_moe_finalize() + output = run() + assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked + if chunked: + torch.testing.assert_close(output, finalized, atol=0, rtol=0) + return + + assert isinstance(output, UnfinalizedMoEOutput) + if expected_weights is not None: + torch.testing.assert_close( + output.expert_weights, expected_weights, atol=0, rtol=0 + ) + reference = torch.empty_like(finalized) + moe_unpermute( + reference, + output.gemm2_permuted, + output.expert_weights.float(), + output.expanded_idx_to_permuted_idx, + # The kernel reads its valid-row count through this pointer either way. + expert_first_token_offset=output.gemm2_permuted.new_full( + (1,), output.gemm2_permuted.shape[0], dtype=torch.int64 + ), + ) + torch.testing.assert_close(reference, finalized, atol=0, rtol=0) + + def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8): """Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs. diff --git a/tests/kernels/quant_utils.py b/tests/kernels/quant_utils.py index 7e14ec95d3..188fa74b22 100644 --- a/tests/kernels/quant_utils.py +++ b/tests/kernels/quant_utils.py @@ -181,12 +181,15 @@ def native_per_token_group_quant_fp8( return x_q, x_s -def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8): +def native_per_token_group_quant_int8( + x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True +): """Function to perform per-token-group quantization on an input tensor `x` using native torch. It converts the tensor values into int8 values and returns the quantized tensor along with the scaling factor used for quantization. + Per-token quantization rounds to nearest; block quantization truncates. """ assert x.shape[-1] % group_size == 0, ( "the last dimension of `x` must be divisible by `group_size`" @@ -201,9 +204,10 @@ def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8 # Use float32 for scale calculation for stability amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) x_s = amax / int8_max - x_q = ( - (x_.to(torch.float32) / x_s).round().clamp(min=int8_min, max=int8_max).to(dtype) - ) # Round before clamping + x_q = x_.to(torch.float32) / x_s + if round_to_nearest: + x_q = x_q.round() + x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype) x_q = x_q.reshape(x.shape) x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) @@ -249,25 +253,6 @@ def dequant( return t.to(out_dtype) -def batched_dequant( - t: torch.Tensor, - scale: torch.Tensor | None, - block_shape: list[int] | None, - per_act_token_quant: bool, - out_dtype: torch.dtype | None = torch.float32, -) -> torch.Tensor: - if scale is not None: - assert t.shape[0] == scale.shape[0] - out = torch.empty_like(t, dtype=out_dtype) - for e in range(t.shape[0]): - out[e] = dequant( - t[e], scale[e], block_shape, per_act_token_quant, out_dtype - ) - return out - - return t.to(out_dtype) - - def native_batched_masked_quant_matmul( A: torch.Tensor, B: torch.Tensor, diff --git a/tests/kernels/quantization/test_allspark_gemm.py b/tests/kernels/quantization/test_allspark_gemm.py deleted file mode 100644 index 7e4ce1afaf..0000000000 --- a/tests/kernels/quantization/test_allspark_gemm.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest -import torch - -from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck -from vllm import _custom_ops as ops -from vllm.model_executor.layers.quantization.utils.allspark_utils import ( - ALLSPARK_AMPERE_K_ALIGN, - ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - ALLSPARK_AMPERE_N_ALIGN, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import quantize_weights -from vllm.platforms import current_platform -from vllm.scalar_type import scalar_types -from vllm.utils.platform_utils import num_compute_units - - -def is_gptq_allspark_supported(min_capability: int, max_capability: int) -> bool: - if not current_platform.is_cuda(): - return False - - capability = current_platform.get_device_capability() - assert capability is not None - - return ( - capability.to_int() >= min_capability and capability.to_int() <= max_capability - ) - - -MNK_FACTORS = [ - (1, 4, 8), - (13, 17, 67), - (26, 37, 13), - (48, 16, 24), - (67, 13, 88), - (257, 13, 11), - (658, 13, 11), - (1033, 9, 17), -] - -DTYPES = [torch.float16, torch.bfloat16] -HAS_ZP_OPTS = [False, True] - - -def compute_max_diff(output, output_ref): - return torch.mean(torch.abs(output - output_ref)) / torch.mean( - torch.abs(output_ref) - ) - - -def rand_data(shape, dtype=torch.float16): - return torch.randn(shape, dtype=dtype, device="cuda") - - -@pytest.mark.skipif( - not is_gptq_allspark_supported(80, 89), - reason="AllSpark Ampere kernel is not supported on this GPU type.", -) -@pytest.mark.parametrize("mnk_factors", MNK_FACTORS) -@pytest.mark.parametrize("group_size", [-1]) -@pytest.mark.parametrize("has_zp", HAS_ZP_OPTS) -@pytest.mark.parametrize("dtype", DTYPES) -def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype): - m_factor, n_factor, k_factor = mnk_factors - m = m_factor - n = n_factor * ALLSPARK_AMPERE_N_ALIGN - k = k_factor * ALLSPARK_AMPERE_K_ALIGN - - input = rand_data((m, k), dtype=dtype) - weight = rand_data((k, n), dtype=dtype) - - # Quantize weights. - w_ref, qw, s, zp = quantize_weights( - weight, scalar_types.uint8b128, group_size, has_zp - ) - - qw = qw.to(torch.uint8) - if has_zp: - zp = zp.to(dtype) - properties = torch.cuda.get_device_properties(qw.device.index) - sm_count = num_compute_units(qw.device.index) - sm_version = properties.major * 10 + properties.minor - - n_32align = (n + 32 - 1) // 32 * 32 - - qw_reorder, s_reorder, zp_reorder = ops.allspark_repack_weight(qw, s, zp, has_zp) - opcheck( - torch.ops._C.rearrange_kn_weight_as_n32k16_order, - (qw, s, zp, has_zp, qw_reorder, s_reorder, zp_reorder, k, n, n_32align), - ) - - opcheck( - torch.ops._C.allspark_w8a16_gemm, - ( - input, - qw_reorder, - s_reorder, - zp_reorder, - n, - group_size, - sm_count, - sm_version, - ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - has_zp, - True, - ), - test_utils=DEFAULT_OPCHECK_TEST_UTILS, - ) - output = ops.allspark_w8a16_gemm( - input, - qw_reorder, - s_reorder, - zp_reorder, - n, - group_size, - sm_count, - sm_version, - ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - has_zp, - True, - ) - - output_ref = torch.matmul(input, w_ref) - torch.accelerator.synchronize() - max_diff = compute_max_diff(output, output_ref) - - assert max_diff < 0.04 diff --git a/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py b/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py index 315e35b782..1b6a373071 100644 --- a/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py +++ b/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py @@ -50,7 +50,9 @@ def test_flashinfer_trtllm_mxfp8_linear_numerics( x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda") weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda") layer = _make_layer(weight) - kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel = FlashInferTrtllmMxfp8LinearKernel( + Mxfp8LinearLayerConfig(weight_shape=(n, k)) + ) kernel.process_weights_after_loading(layer) output = kernel.apply_weights(layer, x) @@ -69,7 +71,9 @@ def test_flashinfer_trtllm_mxfp8_custom_ops() -> None: x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda") weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda") layer = _make_layer(weight) - kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel = FlashInferTrtllmMxfp8LinearKernel( + Mxfp8LinearLayerConfig(weight_shape=(256, 512)) + ) kernel.process_weights_after_loading(layer) torch.library.opcheck( @@ -104,7 +108,9 @@ def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None: m, n, k = 7, 130, 512 weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda") layer = _make_layer(weight) - kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel = FlashInferTrtllmMxfp8LinearKernel( + Mxfp8LinearLayerConfig(weight_shape=(n, k)) + ) kernel.process_weights_after_loading(layer) static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda") diff --git a/tests/kernels/quantization/test_fp8_quant.py b/tests/kernels/quantization/test_fp8_quant.py index cec6d37e12..29bbb23af6 100644 --- a/tests/kernels/quantization/test_fp8_quant.py +++ b/tests/kernels/quantization/test_fp8_quant.py @@ -108,6 +108,36 @@ def test_dynamic_per_tensor_fp8_quant( opcheck_fp8_quant(ops_out, x) +@pytest.mark.parametrize("pad", [8, 16]) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("seed", SEEDS) +@torch.inference_mode() +def test_dynamic_per_tensor_fp8_quant_strided_input( + pad: int, dtype: torch.dtype, seed: int +) -> None: + """Non-contiguous rows take the strided abs-max kernel; contiguous rows take + the flat one. Both must produce exactly the same scale and bytes. + + pad=8 (16 B for bf16) leaves rows misaligned for 32 B vector loads, pad=16 + keeps them aligned but non-contiguous. + """ + set_random_seed(seed) + num_tokens, hidden_size = 64, 4096 + + full = torch.rand(num_tokens, hidden_size + pad, dtype=dtype, device="cuda") + x = full[:, :hidden_size] + assert not x.is_contiguous() + + ops_out, ops_scale = ops.scaled_fp8_quant(x) + flat_out, flat_scale = ops.scaled_fp8_quant(x.contiguous()) + assert torch.equal(ops_scale, flat_scale) + assert torch.equal(ops_out.view(torch.uint8), flat_out.view(torch.uint8)) + + ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x) + assert torch.equal(ref_scale, ops_scale) + assert torch.equal(ref_out.view(torch.uint8), ops_out.view(torch.uint8)) + + # Regression test for a case with large activations where an int32 index cannot # represent the number of elements. @torch.inference_mode() diff --git a/tests/kernels/quantization/test_marlin_tile_padding.py b/tests/kernels/quantization/test_marlin_tile_padding.py index d224b0a2b2..8b55910a78 100644 --- a/tests/kernels/quantization/test_marlin_tile_padding.py +++ b/tests/kernels/quantization/test_marlin_tile_padding.py @@ -408,7 +408,9 @@ def test_fp8_block_marlin_padded_round_trip(shape): _gpu_marlin_unsupported() or not is_fp8_marlin_supported(), reason="FP8 Marlin is not supported on this GPU type.", ) -@pytest.mark.parametrize("shape", [(200, 288), (4640, 512)]) +@pytest.mark.parametrize( + "shape", [(32, 256), (96, 256), (130, 256), (200, 288), (4640, 512)] +) def test_mxfp8_marlin_padded_round_trip(shape): """MXFP8 exercises the e8m0 scale path, where padded 0.0 scales clamp to 2^-127 instead of zero and must still contribute nothing.""" diff --git a/tests/kernels/quantization/test_mxfp8_kernel_selection.py b/tests/kernels/quantization/test_mxfp8_kernel_selection.py new file mode 100644 index 0000000000..f559eca4dd --- /dev/null +++ b/tests/kernels/quantization/test_mxfp8_kernel_selection.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for MXFP8 linear kernel selection logic (CPU-only) + +Run `pytest tests/kernels/quantization/test_mxfp8_kernel_selection.py`. +""" + +from contextlib import contextmanager +from unittest.mock import patch + +import pytest + +from vllm.model_executor.kernels.linear import ( + FlashInferCutedslMxfp8LinearKernel, + FlashInferCutlassMxfp8LinearKernel, + MarlinMxfp8LinearKernel, + Mxfp8LinearLayerConfig, + init_mxfp8_linear_kernel, +) +from vllm.platforms import PlatformEnum + +pytestmark = pytest.mark.cpu_test + +# Kernels backed by FlashInfer mm_mxfp8, which requires N, K >= 128 and +# K % 32 == 0. +_MM_MXFP8_KERNELS = [ + FlashInferCutedslMxfp8LinearKernel, + FlashInferCutlassMxfp8LinearKernel, +] + +_SUPPORTED_SHAPE = (4096, 4096) +# (N, K): N < 128, K < 128, K % 32 != 0. +_UNSUPPORTED_SHAPES = [(64, 4096), (4096, 64), (4096, 4112)] + +# (N, K) with N < 128 or K < 128, which only Marlin runs. +_MARLIN_ONLY_SHAPES = [(32, 256), (64, 256), (96, 256), (256, 64)] +# (N, K): N <= 0, K <= 0, K % 32 != 0. +_MARLIN_UNSUPPORTED_SHAPES = [(0, 256), (256, 0), (256, 100)] + +_SELECTION_CASES = [ + (100, (128, 128), FlashInferCutedslMxfp8LinearKernel), + (100, (130, 256), FlashInferCutedslMxfp8LinearKernel), + (103, (128, 128), FlashInferCutedslMxfp8LinearKernel), + (103, (130, 256), FlashInferCutedslMxfp8LinearKernel), + (120, (128, 128), FlashInferCutlassMxfp8LinearKernel), + (120, (130, 256), MarlinMxfp8LinearKernel), + (121, (128, 128), FlashInferCutlassMxfp8LinearKernel), + (121, (130, 256), MarlinMxfp8LinearKernel), + *[ + (capability, shape, MarlinMxfp8LinearKernel) + for capability in (100, 103, 120, 121) + for shape in _MARLIN_ONLY_SHAPES + ], +] + + +@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS) +def test_mm_mxfp8_kernels_accept_supported_shape(kernel_cls): + config = Mxfp8LinearLayerConfig(weight_shape=_SUPPORTED_SHAPE) + can_implement, reason = kernel_cls.can_implement(config) + assert can_implement, reason + + +@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS) +@pytest.mark.parametrize("weight_shape", _UNSUPPORTED_SHAPES) +def test_mm_mxfp8_kernels_reject_unsupported_shape(kernel_cls, weight_shape): + config = Mxfp8LinearLayerConfig(weight_shape=weight_shape) + can_implement, reason = kernel_cls.can_implement(config) + assert not can_implement + assert reason + + +@pytest.mark.parametrize("weight_shape", _MARLIN_UNSUPPORTED_SHAPES) +def test_marlin_mxfp8_rejects_unsupported_shape(weight_shape): + config = Mxfp8LinearLayerConfig(weight_shape=weight_shape) + can_implement, reason = MarlinMxfp8LinearKernel.can_implement(config) + assert not can_implement + assert reason + + +@contextmanager +def _patch_cuda_platform(capability: int): + """Patch the platform so kernel selection runs as on a CUDA GPU with the + given compute capability. The FlashInfer and Marlin kernels report as + supported, CuTe-DSL only on SM100/103.""" + with ( + patch("vllm.model_executor.kernels.linear.current_platform") as platform, + patch( + "vllm.model_executor.kernels.linear.mxfp8.flashinfer.current_platform." + "is_device_capability_family", + side_effect=lambda cap, device_id=0: cap // 10 == capability // 10, + ), + patch.object( + FlashInferCutedslMxfp8LinearKernel, + "is_supported", + return_value=(capability in (100, 103), None), + ), + patch.object( + FlashInferCutlassMxfp8LinearKernel, + "is_supported", + return_value=(True, None), + ), + patch.object( + MarlinMxfp8LinearKernel, "is_supported", return_value=(True, None) + ), + ): + platform._enum = PlatformEnum.CUDA + yield + + +@pytest.mark.parametrize( + ("capability", "expected"), + [(100, True), (103, True), (120, False), (121, False)], +) +def test_cutlass_mxfp8_requires_aligned_n_only_on_sm12x(capability, expected): + """(130, 256) passes the generic mm_mxfp8 limits. Only the SM12x CUTLASS + backend needs N % 32 == 0, so SM100/103 must keep accepting it.""" + config = Mxfp8LinearLayerConfig(weight_shape=(130, 256)) + with _patch_cuda_platform(capability): + can_implement, _ = FlashInferCutlassMxfp8LinearKernel.can_implement(config) + assert can_implement == expected + + +@pytest.mark.parametrize( + ("capability", "weight_shape", "expected_kernel_cls"), _SELECTION_CASES +) +def test_init_mxfp8_linear_kernel(capability, weight_shape, expected_kernel_cls): + """A layer that mm_mxfp8 cannot handle must fall through to the next + kernel in the CUDA priority list instead of being selected and failing + in apply_weights.""" + with _patch_cuda_platform(capability): + kernel = init_mxfp8_linear_kernel(weight_shape=weight_shape) + + assert isinstance(kernel, expected_kernel_cls) + assert kernel.config == Mxfp8LinearLayerConfig(weight_shape=weight_shape) diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py index 33295b8784..3a136484ac 100644 --- a/tests/kernels/quantization/test_nvfp4_emulation.py +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -4,9 +4,21 @@ from typing import cast import pytest import torch +from nvfp4_utils import ( + FLOAT4_E2M1_MAX, + FLOAT8_E4M3_MAX, + break_fp4_bytes, + kE2M1ToFloat, +) from safetensors import safe_open import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.config.kernel import KernelConfig +from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel +from vllm.model_executor.kernels.linear.nvfp4.emulation import ( + EmulationNvFp4LinearKernel, +) from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -34,6 +46,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api from vllm.triton_utils import triton +from vllm.utils.torch_utils import set_random_seed if current_platform.is_rocm(): from vllm.platforms.rocm import on_gfx950 @@ -769,3 +782,85 @@ def test_nvfp4_moe_correctness( atol=0.0 if on_gfx950() else 0.02, rtol=0, ) + + +def _dense_nvfp4_reference( + layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None +) -> torch.Tensor: + """Decode E2M1 nibbles and both scale levels without production helpers.""" + levels = kE2M1ToFloat.cpu() + weight = break_fp4_bytes(layer.weight.cpu(), torch.float32) + weight *= ( + layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu() + ).repeat_interleave(16, dim=-1) + blocks = x.cpu().float().reshape(x.shape[0], -1, 16) + global_scale = layer.input_global_scale_inv.cpu() + scales = ( + (blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale) + .clamp(max=FLOAT8_E4M3_MAX) + .to(torch.float8_e4m3fn) + .float() + ) + inverse = torch.where(scales == 0, 0, global_scale / scales) + scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX) + distances = (scaled.unsqueeze(-1) - levels).abs() + # E2M1 ties round to the even significand (even encoding). + nearest_mask = distances == distances.amin(-1, keepdim=True) + even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0) + candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask) + nearest = candidates.to(torch.int8).argmax(-1) + activation = levels[nearest] * blocks.sign() * (scales / global_scale) + expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T + if bias is not None: + expected += bias.cpu() + return expected + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)]) +@pytest.mark.parametrize("with_bias", [False, True]) +def test_nvfp4_dense_emulation( + dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool +) -> None: + """W4A4 fallback preserves packed scales, bias and outputs across replay.""" + set_random_seed(47) + layer = torch.nn.Module() + # Every signed E2M1 code appears, including both zero encodings. + codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k) + layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8) + layer.weight_scale = ( + torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8 + ).to(torch.float8_e4m3fn) + layer.weight_global_scale = torch.tensor(0.25, device="cuda") + layer.input_global_scale_inv = torch.tensor(3.0, device="cuda") + x = torch.randn(m, k, device="cuda", dtype=dtype) + x[:, :16] = 0 + bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None + with set_current_vllm_config( + VllmConfig(kernel_config=KernelConfig(linear_backend="emulation")) + ): + kernel = init_nvfp4_linear_kernel() + assert isinstance(kernel, EmulationNvFp4LinearKernel) + kernel.process_weights_after_loading(layer) + original_weight = layer.weight.clone() + original_scale = layer.weight_scale.view(torch.uint8).clone() + actual = kernel.apply_weights(layer, x, bias) + torch.testing.assert_close( + actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01 + ) + assert actual.abs().max() > 0.1 + saved = actual.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = kernel.apply_weights(layer, x, bias) + for sign in (-1, 1): + x.mul_(sign * 0.75) + graph.replay() + torch.accelerator.synchronize() + torch.testing.assert_close( + captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01 + ) + torch.testing.assert_close(actual, saved, atol=0, rtol=0) + assert torch.equal(layer.weight, original_weight) + assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale) diff --git a/tests/kernels/quantization/test_per_token_group_quant.py b/tests/kernels/quantization/test_per_token_group_quant.py index 6d3c91022e..a9ac45da03 100644 --- a/tests/kernels/quantization/test_per_token_group_quant.py +++ b/tests/kernels/quantization/test_per_token_group_quant.py @@ -26,6 +26,8 @@ from vllm.platforms import current_platform (16, 512), (3, 2688), (3, 5376), + (333, 128), + (512, 6144), ], ) @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) @@ -39,7 +41,7 @@ from vllm.platforms import current_platform ) def test_per_token_group_quant_fp8( shape, - dtype, + dtype: torch.dtype, column_major: bool, tma_aligned: bool, scale_ue8m0: bool, @@ -78,6 +80,50 @@ def test_per_token_group_quant_fp8( assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01) +def _misaligned_clone(x: torch.Tensor) -> torch.Tensor: + """Contiguous copy of ``x`` whose ``data_ptr`` is not 16B aligned.""" + numel = x.numel() + base = torch.empty(numel + 16 // x.element_size(), device=x.device, dtype=x.dtype) + view = base[1 : 1 + numel] + if view.data_ptr() % 16 == 0: + pytest.skip("allocator did not yield a misaligned offset") + return view.view(x.shape).copy_(x) + + +@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)]) +@pytest.mark.parametrize("column_major", [False, True]) +@pytest.mark.parametrize("scale_ue8m0", [False, True]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="register-resident schedule is CUDA/ROCm only", +) +def test_per_token_group_quant_fp8_register_matches_smem( + shape, column_major: bool, scale_ue8m0: bool, dtype: torch.dtype +): + """group_size=128 with 16-bit input takes a register-resident schedule that + needs 16B-aligned buffers; a misaligned input falls back to the + shared-memory schedule. The two must agree bit-for-bit, which the + tolerance-based comparison against Triton above cannot establish. + """ + device = current_platform.device_type + torch.manual_seed(42) + x = torch.randn(shape, device=device, dtype=dtype) * 8 + + reg_q, reg_s = fp8_utils.per_token_group_quant_fp8( + x, 128, column_major_scales=column_major, use_ue8m0=scale_ue8m0 + ) + smem_q, smem_s = fp8_utils.per_token_group_quant_fp8( + _misaligned_clone(x), + 128, + column_major_scales=column_major, + use_ue8m0=scale_ue8m0, + ) + + assert torch.equal(reg_q.view(torch.uint8), smem_q.view(torch.uint8)) + assert torch.equal(reg_s, smem_s) + + @pytest.mark.parametrize( "num_tokens,hidden_dim,group_size", [ @@ -472,3 +518,24 @@ def test_per_token_group_quant_int8(shape, dtype, group_size: int): assert torch.allclose(out_q.float(), ref_q.float(), atol=0.15, rtol=0.15) assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01) + + +@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="register-resident schedule is CUDA/ROCm only", +) +def test_per_token_group_quant_int8_register_matches_smem(shape, dtype: torch.dtype): + """int8 output shares the register-resident schedule with fp8, so it needs + the same bit-exactness guard against the shared-memory schedule. + """ + device = current_platform.device_type + torch.manual_seed(42) + x = torch.randn(shape, device=device, dtype=dtype) * 8 + + reg_q, reg_s = int8_utils.per_token_group_quant_int8(x, 128) + smem_q, smem_s = int8_utils.per_token_group_quant_int8(_misaligned_clone(x), 128) + + assert torch.equal(reg_q, smem_q) + assert torch.equal(reg_s, smem_s) diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index d4e16629af..b2547c080a 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -614,11 +614,13 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K): out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale) assert out.shape == (M, N) + assert out.dtype == torch.bfloat16 assert not torch.any(torch.isnan(out)) + # Accumulate in FP32, then match the GEMM's BF16 output rounding. A_dq = quant_dequant_mxfp4(A) B_dq = quant_dequant_mxfp4(B) - ref = torch.matmul(A_dq.float(), B_dq.t().float()) + ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float() _print_close_stats( f"skinny_gemm M={M} N={N} K={K}", diff --git a/tests/kernels/quantization/test_rocm_mxfp8_linear.py b/tests/kernels/quantization/test_rocm_mxfp8_linear.py index 1d5e62c6c4..d5e48f2aaf 100644 --- a/tests/kernels/quantization/test_rocm_mxfp8_linear.py +++ b/tests/kernels/quantization/test_rocm_mxfp8_linear.py @@ -66,7 +66,7 @@ def test_rocm_mxfp8_linear_matches_dequant_reference(name, n, k, num_tokens): torch.manual_seed(num_tokens) device = "cuda" layer = _make_layer(n, k, device) - kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None)) + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k))) kernel.process_weights_after_loading(layer) x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5 @@ -103,7 +103,7 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes(): device = "cuda" n, k = 5120, 2048 # wo_b at TP=4 layer = _make_layer(n, k, device) - kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None)) + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k))) kernel.process_weights_after_loading(layer) x = torch.randn(8, k, device=device, dtype=torch.bfloat16) @@ -117,6 +117,148 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes(): assert (out[0] == 0).all(), "an all-zero activation row must give a zero row" +# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The +# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has +# K % 128 != 0, which only the block-scaled GEMM runs natively. +V41_BLOCK32_SHAPES = [ + ("fused_wqa_wkv", 1280 + 512, 5120), + ("wq_b_tp2", 64 * 512 // 2, 1280), + ("indexer_wq_b", 32 * 128, 1280), + ("wo_b_tp2", 5120, 8192 // 2), + ("shared_gate_up_tp2", 2 * 2304 // 2, 5120), + ("shared_down_tp2", 5120, 2304 // 2), + ("shared_down_tp4", 5120, 2304 // 4), +] + + +def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module: + """An MXFP8 layer with 32x32 block scales, expanded per row as loaded.""" + blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32) + amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny) + bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254) + weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k) + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter( + weight.to(torch.float8_e4m3fn), requires_grad=False + ) + layer.weight_scale = torch.nn.Parameter( + bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False + ) + return layer + + +@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v)) +@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024]) +@torch.inference_mode() +def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens): + """32x32 block scales stay compact and the block-scaled GEMM is exact. + + The token counts cover the packed, tiled and split-K configurations. + """ + from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import ( + Mxfp8LinearLayerConfig, + ) + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + RocmDotScaledMxfp8LinearKernel, + ) + + torch.manual_seed(num_tokens) + device = "cuda" + layer = _make_block32_layer(n, k, device) + row_scale = layer.weight_scale.data.clone() + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k))) + kernel.process_weights_after_loading(layer) + assert layer.weight.dtype == torch.float8_e4m3fn + assert layer.weight_scale.shape == (n // 32, k // 32) + + x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5 + out = kernel.apply_weights(layer, x) + + x_q, x_scale = mxfp8_e4m3_quantize(x) + expected = ( + dequant_mxfp8_to_bf16(x_q, x_scale).float() + @ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T + ) + assert out.shape == (num_tokens, n) + rel = (out.float() - expected).norm() / expected.norm() + assert rel < 5e-3, f"{name}: relative error {rel:.4f}" + + +@torch.inference_mode() +def test_rocm_mxfp8_linear_keeps_per_row_scales(): + """Scales that differ within a 32-row block are not compacted.""" + from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import ( + Mxfp8LinearLayerConfig, + ) + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + RocmDotScaledMxfp8LinearKernel, + ) + + n, k = 64, 256 + layer = _make_block32_layer(n, k, "cuda") + layer.weight_scale.data[1, 0] += 1 + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k))) + kernel.process_weights_after_loading(layer) + assert layer.weight_scale.shape == (n, k // 32) + + +def _split_k_case(packed: bool): + """A 3-way in-launch split-K on shared_down TP4, with K and M tails.""" + from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g + + n, k, m = 5120, 576, 19 + layer = _make_block32_layer(n, k, "cuda") + x_q, x_scale = mxfp8_e4m3_quantize( + torch.randn(m, k, device="cuda", dtype=torch.bfloat16) + ) + expected = ( + dequant_mxfp8_to_bf16(x_q, x_scale).float() + @ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T + ) + if packed: + cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3) + else: + cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3) + args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous()) + return g, cfg, args, expected + + +def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float: + return ((out.float() - expected).norm() / expected.norm()).item() + + +@pytest.mark.parametrize("packed", [True, False]) +@torch.inference_mode() +def test_rocm_mxfp8_block32_split_k_in_launch(packed): + """Splits summed by the tile's last program are exact and repeatable.""" + torch.manual_seed(0) + g, cfg, args, expected = _split_k_case(packed) + out = torch.empty_like(expected, dtype=torch.bfloat16) + g._launch(*args, out, cfg) + assert _rel_err(out, expected) < 5e-3 + first = out.clone() + for _ in range(20): + g._launch(*args, out, cfg) + assert torch.equal(out, first) + + +@pytest.mark.parametrize("packed", [True, False]) +@torch.inference_mode() +def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed): + """A stream whose first split-K launch is being captured has no counters + yet; it must fall back to an unfused split instead of failing.""" + torch.manual_seed(0) + g, cfg, args, expected = _split_k_case(packed) + out = torch.zeros_like(expected, dtype=torch.bfloat16) + stream = torch.cuda.Stream() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + g._launch(*args, out, cfg) + graph.replay() + torch.accelerator.synchronize() + assert _rel_err(out, expected) < 5e-3 + + @pytest.mark.parametrize("num_rows", [1, 64, 65]) @torch.inference_mode() def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows): diff --git a/tests/kernels/quantization/test_rocm_skinny_gemms.py b/tests/kernels/quantization/test_rocm_skinny_gemms.py index 7a7fe326bf..838ce34419 100644 --- a/tests/kernels/quantization/test_rocm_skinny_gemms.py +++ b/tests/kernels/quantization/test_rocm_skinny_gemms.py @@ -132,7 +132,8 @@ M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16] NKM_FACTORS_WVSPLITKRC_LARGE_K = [ (128, 6144, 128), (96, 8192, 128), - (128, 12288, 128), + # Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs. + (128, 12288, 64), (32, 12288, 128), ] diff --git a/tests/kernels/test_concat_mla_q.py b/tests/kernels/test_concat_mla_q.py index 39c2742e4f..6d9a102106 100644 --- a/tests/kernels/test_concat_mla_q.py +++ b/tests/kernels/test_concat_mla_q.py @@ -156,7 +156,11 @@ def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled): """FP8 query packing preserves values with empty RoPE and strided BMM output.""" from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.attention.mla_attention import _DecodeConcatQuantFP8 - from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, + get_fp8_min_max, + ) + from vllm.platforms import current_platform torch.manual_seed(42) shape = (8, 17, 512) if transposed else (17, 8, 512) @@ -166,8 +170,9 @@ def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled): qp = torch.randn(17, 8, rope_dim, dtype=torch.bfloat16, device="cuda") * 128 scale = torch.tensor(0.3, device="cuda", dtype=torch.float32) joined = torch.cat((ql, qp), dim=-1) - expected = (joined.float() * scale.reciprocal()).clamp(-448, 448) - expected = expected.to(torch.float8_e4m3fn) + fp8_min, fp8_max = get_fp8_min_max() + expected = (joined.float() * scale.reciprocal()).clamp(fp8_min, fp8_max) + expected = expected.to(current_platform.fp8_dtype()) custom_ops = ["none"] + (["+mla_decode_concat_quant_fp8"] if enabled else []) config = VllmConfig( compilation_config=CompilationConfig(mode=0, custom_ops=custom_ops) @@ -175,5 +180,6 @@ def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled): with set_current_vllm_config(config): quant = _DecodeConcatQuantFP8(static=True, group_shape=GroupShape.PER_TENSOR) actual = quant(ql, qp, scale) + assert actual.dtype == current_platform.fp8_dtype() torch.testing.assert_close(actual.float(), expected.float(), atol=0, rtol=0) assert actual.is_contiguous() diff --git a/tests/kernels/test_engram.py b/tests/kernels/test_engram.py index 45cebcfe57..9f62b0d438 100644 --- a/tests/kernels/test_engram.py +++ b/tests/kernels/test_engram.py @@ -728,12 +728,79 @@ def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens): assert torch.equal(out, expected) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") +@pytest.mark.parametrize("register_fails", [False, True]) +def test_engram_use_thp_lookup_and_fallback(monkeypatch, register_fails): + """Huge-page tables and their pinned fallback both give exact lookup rows.""" + monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1) + monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0) + if register_fails: + monkeypatch.setattr( + torch.cuda.cudart(), "cudaHostRegister", lambda *_: SimpleNamespace(value=1) + ) + rows, dim = 32769, 64 + with torch.device("cuda"): + layer = ParallelEngramEmbedding( + rows, dim, (rows,), cpu_offload=True, use_thp=True + ) + assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned() + same_storage = layer.weight.untyped_storage().data_ptr() == ( + layer.weight_scale_inv.untyped_storage().data_ptr() + ) + assert same_storage != register_fails + layer.weight.data.fill_(2) + layer.weight_scale_inv.data.fill_(127) + ids = torch.tensor( + [[0], [rows - 1], [-1], [rows]], device="cuda", dtype=torch.int32 + ) + out = torch.empty(4, 1, dim, device="cuda", dtype=torch.bfloat16) + layer.lookup(ids, out, background=True) + expected = torch.zeros_like(out) + expected[:2] = 2 + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") +def test_engram_shared_prefetch_stream_waits_per_layer(): + """Consuming a layer must not wait for later layers on the shared stream.""" + stream = torch.cuda.Stream() + modules = [] + for _ in range(2): + module = Engram.__new__(Engram) + torch.nn.Module.__init__(module) + module._prefetch_stream = stream + module.embed_tokens = _make_embedding(cpu_offload=True) + module.use_sequence_parallel = False + with torch.device("cuda"): + module._init_staging(8, module.embed_tokens.dim) + assert module._prefetch_stream is stream + modules.append(module) + hashes = torch.randint(1024, 3072, (8, 2, 24), device="cuda", dtype=torch.int32) + modules[0].prepare_embeddings(hashes[:, 0]) + with torch.cuda.stream(stream): + torch.cuda._sleep(100_000_000) # ~50 ms ahead of the second lookup + modules[1].prepare_embeddings(hashes[:, 1]) + rows = modules[0].embed(hashes[:, 0]) + torch.cuda.current_stream().synchronize() + assert not modules[1]._prefetch_done.query(), "waited for the second lookup" + layer = modules[0].embed_tokens + expected = _reference_lookup( + layer.weight.cuda(), + layer.weight_scale_inv.cuda(), + hashes[:, 0], + layer.vocab_start_idx, + layer.vocab_end_idx, + ) + torch.testing.assert_close(rows, expected, rtol=0, atol=0) + torch.accelerator.synchronize() + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") @pytest.mark.parametrize( "backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)] ) def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): - """Default offload uses pinned storage and a side stream; False uses HBM.""" + """Default offload uses pinned storage and the given stream; False uses HBM.""" from vllm.config import EngramConfig config = SimpleNamespace(hidden_size=16, hc_mult=1, rms_norm_eps=1e-6) @@ -760,11 +827,13 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0) monkeypatch.setattr( - engram_ops, "ReplicatedLinear", lambda *a, **k: torch.nn.Identity() + engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity() ) + stream = torch.cuda.Stream() + kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {} with torch.device("cuda"): cls = Engram if backend == "nvidia" else CommonEngram - module = cls(config, None, layout, 0, False, "engram") + module = cls(config, None, layout, 0, False, "engram", **kwargs) layer = module.embed_tokens if offloaded: assert layer.weight.device.type == "cpu" and layer.weight.is_pinned() @@ -798,8 +867,7 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): ) torch.testing.assert_close(module.embed(ids), expected, atol=0, rtol=0) if offloaded: - assert streams == [module._prefetch_stream] - assert streams[0] != main + assert streams == [stream] else: assert streams == [main] @@ -836,8 +904,9 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende hash_ids.record_stream(stream) with torch.cuda.stream(stream): self.embed_tokens.lookup(hash_ids, rows, background=True) + self._prefetch_done.record(stream) - def finish(self, stream): + def finish(self, event): pass monkeypatch.setattr(Engram, "_start_prefetch", start) @@ -879,6 +948,7 @@ def _run_engram_prepared_rows( torch.nn.Module.__init__(engram) engram.embed_tokens = layer engram._prefetch_stream = torch.cuda.Stream() if cpu_offload else None + engram._prefetch_done = torch.cuda.Event() if cpu_offload else None # Exercise the production eager boundaries even when the test process # imported Engram before breakable graphs were enabled. if capture == "breakable": diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index 61f977ddbd..df9d45a99d 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -24,6 +24,8 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses). """ +from typing import NamedTuple + import pytest import torch @@ -33,19 +35,30 @@ from vllm.platforms import current_platform FP8 = torch.float8_e4m3fn FP8_MAX = 448.0 -# GLM-5.2 / DeepSeek-V3.2 shapes (TP8 local heads). -Q_LORA = 2048 -KV_LORA = 512 -ROPE_DIM = 64 -NUM_HEADS = 8 -INDEX_HEADS = 32 -INDEX_HEAD_DIM = 128 -HIDDEN = 6144 EPS = 1e-6 + +class ModelConfig(NamedTuple): + """Per-model attention dims (num_heads is the TP8 local count).""" + + q_lora: int + kv_lora: int + rope_dim: int + num_heads: int + index_heads: int + index_head_dim: int + hidden: int + + +MODEL_CONFIGS = [ + ModelConfig(2048, 512, 64, 8, 32, 128, 6144), # GLM-5.2 + ModelConfig(1536, 512, 64, 16, 64, 128, 7168), # DeepSeek-V3.2 +] +MODEL_IDS = ["glm5.2", "dsv3.2"] + pytestmark = pytest.mark.skipif( - not current_platform.is_cuda() or not current_platform.has_device_capability(89), - reason="deepseek_v32 fused kernels require CUDA with fp8 (SM89+)", + not current_platform.supports_fp8(), + reason="deepseek_v32 fused kernels require fp8 support", ) @@ -144,29 +157,32 @@ def assert_fp8(got: torch.Tensor, ref: torch.Tensor, msg: str): # ── fused_norm_rope ────────────────────────────────────────────────────────── +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096]) @pytest.mark.parametrize("index_interleave", [True, False]) @pytest.mark.parametrize("mla_dtype", ["auto", "bfloat16", "fp8"]) -def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str): +def test_fused_norm_rope( + num_tokens: int, index_interleave: bool, mla_dtype: str, cfg: ModelConfig +): torch.manual_seed(0) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) - ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16) + ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) + ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) - mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) # MLA k_pe: interleaved - idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) # MLA k_pe: interleaved + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) bs = max_pos # single block covering all tokens - mla_dim = KV_LORA + ROPE_DIM + mla_dim = cfg.kv_lora + cfg.rope_dim mla_fp8 = mla_dtype == "fp8" if mla_fp8: mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.uint8) @@ -174,7 +190,7 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str else: mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.bfloat16) mla_k_scale = None - idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 # 132 + idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4 # 132 idx_cache = torch.zeros(1, bs, idx_row, device=dev, dtype=torch.uint8) slot = torch.arange(num_tokens, device=dev, dtype=torch.int64) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) @@ -213,20 +229,20 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str if mla_fp8: cache = mla_cache.view(FP8)[0, :num_tokens] s = mla_k_scale.item() - assert_fp8(cache[:, :KV_LORA], (kv_ref / s).to(FP8), "MLA kv fp8") - assert_fp8(cache[:, KV_LORA:], (kpe_ref / s).to(FP8), "MLA k_pe fp8") + assert_fp8(cache[:, : cfg.kv_lora], (kv_ref / s).to(FP8), "MLA kv fp8") + assert_fp8(cache[:, cfg.kv_lora :], (kpe_ref / s).to(FP8), "MLA k_pe fp8") else: cache = mla_cache[0, :num_tokens] - assert_bf16(cache[:, :KV_LORA], kv_ref, "MLA kv bf16") - assert_bf16(cache[:, KV_LORA:], kpe_ref, "MLA k_pe bf16") + assert_bf16(cache[:, : cfg.kv_lora], kv_ref, "MLA kv bf16") + assert_bf16(cache[:, cfg.kv_lora :], kpe_ref, "MLA k_pe bf16") # Indexer-K cache (packed [bs*head_dim fp8 | bs*4 fp32 scale]). ik_ref = layer_norm(ik, ikw, ikb) ik_ref = rope(ik_ref, pos, idx_cos_sin, interleave=index_interleave) q_ref, s_ref = ue8m0_quant(ik_ref) flat = idx_cache[0].reshape(-1) - vals = flat[: bs * INDEX_HEAD_DIM].view(FP8).reshape(bs, INDEX_HEAD_DIM) - scales = flat[bs * INDEX_HEAD_DIM :].view(torch.float32) + vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim) + scales = flat[bs * cfg.index_head_dim :].view(torch.float32) assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8") torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0) @@ -234,23 +250,24 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str assert (topk == -1).all(), "topk buffer not cleared on indexer layer" -def test_fused_norm_rope_packed_indexer_block_stride(): +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) +def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig): """Indexer writes use their HMA block stride and independent slot map.""" torch.manual_seed(7) dev = "cuda" num_tokens = block_size = 2 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) - ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - cos_sin = make_cos_sin(32, ROPE_DIM, dev) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16) + ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) + ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) + cos_sin = make_cos_sin(32, cfg.rope_dim, dev) - idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 + idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4 packed_block_stride = block_size * idx_row + 64 backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8) idx_cache = torch.as_strided( @@ -261,7 +278,7 @@ def test_fused_norm_rope_packed_indexer_block_stride(): mla_cache = torch.zeros( 1, block_size, - KV_LORA + ROPE_DIM, + cfg.kv_lora + cfg.rope_dim, device=dev, dtype=torch.bfloat16, ) @@ -296,15 +313,20 @@ def test_fused_norm_rope_packed_indexer_block_stride(): q_ref, s_ref = ue8m0_quant(ik_ref) packed = idx_cache[1].reshape(-1) values = ( - packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM) + packed[: block_size * cfg.index_head_dim] + .view(FP8) + .view(block_size, cfg.index_head_dim) ) - scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32) + scales = packed[block_size * cfg.index_head_dim :].view(torch.float32) assert_fp8(values, q_ref, "packed indexer-K fp8") torch.testing.assert_close(scales, s_ref, rtol=0, atol=0) assert (backing[block_size * idx_row : packed_block_stride] == 0).all() -def test_fused_norm_rope_materializes_qk_without_local_cache_slots(): +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) +def test_fused_norm_rope_materializes_qk_without_local_cache_slots( + cfg: ModelConfig, +): """DCP non-owner ranks still need valid query shards for query AllGather.""" torch.manual_seed(7) dev = "cuda" @@ -312,20 +334,20 @@ def test_fused_norm_rope_materializes_qk_without_local_cache_slots(): max_pos = 16 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) - ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16) + ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) + ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) + cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) mla_cache = torch.zeros( - 1, max_pos, KV_LORA + ROPE_DIM, device=dev, dtype=torch.bfloat16 + 1, max_pos, cfg.kv_lora + cfg.rope_dim, device=dev, dtype=torch.bfloat16 ) - idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 + idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4 idx_cache = torch.zeros(1, max_pos, idx_row, device=dev, dtype=torch.uint8) no_local_slots = torch.full((num_tokens,), -1, device=dev, dtype=torch.int64) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) @@ -371,23 +393,26 @@ def test_fused_norm_rope_materializes_qk_without_local_cache_slots(): assert (topk == -1).all(), "topk buffer not cleared on non-owner rank" +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 17, 512]) -def test_fused_norm_rope_no_indexer(num_tokens: int): +def test_fused_norm_rope_no_indexer(num_tokens: int, cfg: ModelConfig): """Shared (no-indexer) layer: q + kv/MLA only; top-k buffer untouched.""" torch.manual_seed(1) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) bs = max_pos - mla_cache = torch.zeros(1, bs, KV_LORA + ROPE_DIM, device=dev, dtype=torch.bfloat16) + mla_cache = torch.zeros( + 1, bs, cfg.kv_lora + cfg.rope_dim, device=dev, dtype=torch.bfloat16 + ) slot = torch.arange(num_tokens, device=dev, dtype=torch.int64) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) @@ -418,9 +443,9 @@ def test_fused_norm_rope_no_indexer(num_tokens: int): assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (no-indexer)") cache = mla_cache[0, :num_tokens] - assert_bf16(cache[:, :KV_LORA], rms_norm(kv_c, kvw), "MLA kv (no-indexer)") + assert_bf16(cache[:, : cfg.kv_lora], rms_norm(kv_c, kvw), "MLA kv (no-indexer)") assert_bf16( - cache[:, KV_LORA:], + cache[:, cfg.kv_lora :], rope(k_pe.float(), pos, mla_cos_sin, interleave=True), "MLA k_pe (no-indexer)", ) @@ -428,36 +453,39 @@ def test_fused_norm_rope_no_indexer(num_tokens: int): assert (topk == 7).all(), "topk buffer should be untouched on shared layer" +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("has_indexer", [False, True]) -def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool): +def test_fused_norm_rope_materializes_pcp_cache_inputs( + has_indexer: bool, cfg: ModelConfig +): """PCP gets local normalized/rotated K rows without direct cache writes.""" torch.manual_seed(6) dev = "cuda" num_tokens = 17 max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) ik = ( - torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) + torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16) if has_indexer else None ) ikw = ( - torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) if has_indexer else None ) ikb = ( - torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32) if has_indexer else None ) - mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) - idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None + mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) if has_indexer else None q_out = torch.empty_like(q_c) kv_out = torch.empty_like(kv_c) kpe_out = torch.empty_like(k_pe) @@ -503,8 +531,9 @@ def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool): assert_bf16(ik_out, ik_ref, "PCP indexer-K") +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) -def test_fused_norm_rope_ds_mla(num_tokens: int): +def test_fused_norm_rope_ds_mla(num_tokens: int, cfg: ModelConfig): """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). Per-token 656-byte entry: 512 fp8 NoPE (4 per-128 tiles, power-of-two scales @@ -515,12 +544,12 @@ def test_fused_norm_rope_ds_mla(num_tokens: int): max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) bs = max_pos mla_cache = torch.zeros(1, bs, 656, device=dev, dtype=torch.uint8) @@ -559,13 +588,13 @@ def test_fused_norm_rope_ds_mla(num_tokens: int): tiles = kv_ref.view(num_tokens, 4, 128) ref_scale = torch.clamp(tiles.abs().amax(dim=-1) / FP8_MAX, min=1e-4) ref_scale = torch.exp2(torch.ceil(torch.log2(ref_scale))) - ref_nope = (tiles / ref_scale[..., None]).reshape(num_tokens, KV_LORA).to(FP8) + ref_nope = (tiles / ref_scale[..., None]).reshape(num_tokens, cfg.kv_lora).to(FP8) cache = mla_cache[0, :num_tokens] # [N, 656] uint8 - nope = cache[:, :KV_LORA].view(FP8) - scales = cache.view(torch.float32)[:, KV_LORA // 4 : KV_LORA // 4 + 4] - rope_off = KV_LORA // 2 + 8 - rope_vals = cache.view(torch.bfloat16)[:, rope_off : rope_off + ROPE_DIM] + nope = cache[:, : cfg.kv_lora].view(FP8) + scales = cache.view(torch.float32)[:, cfg.kv_lora // 4 : cfg.kv_lora // 4 + 4] + rope_off = cfg.kv_lora // 2 + 8 + rope_vals = cache.view(torch.bfloat16)[:, rope_off : rope_off + cfg.rope_dim] torch.testing.assert_close(scales, ref_scale, rtol=0, atol=0) assert_fp8(nope, ref_nope, "ds_mla NoPE fp8") @@ -598,8 +627,9 @@ def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor: not current_platform.is_device_capability(100), reason="nvfp4_ds_mla requires SM100 (Blackwell)", ) +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) -def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): +def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int, cfg: ModelConfig): """nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only). Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble @@ -611,8 +641,8 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) + q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16) # Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3 # subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale # BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a @@ -622,10 +652,10 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): kv_c[:, :16] = ( 7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16) ) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16) - mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16) + qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16) + kvw = torch.ones(cfg.kv_lora, device=dev, dtype=torch.bfloat16) + mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) bs = max_pos mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8) @@ -675,19 +705,19 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): ) ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1)) - ref_codes = ref_codes.reshape(num_tokens, KV_LORA) + ref_codes = ref_codes.reshape(num_tokens, cfg.kv_lora) cache = mla_cache[0, :num_tokens] # [N, 352] uint8 - packed = cache[:, : KV_LORA // 2] + packed = cache[:, : cfg.kv_lora // 2] got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1) - got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem + got_codes = got_codes.reshape(num_tokens, cfg.kv_lora) # low nibble = even elem torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0) - got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8) + got_rope = cache[:, cfg.kv_lora // 2 : cfg.kv_lora // 2 + cfg.rope_dim].view(FP8) assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3") perm = nvfp4_sf_byte(torch.arange(32, device=dev)) - got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm] + got_scale = cache[:, cfg.kv_lora // 2 + cfg.rope_dim :].view(FP8)[:, perm] torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0) # No indexer on this call: top-k buffer must be untouched. @@ -737,39 +767,34 @@ def test_fused_norm_rope_supports_large_token_count(): # ── fused_q ────────────────────────────────────────────────────────────────── -@pytest.mark.parametrize( - ("num_tokens", "num_q_heads"), - [ - (1, NUM_HEADS), - (4, NUM_HEADS), - (17, NUM_HEADS), - (512, NUM_HEADS), - (4096, NUM_HEADS), - (17, 64), - ], -) +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) +@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096]) @pytest.mark.parametrize("index_interleave", [True, False]) -def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool): +def test_fused_q(num_tokens: int, index_interleave: bool, cfg: ModelConfig): torch.manual_seed(2) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos q_pe = torch.randn( - num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16 ) ql_nope = torch.randn( - num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16 ) index_q = torch.randn( - num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, + cfg.index_heads, + cfg.index_head_dim, + device=dev, + dtype=torch.bfloat16, ) - index_w = torch.randn(num_tokens, INDEX_HEADS, device=dev, dtype=torch.float32) + index_w = torch.randn(num_tokens, cfg.index_heads, device=dev, dtype=torch.float32) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) - softmax_scale = INDEX_HEAD_DIM**-0.5 - head_scale = INDEX_HEADS**-0.5 - q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) # q_pe: interleaved - idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + softmax_scale = cfg.index_head_dim**-0.5 + head_scale = cfg.index_heads**-0.5 + q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) # q_pe: interleaved + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) iq_fp8, iw_out, mqa = K.fused_q( pos, @@ -791,18 +816,18 @@ def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool): mqa_nope_ref = (ql_nope.float() / s).to(FP8) qpe_ref = rope( q_pe.float(), - pos.unsqueeze(-1).expand(num_tokens, num_q_heads), + pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads), q_cos_sin, interleave=True, ) mqa_pe_ref = (qpe_ref / s).to(FP8) - assert_fp8(mqa[:, :, :KV_LORA], mqa_nope_ref, "mqa ql_nope") - assert_fp8(mqa[:, :, KV_LORA:], mqa_pe_ref, "mqa q_pe") + assert_fp8(mqa[:, :, : cfg.kv_lora], mqa_nope_ref, "mqa ql_nope") + assert_fp8(mqa[:, :, cfg.kv_lora :], mqa_pe_ref, "mqa q_pe") # Indexer-Q: RoPE + UE8M0 fp8 quant; index weights fold in q-scale. iq_ref = rope( index_q.float(), - pos.unsqueeze(-1).expand(num_tokens, INDEX_HEADS), + pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads), idx_cos_sin, interleave=index_interleave, ) @@ -812,20 +837,21 @@ def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool): torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3) +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 17, 512]) -def test_fused_q_no_indexer(num_tokens: int): +def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig): torch.manual_seed(3) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) q_pe = torch.randn( - num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16 ) ql_nope = torch.randn( - num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16 ) q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32) - q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) _, _, mqa = K.fused_q( pos, q_pe, @@ -841,19 +867,20 @@ def test_fused_q_no_indexer(num_tokens: int): index_rope_interleave=False, ) s = q_scale.item() - assert_fp8(mqa[:, :, :KV_LORA], (ql_nope.float() / s).to(FP8), "mqa ql_nope") + assert_fp8(mqa[:, :, : cfg.kv_lora], (ql_nope.float() / s).to(FP8), "mqa ql_nope") qpe_ref = rope( q_pe.float(), - pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS), + pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads), q_cos_sin, interleave=True, ) - assert_fp8(mqa[:, :, KV_LORA:], (qpe_ref / s).to(FP8), "mqa q_pe") + assert_fp8(mqa[:, :, cfg.kv_lora :], (qpe_ref / s).to(FP8), "mqa q_pe") +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("has_indexer", [True, False]) -def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): +def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig): """bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is produced (bf16, unquantized); ql_nope is consumed directly by the caller.""" torch.manual_seed(6) @@ -862,21 +889,27 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos q_pe = torch.randn( - num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16 ) ql_nope = torch.randn( - num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16 + num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16 ) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) - q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) index_q = index_w = idx_cos_sin = None if has_indexer: index_q = torch.randn( - num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, + cfg.index_heads, + cfg.index_head_dim, + device=dev, + dtype=torch.bfloat16, ) - index_w = torch.randn(num_tokens, INDEX_HEADS, device=dev, dtype=torch.float32) - idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + index_w = torch.randn( + num_tokens, cfg.index_heads, device=dev, dtype=torch.float32 + ) + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) iq_fp8, iw_out, q_pe_out = K.fused_q( pos, @@ -887,8 +920,8 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): ql_nope, q_scale, index_w, - INDEX_HEAD_DIM**-0.5, - INDEX_HEADS**-0.5, + cfg.index_head_dim**-0.5, + cfg.index_heads**-0.5, has_indexer=has_indexer, index_rope_interleave=False, quantize_mqa=False, @@ -896,10 +929,10 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): # MQA query: only the RoPE'd q_pe, bf16, unquantized. assert q_pe_out.dtype == torch.bfloat16 - assert q_pe_out.shape == (num_tokens, NUM_HEADS, ROPE_DIM) + assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim) qpe_ref = rope( q_pe.float(), - pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS), + pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads), q_cos_sin, interleave=True, ) @@ -910,13 +943,15 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): assert index_q is not None iq_ref = rope( index_q.float(), - pos.unsqueeze(-1).expand(num_tokens, INDEX_HEADS), + pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads), idx_cos_sin, interleave=False, ) q_ref, scale_ref = ue8m0_quant(iq_ref) assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)") - iw_ref = index_w * scale_ref * (INDEX_HEAD_DIM**-0.5) * (INDEX_HEADS**-0.5) + iw_ref = ( + index_w * scale_ref * (cfg.index_head_dim**-0.5) * (cfg.index_heads**-0.5) + ) torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3) @@ -956,21 +991,22 @@ def test_fused_q_triton_supports_large_token_count(): # ── fused_eh_norm (MTP) ────────────────────────────────────────────────────── +@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) -def test_fused_eh_norm(num_tokens: int): +def test_fused_eh_norm(num_tokens: int, cfg: ModelConfig): torch.manual_seed(4) dev = "cuda" # Mix in a position-0 token to exercise the embeds-zeroing branch. pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos[0] = 0 - embeds = torch.randn(num_tokens, HIDDEN, device=dev, dtype=torch.bfloat16) - prev = torch.randn(num_tokens, HIDDEN, device=dev, dtype=torch.bfloat16) - ew = torch.randn(HIDDEN, device=dev, dtype=torch.bfloat16) - hw = torch.randn(HIDDEN, device=dev, dtype=torch.bfloat16) + embeds = torch.randn(num_tokens, cfg.hidden, device=dev, dtype=torch.bfloat16) + prev = torch.randn(num_tokens, cfg.hidden, device=dev, dtype=torch.bfloat16) + ew = torch.randn(cfg.hidden, device=dev, dtype=torch.bfloat16) + hw = torch.randn(cfg.hidden, device=dev, dtype=torch.bfloat16) out = K.fused_eh_norm(pos, embeds, prev, ew, hw, EPS) masked = torch.where(pos.unsqueeze(-1) == 0, torch.zeros_like(embeds), embeds) ref = torch.cat([rms_norm(masked, ew), rms_norm(prev, hw)], dim=-1) - assert out.shape == (num_tokens, 2 * HIDDEN) + assert out.shape == (num_tokens, 2 * cfg.hidden) assert_bf16(out, ref, "eh_norm") diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index 03d5b7921f..3b5b24fe49 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -21,7 +21,12 @@ Usage: import pytest import torch +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_fp8_weight_block, +) from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import fp8_einsum # -- Default dimensions matching DeepSeek V3/V4 -------------------------- HEAD_DIM = 512 @@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn EPS = 1e-10 +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103" +) +@pytest.mark.parametrize("groups", [2, 4]) +@pytest.mark.parametrize("tokens", [1, 6, 18, 32]) +def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens): + """Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize.""" + from flashinfer import mxfp8_quantize + + from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL + + torch.manual_seed(0) + n = groups * 1024 + x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128] + positions = torch.randint(0, 128, (tokens,), device="cuda") + rope = make_cos_sin_cache(128, device="cuda") + wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE) + scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float()) + wq, ws = deepgemm_post_process_fp8_weight_block( + wq, scale, (1, 32), False, True, groups + ) + q, sf = _FUSED_WO_A_KERNEL( + x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws + ) + + a, sa = fused_inv_rope_fp8_quant( + x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True + ) + y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) + fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) + ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") + + def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks] + return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1) + + actual, expected = ( + d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0) + for d, s in ((q, sf), (ref, ref_sf)) + ) + assert (actual - expected).norm() / expected.norm() < 3e-3 + assert not rows(sf)[tokens:].any(), "padded scale rows must be zero" + + # ========================================================================= # Helpers # ========================================================================= diff --git a/tests/kernels/test_kpool_decode_update_batched.py b/tests/kernels/test_kpool_decode_update_batched.py index 37ba4b58c8..b0fbcf7faf 100644 --- a/tests/kernels/test_kpool_decode_update_batched.py +++ b/tests/kernels/test_kpool_decode_update_batched.py @@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos): @pytest.mark.parametrize("pool_size", [4, 16]) -def test_decode_writer_matches_prefill_writer(pool_size): - """Compare production decode and prefill writers for pool sizes 4 and 16.""" +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_decode_writer_matches_prefill_writer(pool_size, ring_pools): + ring = ring_pools * pool_size n_pools, page, nblk = 8, 64, 4 n_tok = n_pools * pool_size dev = "cuda" @@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size): # One request owning tail block 0, fed one token per decode step. kv_decode = torch.zeros_like(kv_prefill) - tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev) + tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev) for t in range(n_tok): completes = t % pool_size == pool_size - 1 kpool_decode_update_and_maybe_write_cache_batched( kv_decode, tail, # token-granular: every token has a valid tail slot - torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev), + torch.tensor([[t % ring]], dtype=torch.int32, device=dev), k[t].view(1, 1, HEAD_DIM), score[t].view(1, 1, HEAD_DIM), ape, @@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size): ) +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_rejected_draft_redo_needs_ring_slots(ring_pools): + """With a one-pool ring, the drafts behind a rejected pool-completing draft + overwrote the pool's earlier keys, so its redo compressed wrong keys.""" + pool, spec, page, nblk = 4, 3, 64, 2 + ring = ring_pools * pool + dev = "cuda" + torch.manual_seed(1) + n_tok = 3 * pool + k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev) + score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev) + ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev) + kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev) + kpool_compress_and_write_cache( + kv_ref, + k.view(3, pool, HEAD_DIM), + score.view(3, pool, HEAD_DIM), + ape, + torch.arange(3, dtype=torch.int64, device=dev), + pool_size=pool, + head_dim=HEAD_DIM, + round_scale=ROUND_SCALE, + ) + + kv = torch.zeros_like(kv_ref) + tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev) + + def pool_bytes(cache, p): + # Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales]. + flat = cache[0].reshape(-1) + k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM] + s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)] + return torch.cat([k_bytes, s_bytes]) + + def step(positions, keys, scores): + pos = torch.tensor([positions], dtype=torch.int32, device=dev) + slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions] + kpool_decode_update_and_maybe_write_cache_batched( + kv, + tail, + pos % ring, + keys.view(1, -1, HEAD_DIM), + scores.view(1, -1, HEAD_DIM), + ape, + torch.tensor([slots], dtype=torch.int32, device=dev), + pos, + pool, + HEAD_DIM, + round_scale=ROUND_SCALE, + ) + + for t in range(7): + step([t], k[t], score[t]) + # Control: verified token 7 completes pool 1 before drafts 8..10 are stashed. + drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev) + draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev) + step( + [7, 8, 9, 10], + torch.cat([k[7:8], drafts]), + torch.cat([score[7:8], draft_scores]), + ) + step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected + for p in (1, 2): + assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p + + # Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts + # 8 and 9 overwrite the slots of positions 4 and 5, which are read by the + # redo of 7. + kv.zero_() + tail.zero_() + for t in range(6): + step([t], k[t], score[t]) + step( + [6, 7, 8, 9], + torch.cat([k[6:7], drafts]), + torch.cat([score[6:7], draft_scores]), + ) + step([7, 8, 9, 10], k[7:11], score[7:11]) + pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1)) + if ring_pools == 1: + assert not pool1_ok, "expected a one-pool ring to corrupt pool 1" + else: + assert pool1_ok + + def test_leading_invalid_tail_slot(): """A request whose FIRST token carries an invalid (-1) tail slot while a later token is a real pool completion. diff --git a/tests/kernels/test_minimax_m3_amd_ops.py b/tests/kernels/test_minimax_m3_amd_ops.py index 27bd74ce41..ffb2a5d778 100644 --- a/tests/kernels/test_minimax_m3_amd_ops.py +++ b/tests/kernels/test_minimax_m3_amd_ops.py @@ -431,7 +431,7 @@ def test_mxfp8_linear_emulation_bf16_at_load( layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False) layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False) - kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(N, K))) kernel.process_weights_after_loading(layer) if dequant_at_load: @@ -482,7 +482,9 @@ def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape): layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False) return layer - kernel = rocm_native.RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel = rocm_native.RocmDotScaledMxfp8LinearKernel( + Mxfp8LinearLayerConfig(weight_shape=(N, K)) + ) layer = make_layer() kernel.process_weights_after_loading(layer) assert (layer.weight.element_size() >= 2) is not aligned diff --git a/tests/kernels/utils.py b/tests/kernels/utils.py index cf68565e9d..1c1ffc2901 100644 --- a/tests/kernels/utils.py +++ b/tests/kernels/utils.py @@ -2,11 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Kernel test utils.""" -import itertools -import random from collections.abc import Sequence -from numbers import Number -from typing import Any, NamedTuple +from typing import Any from unittest.mock import patch import torch @@ -17,8 +14,6 @@ from vllm.model_executor.custom_op import op_registry from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input -from vllm.utils.torch_utils import make_tensor_with_pad -from vllm.v1.attention.backend import AttentionType # For now, disable "test_aot_dispatch_dynamic" since there are some # bugs related to this test in PyTorch 2.4. @@ -93,721 +88,6 @@ def _assert_deterministic( ) -class QKVInputs(NamedTuple): - """Data structure for representing unpacked attention inputs, - query/key/values and their sequence lengths. - - Attributes: - * {query,key,value}: unpacked (batch_size x padded_seq_len x - num_heads x head_size) attention inputs - * q_seq_lens: query sequence lengths list - * kv_seq_lens: shared key/value sequence lengths list - - """ - - query: torch.Tensor - key: torch.Tensor - value: torch.Tensor - q_seq_lens: list[int] - kv_seq_lens: list[int] - - -class QKVO(NamedTuple): - """Data structure for representing unpacked attention inputs, - alongside unpacked known-correct attention output - - Attributes: - * qkv: unpacked (batch_size x padded_seq_len x - num_heads x head_size) attention inputs - * ideal_output: unpacked (batch_size x padded_seq_len x - num_heads x head_size) known-correct attention output - - """ - - qkv: QKVInputs - ideal_output: torch.Tensor - - -class PackedQKVInputs(NamedTuple): - """Data structure for representing packed attention inputs. - - Attributes: - * {query,key,value}: packed (number_of_tokens x num_heads - x head_size) attention inputs - * q_start_loc_list: list of query start locations within packed tensor - * kv_start_loc_list: shared list of key/value start locations within - packed tensor - * q_seq_lens: query sequence lengths list - * kv_seq_lens: shared key/value sequence lengths list - - """ - - query: torch.Tensor - key: torch.Tensor - value: torch.Tensor - q_start_loc_list: list[int] | None - kv_start_loc_list: list[int] | None - q_seq_lens: list[int] | None - kv_seq_lens: list[int] | None - - -class PackedQKVO(NamedTuple): - """Data structure for representing packed attention inputs, - alongside packed known-correct attention output - - Attributes: - * packed_qkv: packed (number_of_tokens x num_heads - x head_size) attention inputs - * ideal_output: packed (number_of_tokens x num_heads - x head_size) known-correct attention output - - """ - - packed_qkv: PackedQKVInputs | None - ideal_output: torch.Tensor - - -class KVMemoryMap(NamedTuple): - """Data structure for encapsulating KV cache memory mapping. - - Attributes: - * block_tables: KV cache block tables - * slot_mapping: mapping of sequence offset to physical address - - """ - - block_tables: torch.Tensor - slot_mapping: torch.Tensor - - -class PhaseTestParameters(NamedTuple): - """Data structure for encapsulating the test parameters - for a given test "phase" (prefill or decode phase) and attention - scenario (encoder, decoder-self, encoder/decoder-cross) - - Attributes: - * packed_qkvo: packed (number_of_tokens x num_heads - x head_size) attention inputs & known-correct - output - * kv_mmap: KV cache memory mapping, specific to this test phase & - attention scenario - - """ - - packed_qkvo: PackedQKVO - kv_mmap: KVMemoryMap | None - - -def maybe_make_int_tensor( - _list: list[int] | None, - device: torch.device | str, -) -> torch.Tensor: - """Convert Python int list to a 1D int torch.Tensor on `device`. - - Returns: - If _list is not None: 1D int torch.Tensor on `device`, `None` otherwise. - - """ - return ( - None if _list is None else torch.tensor(_list, dtype=torch.int, device=device) - ) - - -def maybe_make_long_tensor( - _list: list[int] | None, - device: torch.device | str, -) -> torch.Tensor: - """Convert Python int list to a 1D long torch.Tensor on `device`. - - Returns: - If _list is not None: 1D long torch.Tensor on `device`, `None` otherwise. - - """ - return ( - None if _list is None else torch.tensor(_list, dtype=torch.long, device=device) - ) - - -def maybe_max(_list: list | None) -> Number | None: - """Compute the maximum of a list, if it exists. - - Returns: - If _list is not None: max(_list), `None` otherwise. - - """ - return None if _list is None else max(_list) - - -def make_causal_mask( - q_max_seq_len: int, - kv_max_seq_len: int, -) -> torch.Tensor: - """Create a q_max_seq_len x kv_max_seq_len causal mask. - - Arguments: - q_max_seq_len: query max seq len - kv_max_seq_len: key/value max seq len - - Returns: - 2D tensor, q_max_seq_len x kv_max_seq_len - - """ - # Create a matrix where entry (i, j) is True if i >= j - mask = torch.triu(torch.ones(q_max_seq_len, kv_max_seq_len), diagonal=1) - # Replace True with float('-inf') and False with 0 - mask = mask.masked_fill(mask == 1, float("-inf")).masked_fill(mask == 0, 0.0) - return mask - - -def ref_masked_attention( - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - scale: float, - custom_mask: torch.Tensor | None = None, - q_seq_lens: list | None = None, - kv_seq_lens: list | None = None, -) -> torch.Tensor: - """Reference ("golden") masked attention. Supports two types of masking: - - * Basic attention mask, utilizing {q,kv}_seq_lens args to mask out - padding elements - * Custom attention mask, which can force an arbitrary mask tensor, i.e. - causal - - Arguments: - query: batch_size x q_padded_seq_len x num_heads x head_size - key: batch_size x kv_padded_seq_len x num_heads x head_size - value: batch_size x kv_padded_seq_len x num_heads x head_size - scale: Attention scale factor - custom_mask: custom attention mask; good place to inject a causal attention mask - q_seq_lens: list of unpadded query seq_lens for each batch index - kv_seq_lens: list of unpadded key/value seq_lens for each batch index - - Returns: - Attention result, batch_size x q_padded_seq_len x num_heads x head_size - - """ - assert q_seq_lens is not None - assert kv_seq_lens is not None - - batch_size = query.shape[0] - assert len(q_seq_lens) == batch_size - assert len(kv_seq_lens) == batch_size - - attn_weights = scale * torch.einsum("bqhd,bkhd->bhqk", query, key).float() - - # Basic attention mask, derived from seq lens - if (q_seq_lens is not None) or (kv_seq_lens is not None): - attn_mask = torch.zeros_like(attn_weights) - if q_seq_lens is not None: - for bdx, plen in enumerate(q_seq_lens): - attn_mask[bdx, :, plen:, :] = -torch.inf - if kv_seq_lens is not None: - for bdx, plen in enumerate(kv_seq_lens): - attn_mask[bdx, :, :, plen:] = -torch.inf - - attn_weights = attn_weights + attn_mask.float() - - # Custom attention mask - if custom_mask is not None: - attn_weights = attn_weights + custom_mask.float() - - attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype) - out = torch.einsum("bhqk,bkhd->bqhd", attn_weights, value) - return out - - -def make_qkv( - batch_size: int, - max_q_seq_len: int, - max_kv_seq_len: int | None, - num_heads: int, - head_size: int, - device: torch.device | str, - force_kv_seq_lens: list[int] | None = None, - attn_type: AttentionType = AttentionType.ENCODER_DECODER, - force_max_len: bool = False, -) -> tuple[QKVInputs, QKVInputs, QKVInputs]: - """Construct QKV test tensors for self- and cross-attention. - - Generates three query/key/value triplets: - - * "Baseline" query/key/value (for input to reference attention function) - * "Prefill" query/key/value (last sequence offset zero'd out, for use as - input to prefill kernel) - * "Decode" query/key/value (only the last sequence offset from baseline, - for use as input to decode kernel) - - Each Q/K/V triplet is associated with a list of q seqlens and a list of k/v - seqlens - - Arguments: - batch_size: number of sequences in the batch - max_q_seq_len: max query seq len - max_kv_seq_len: max key/value seq len - num_heads: number of attention heads - head_size: head dimension - is_encoder_decoder_attn: if True, query seqlen may differ from - key/value seqlen (as is often the case for cross-attention); - o/w, query/key/value seqlens match at each batch index - (max_kv_seq_len is unused) - force_kv_seq_lens: if not None, overrides kv sequence lengths - attn_type: encoder, decoder self, or enc/dec cross attention - force_max_len: if True, all query seqlens are max_q_seq_len; o/w query - seqlens are random in [2,max_q_seq_lens]. Same for key/value seqlens - and max_kv_seq_len, unless forced by is_encoder_decoder_attn=False - device: CPU or CUDA device - - Returns: - Overall QKVInputs structure (containing full unpacked Q/K/V tensors) - Prefill QKVInputs structure (containing all but the last sequence offset) - Decode QKVInputs structure (containing all only the last sequence offset) - - """ - if force_max_len: - q_seq_lens = [max_q_seq_len for _ in range(batch_size)] - else: - q_seq_lens = [random.randint(2, max_q_seq_len) for _ in range(batch_size)] - kv_seq_lens = None - if force_kv_seq_lens is not None: - kv_seq_lens = force_kv_seq_lens - elif attn_type != AttentionType.ENCODER_DECODER: - # K,V seq lens match Q for self-attention - kv_seq_lens = q_seq_lens - else: - # K,V seq lens are distinct from Q seq lens & random - assert max_kv_seq_len is not None - if force_max_len: - kv_seq_lens = [max_kv_seq_len] * batch_size - else: - kv_seq_lens = [random.randint(2, max_kv_seq_len) for _ in range(batch_size)] - - query = torch.rand((batch_size, max_q_seq_len, num_heads, head_size)).to(device) - key = torch.rand((batch_size, max_kv_seq_len, num_heads, head_size)).to(device) - value = torch.rand((batch_size, max_kv_seq_len, num_heads, head_size)).to(device) - - prefill_query = torch.zeros((batch_size, max_q_seq_len, num_heads, head_size)).to( - device - ) - prefill_key = torch.zeros((batch_size, max_kv_seq_len, num_heads, head_size)).to( - device - ) - prefill_value = torch.zeros((batch_size, max_kv_seq_len, num_heads, head_size)).to( - device - ) - - decode_query = torch.zeros((batch_size, 1, num_heads, head_size)).to(device) - decode_key = torch.zeros((batch_size, 1, num_heads, head_size)).to(device) - decode_value = torch.zeros((batch_size, 1, num_heads, head_size)).to(device) - - for bdx, (q_seq_len, kv_seq_len) in enumerate(zip(q_seq_lens, kv_seq_lens)): - query[bdx, q_seq_len:, :, :] = 0 - key[bdx, kv_seq_len:, :, :] = 0 - value[bdx, kv_seq_len:, :, :] = 0 - - prefill_query[bdx, 0 : (q_seq_len - 1), :, :] = query[ - bdx, 0 : (q_seq_len - 1), :, : - ] - prefill_key[bdx, 0 : (kv_seq_len - 1), :, :] = key[ - bdx, 0 : (kv_seq_len - 1), :, : - ] - prefill_value[bdx, 0 : (kv_seq_len - 1), :, :] = value[ - bdx, 0 : (kv_seq_len - 1), :, : - ] - - decode_query[bdx, :, :, :] = query[bdx, (q_seq_len - 1) : q_seq_len, :, :] - decode_key[bdx, :, :, :] = key[bdx, (kv_seq_len - 1) : kv_seq_len, :, :] - decode_value[bdx, :, :, :] = value[bdx, (kv_seq_len - 1) : kv_seq_len, :, :] - - prefill_q_seq_lens = [plen - 1 for plen in q_seq_lens] - prefill_kv_seq_lens = [plen - 1 for plen in kv_seq_lens] - - decode_q_seq_lens = [1 for _ in q_seq_lens] - decode_kv_seq_lens = [1 for _ in kv_seq_lens] - - return ( - QKVInputs( - query, # Overall QKV inputs - key, - value, - q_seq_lens, - kv_seq_lens, - ), - QKVInputs( - prefill_query, # Prefill subset of QKV sequences - prefill_key, - prefill_value, - prefill_q_seq_lens, - prefill_kv_seq_lens, - ), - QKVInputs( - decode_query, # Decode subset of KV sequences - decode_key, - decode_value, - decode_q_seq_lens, - decode_kv_seq_lens, - ), - ) - - -def pack_tensor( - unpacked_tensor: torch.Tensor, seq_lens: list[int], device: torch.device | str -) -> tuple[torch.Tensor, list[int]]: - """Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an - unpadded number_of_tokens x num_heads x head_size tensor, where - number_of_tokens = sum(seq_lens) - - Arguments: - unpacked_tensor: batch_size x padded_seq_len x num_heads x head_size - seq_lens: list of token counts for each seq - device: CPU or CUDA device - - Returns: - packed_tensor: number_of_tokens x num_heads x head_size - start_loc_list: start idx of each batch elt in packed_tensor; [0] + - list(itertools.accumulate(seq_lens)) - - """ - num_tok = sum(seq_lens) - num_heads = unpacked_tensor.shape[-2] - head_size = unpacked_tensor.shape[-1] - start_loc_list = [0] + list(itertools.accumulate(seq_lens)) - packed_tensor = torch.zeros((num_tok, num_heads, head_size), device=device) - - for bdx, (seq_len, start_loc) in enumerate(zip(seq_lens, start_loc_list)): - packed_tensor[start_loc : (start_loc + seq_len), :, :] = unpacked_tensor[ - bdx, :seq_len, :, : - ] - - return packed_tensor, start_loc_list - - -def pack_qkv(qkv: QKVInputs, device: torch.device | str) -> PackedQKVInputs: - """Individually pack each of Q, K and V, each with dimensions batch_size x - padded_seq_len x num_heads x head_size, into respective number_of_tokens x - num_heads x head_size tensors. - - For Q, number_of_tokens = sum(q_seq_lens). - - For K and V, number_of_tokens = sum(kv_seq_lens) - - Arguments: - qkv: Unpacked (batch_size x padded_seq_len x num_heads x head_size) - attention inputs - device: CPU or CUDA device - - Returns: - Packed (number_of_tokens x num_heads x head_size) QKV inputs - derived from unpacked inputs - - """ - if qkv.query is None: - packed_query = None - q_start_loc_list = None - else: - packed_query, q_start_loc_list = pack_tensor( - qkv.query, qkv.q_seq_lens, device=device - ) - packed_key, kv_start_loc_list = pack_tensor(qkv.key, qkv.kv_seq_lens, device=device) - packed_value, _ = pack_tensor(qkv.value, qkv.kv_seq_lens, device=device) - return PackedQKVInputs( - packed_query, - packed_key, - packed_value, - q_start_loc_list, - kv_start_loc_list, - (None if q_start_loc_list is None else qkv.q_seq_lens), - qkv.kv_seq_lens, - ) - - -def _make_metadata_tensors( - seq_lens: list[int] | None, - context_lens: list[int] | None, - encoder_seq_lens: list[int] | None, - device: torch.device | str, -) -> tuple[ - torch.Tensor, - torch.Tensor, - Any, - Any, - torch.Tensor | None, - torch.Tensor, - torch.Tensor, - int | None, -]: - """Build scalar & tensor values required to build attention metadata structure. - - Arguments: - seq_lens: list of token-counts for each decoder input seq - context_lens: list of context length values for each seq - encoder_seq_lens: list of token-counts for each encoder input seq - device: CPU or CUDA device - - Returns: - seq_lens_tensor: decoder seq_lens list, as tensor - context_lens_tensor: context_lens list, as tensor - max_context_len: max(context_lens) - max_seq_len: max(seq_lens) - seq_start_loc: start idx of each sequence - encoder_seq_lens_tensor: encoder seq_lens list, as tensor - encoder_seq_start_loc: start idx of each encoder sequence - max_encoder_seq_len: encoder seq_lens list, as tensor - - """ - seq_lens_tensor = maybe_make_int_tensor(seq_lens, device) - context_lens_tensor = maybe_make_int_tensor(context_lens, device) - max_context_len = maybe_max(context_lens) - max_seq_len = maybe_max(seq_lens) - - encoder_seq_lens_tensor = maybe_make_int_tensor(encoder_seq_lens, device) - max_encoder_seq_len = None if encoder_seq_lens is None else max(encoder_seq_lens) - - seq_start_loc = None - - if seq_lens_tensor is not None: - seq_start_loc = torch.zeros( - seq_lens_tensor.shape[0] + 1, - dtype=torch.int32, - device=seq_lens_tensor.device, - ) - torch.cumsum( - seq_lens_tensor, dim=0, dtype=seq_start_loc.dtype, out=seq_start_loc[1:] - ) - - encoder_seq_start_loc = torch.zeros( - encoder_seq_lens_tensor.shape[0] + 1, - dtype=torch.int32, - device=encoder_seq_lens_tensor.device, - ) - torch.cumsum( - encoder_seq_lens_tensor, - dim=0, - dtype=encoder_seq_start_loc.dtype, - out=encoder_seq_start_loc[1:], - ) - - return ( - seq_lens_tensor, - context_lens_tensor, - max_context_len, - max_seq_len, - seq_start_loc, - encoder_seq_lens_tensor, - encoder_seq_start_loc, - max_encoder_seq_len, - ) - - -def make_kv_cache( - num_blocks: int, - num_heads: int, - head_size: int, - block_size: int, - device: torch.device | str, - backend: str, - default_val: float = 0.0, -) -> torch.Tensor: - """Create a fake KV cache. - - Arguments: - num_blocks: number of blocks in the KV cache - num_heads: number of attention heads - head_size: head dimension - block_size: number of offsets within a block - device: CPU or CUDA device - backend: attention backend name; only 'FLASH_ATTN' is supported - default_val: initialization value for KV cache elements - - Returns: - kv_cache: 2 x num_blocks x block_size x num_heads x head_size - for backend 'FLASH_ATTN' - - """ - if backend != "FLASH_ATTN": - raise ValueError(f"Unknown backend value: '{backend}'. Expected 'FLASH_ATTN'.") - kv_cache = torch.rand((2, num_blocks, block_size, num_heads, head_size)).to(device) - if default_val is not None: - kv_cache[:, :, :] = default_val - return kv_cache - - -def _num_tokens_to_min_blocks(num_tokens: int, block_size: int) -> int: - """Compute the minimum number of blocks required to hold num_tokens tokens, - given block_size - """ - return (num_tokens + block_size - 1) // block_size - - -def make_empty_slot_mapping_tensor(device: torch.device | str): - return maybe_make_long_tensor([], device) - - -def make_empty_block_tables_tensor(device: torch.device | str): - return torch.tensor([], device=device) - - -def split_slot_mapping( - slot_mapping_list: torch.Tensor, - seq_lens: list[int], - device: torch.device | str, -): - """Split a slot mapping into valid prefill- and decode-phase slot mappings. - - Context: - * Your goal is to test (1) prefill of N prompts, with prompt-lengths - {K_i \\forall i \\in [0,N)}, followed by (2) decoding of a single token - for all N prompts (N tokens total); the resultant sequence lengths - after decode would be {K_i + 1 for i \\in [0,N)} - * The test you want to do requires (1) having the prefill slot mapping - for all tokens present during prefill, the number of which is - M = \\sum_i{K_i}, and (2) having the decode slot mapping for all N - decoded tokens - - This function consumes a single 1D slot mapping, which is the - concatenation of N slot mappings each of length K_i + 1 (corresponding - to the sequence lengths after decode), with a total length of - P = \\sum_i{K_i + 1} = M + N - - The prefill-phase slot mapping results from excising the (K_i + 1)-th entry - from each of the N subsequences in the slot mapping (i.e. omitting the - decoded token's mapping.) - - The N excised entries are appended to obtain the decode-phase slot mapping - - Arguments: - slot_mapping_list: Length-P 1D slot mapping (as list) reflecting all N - post-decode sequences - seq_lens: list of N post-decode sequence lengths (K_i + 1 in the - description above) - device: cuda, cpu, etc. - - Returns: - prefill_slot_mapping: Length-M 1D slot mapping (as Tensor) - reflecting all N prefill prompts - decode_slot_mapping: Length-N 1D slot mapping (as Tensor) reflecting - all N decoded tokens - - """ - prefill_slot_mapping = [] - decode_slot_mapping = [] - - base_idx = 0 - for seq_len in seq_lens: - prefill_slot_mapping.extend( - slot_mapping_list[base_idx : (base_idx + seq_len - 1)] - ) - decode_slot_mapping.append(slot_mapping_list[base_idx + seq_len - 1]) - base_idx += seq_len - - return ( - maybe_make_long_tensor(prefill_slot_mapping, device), - maybe_make_long_tensor(decode_slot_mapping, device), - ) - - -def make_block_tables_slot_mapping( - block_size: int, - seq_lens: list[int], - device: torch.device | str, - block_base_addr: int = 0, -) -> tuple[torch.Tensor, list[int], int]: - """Construct fake block tables & slot mappings. - - For a sequence with num_tokens tokens the minimum number - of required KV cache blocks is - - num_blocks = (num_tokens + block_size - 1) // block_size - - Then the minimum KV cache size in blocks is - - total_cache_blocks = sum(num_blocks for all seqs) - - Then, the blocktable mapping counts downward from - - block_base_addr + total_cache_blocks - - to - - block_base_addr - - - The constructed block-tables and slot-mapping are sized to the - lengths of the sequences in their entirety (as reflected by seq_lens), - i.e. the total of prefill prompt tokens + decoded tokens. - - Arguments: - block_size: number of offsets per block - seq_lens: list of token-counts for each sequence - block_base_addr: the block table base address - device: CPU or CUDA device - - Return: - * block_tables_tensor: block table for sequence - * slot_mapping_list: slot mapping for sequence - * max_block_idx: the highest block address within this block table - - """ - # Provision minimum number of KV cache blocks - num_blocks_list = [ - _num_tokens_to_min_blocks(num_tokens, block_size) for num_tokens in seq_lens - ] - max_block_table_len = max(num_blocks_list) - block_table_pad_tokens = 10 - - block_tables = [] - slot_mapping_list = [] - # Compute uppermost address of block table - total_cache_blocks = sum(num_blocks_list) - block_base_idx = block_base_addr + total_cache_blocks - max_block_idx = block_base_idx - for sdx, num_tokens in enumerate(seq_lens): - num_blocks = num_blocks_list[sdx] - block_table = list(range(block_base_idx, block_base_idx - num_blocks, -1)) - for idx in range(num_tokens): - mapping_value = (idx % block_size) + block_table[ - idx // block_size - ] * block_size - slot_mapping_list.append(mapping_value) - - block_base_idx -= num_blocks - block_tables.append(block_table) - - block_tables_tensor = make_tensor_with_pad( - block_tables, - max_len=max_block_table_len + block_table_pad_tokens, - pad=0, - dtype=torch.int, - device=device, - ) - - return (block_tables_tensor, slot_mapping_list, max_block_idx) - - -def assert_actual_matches_ideal( - test_params: PhaseTestParameters, output_under_test: torch.Tensor, backend: str -) -> None: - """Assert that observed output matches the ideal output - contained in the test parameters data structure. - - Arguments: - test_params: Test parameters including packed ideal output - output_under_test: actually observed output value - backend: attention backend name; only 'FLASH_ATTN' is supported - - """ - ideal_output = test_params.packed_qkvo.ideal_output - if backend != "FLASH_ATTN": - raise ValueError(f"Unknown backend value: '{backend}'. Expected 'FLASH_ATTN'.") - # For FlashAttention override the accuracy thresholds to non default - # values since we notice a higher difference between the ideal and - # actual output. - torch.testing.assert_close( - ideal_output, output_under_test.view_as(ideal_output), atol=0.01, rtol=0.016 - ) - - # Copied/modified from torch._refs.__init__.py def fp8_allclose( a: TensorLikeType, diff --git a/tests/model_executor/kernels/test_b12x_linear.py b/tests/model_executor/kernels/test_b12x_linear.py index 9d4f9beeb9..ac72cfb845 100644 --- a/tests/model_executor/kernels/test_b12x_linear.py +++ b/tests/model_executor/kernels/test_b12x_linear.py @@ -66,7 +66,7 @@ from vllm.platforms import PlatformEnum "MarlinMxfp8LinearKernel", "EmulationMxfp8LinearKernel", init_mxfp8_linear_kernel, - {}, + {"weight_shape": (2048, 2048)}, ), ( B12xTensorFP8ScaledMMLinearKernel, @@ -365,7 +365,7 @@ def test_b12x_tensor_fp8_apply_quantizes_and_uses_packed_weight( def test_b12x_mxfp8_can_implement_supported_config() -> None: can_implement, reason = B12xMxfp8LinearKernel.can_implement( - Mxfp8LinearLayerConfig() + Mxfp8LinearLayerConfig(weight_shape=(256, 512)) ) assert can_implement diff --git a/tests/model_executor/layers/test_conv.py b/tests/model_executor/layers/test_conv.py new file mode 100644 index 0000000000..22ecc00a29 --- /dev/null +++ b/tests/model_executor/layers/test_conv.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +from torch import nn + +from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer + + +@pytest.mark.parametrize( + "input_shape,bias", + [ + pytest.param((256, 3, 14, 14), False, id="packed_patches"), + pytest.param((2, 3, 28, 28), True, id="multiple_spatial_patches"), + ], +) +def test_conv2d_patch_embedding_correctness(default_vllm_config, input_shape, bias): + device = "cuda" if torch.cuda.is_available() else "cpu" + dtype = torch.bfloat16 if device == "cuda" else torch.float32 + reference = nn.Conv2d(3, 1024, 14, stride=14, bias=bias).to( + device=device, dtype=dtype + ) + layer = Conv2dLayer(3, 1024, 14, stride=14, bias=bias, params_dtype=dtype).to( + device + ) + layer.load_state_dict(reference.state_dict(), strict=True) + assert layer.enable_linear + + x = torch.randn(input_shape, device=device, dtype=dtype) + expected = reference(x) + atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5 + torch.testing.assert_close(layer.forward_native(x), expected, atol=atol, rtol=rtol) + + +@pytest.mark.parametrize( + "input_shape,kernel_size,out_channels,bias", + [ + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1280, False, id="minimax_m3"), + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, False, id="qwen2_vl"), + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, True, id="qwen3_vl"), + pytest.param((4, 3, 1, 14, 14), (1, 14, 14), 1536, True, id="glm_vl"), + pytest.param((1, 3, 4, 28, 28), (2, 14, 14), 32, True, id="multiple_patches"), + ], +) +def test_conv3d_patch_embedding_correctness( + default_vllm_config, input_shape, kernel_size, out_channels, bias +): + device = "cuda" if torch.cuda.is_available() else "cpu" + dtype = torch.bfloat16 if device == "cuda" else torch.float32 + reference = nn.Conv3d( + input_shape[1], out_channels, kernel_size, stride=kernel_size, bias=bias + ).to(device=device, dtype=dtype) + layer = Conv3dLayer( + input_shape[1], + out_channels, + kernel_size, + stride=kernel_size, + bias=bias, + params_dtype=dtype, + ).to(device) + layer.load_state_dict(reference.state_dict(), strict=True) + assert layer.enable_linear + + x = torch.randn(input_shape, device=device, dtype=dtype) + expected = reference(x) + atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5 + torch.testing.assert_close(layer(x), expected, atol=atol, rtol=rtol) diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 08fb6818e3..42f134e6de 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None: def _fp8_config(**kwargs: Any) -> Fp8Config: return Fp8Config( - is_checkpoint_fp8_serialized=True, activation_scheme="dynamic", weight_block_size=[128, 128], **kwargs, @@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None: def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None: """Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them.""" compatible, reason = is_shared_expert_quant_fse_compatible( - Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"), + Fp8Config(activation_scheme="dynamic"), "model.layers.0.mlp.experts", "model.layers.0.mlp.shared_experts", ) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 340d59a392..77c52fb291 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -677,6 +677,40 @@ def test_candidate_blocks_to_sparse_indices_matches_reference( assert not end_buf[1 - end_offset :: 2].any() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_sparse_topk_remap_bounds_nan_rows(): + """DeepSelect writes an out-of-range sentinel to slot 0 of NaN rows (the + rest is left untouched); the remap turns it into -1.""" + from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import ( + has_deep_select, + sparse_topk_remap, + ) + + if not has_deep_select(): + pytest.skip("DeepSelect extension (vllm._deepselect_C) required") + rows, num_sparse, sbk, topk = 4, 128, 8, 512 + width = num_sparse * sbk + logits = torch.randn(rows, width, dtype=torch.bfloat16, device="cuda") + logits[1:, :16] = float("nan") + sparse_indices = torch.arange(num_sparse, dtype=torch.int32, device="cuda") + end = torch.full((rows,), width, dtype=torch.int32, device="cuda") + out = torch.empty(rows, topk, dtype=torch.int32, device="cuda") + cols = torch.zeros_like(out) + + sparse_topk_remap( + logits, + sparse_indices.repeat(rows, 1), + end, + torch.zeros_like(end), + sbk, + topk, + out, + col_indices=cols, + ) + assert (cols[1:, 0] >= width).all() and (out[1:, 0] == -1).all() + assert (out[0] >= 0).all() + + def _skip_unless_sm100_sparse_kernels(): if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: pytest.skip("SM100 required") diff --git a/tests/model_executor/layers/test_mm_input_norm.py b/tests/model_executor/layers/test_mm_input_norm.py new file mode 100644 index 0000000000..fc114eccd0 --- /dev/null +++ b/tests/model_executor/layers/test_mm_input_norm.py @@ -0,0 +1,320 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Tests for FusedMMInputNorm and the fused affine transform kernel.""" + +import pytest +import torch + +from vllm.model_executor.layers.fusion.mm_input_norm import ( + FusedMMInputNorm, + IdentityInputNorm, + NormParams, + fused_mm_input_norm_triton, +) +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON +from vllm.utils.torch_utils import set_random_seed + +# Module-level tests run on any platform: the CPU platform exercises +# forward_native, accelerators exercise the fused kernel. Only the raw +# kernel tests require an accelerator. +_DEVICE = torch.device(current_platform.device_type) + +requires_accelerator = pytest.mark.skipif( + current_platform.is_cpu(), + reason="fused Triton kernel requires a CUDA/XPU accelerator", +) +requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton") +requires_vllm_config = pytest.mark.usefixtures("default_vllm_config") + + +def _reference_input_norm( + pixel_values: torch.Tensor, + image_mean: list[float], + image_std: list[float], + rescale_factor: float, + channel: int, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Straightforward per-channel affine: (x * rescale - mean) / std.""" + patches, size = pixel_values.shape + patch_size = size // channel + mean = torch.tensor( + image_mean, dtype=torch.float32, device=pixel_values.device + ).view(1, channel, 1) + std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view( + 1, channel, 1 + ) + x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size) + x = (x * rescale_factor - mean) / std + return x.view(patches, size).to(out_dtype) + + +# CLIP-style RGB normalization constants reused across tests. +_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073] +_RGB_STD = [0.26862954, 0.26130258, 0.27577711] +_RGB_RESCALE = 1.0 / 255.0 + + +def _check_against_reference( + pixel_values: torch.Tensor, + out_dtype: torch.dtype, + channel: int = 3, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + rescale_factor: float = _RGB_RESCALE, + **close_kwargs, +): + """Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the + eager per-channel affine reference. Uses the RGB constants by default.""" + if image_mean is None: + image_mean = list(_RGB_MEAN) + if image_std is None: + image_std = list(_RGB_STD) + norm = FusedMMInputNorm( + image_mean=image_mean, + image_std=image_std, + rescale_factor=rescale_factor, + channel=channel, + ).to(_DEVICE) + expected = _reference_input_norm( + pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype + ) + torch.testing.assert_close( + norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs + ) + + +# =========================================================================== +# Module-level behavior +# =========================================================================== +@requires_vllm_config +class TestFusedMMInputNormModule: + def test_matches_reference_above_cudnn_grid_limit(self): + """num_patches above the old cuDNN batch-norm grid limit (~65535) + used to raise CUDNN_STATUS_INTERNAL_ERROR.""" + set_random_seed(0) + pixel_values = torch.randint( + 0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE + ) + _check_against_reference(pixel_values, torch.float32) + + def test_identity_passthrough(self): + norm = IdentityInputNorm() + + pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE) + out = norm(pixel_values, visual_dtype=torch.bfloat16) + torch.testing.assert_close(out, pixel_values.to(torch.bfloat16)) + + +# =========================================================================== +# dtype coverage +# =========================================================================== +@requires_vllm_config +class TestFusedMMInputNormDtypes: + @pytest.mark.parametrize( + "in_dtype,out_dtype", + [ + (torch.float32, torch.float32), + (torch.float32, torch.bfloat16), + (torch.float32, torch.float16), + (torch.bfloat16, torch.bfloat16), + (torch.bfloat16, torch.float32), + (torch.float16, torch.float16), + (torch.uint8, torch.bfloat16), + (torch.uint8, torch.float32), + ], + ) + def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype): + set_random_seed(0) + if in_dtype == torch.uint8: + pixel_values = torch.randint( + 0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE + ) + else: + pixel_values = torch.rand( + (32, 3 * 16), dtype=torch.float32, device=_DEVICE + ).to(in_dtype) + + _check_against_reference( + pixel_values, + out_dtype, + image_mean=[0.5, 0.5, 0.5], + image_std=[0.5, 0.5, 0.5], + atol=1e-2, + rtol=1e-2, + ) + + +# =========================================================================== +# Shape / channel coverage +# =========================================================================== +@requires_vllm_config +class TestFusedMMInputNormShapes: + @pytest.mark.parametrize("channel", [1, 3, 4]) + def test_channel_variants(self, channel: int): + set_random_seed(0) + pixel_values = torch.randint( + 0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE + ) + _check_against_reference( + pixel_values, + torch.float32, + channel=channel, + image_mean=[0.5] * channel, + image_std=[0.25] * channel, + ) + + @pytest.mark.parametrize("patch_size", [1, 2048]) + def test_block_boundaries(self, patch_size: int): + """Masked-tail (12 elements) and exact-multiple (24 blocks) paths.""" + set_random_seed(0) + pixel_values = torch.randint( + 0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE + ) + _check_against_reference(pixel_values, torch.float32) + + +# =========================================================================== +# Input handling: non-contiguous inputs +# =========================================================================== +@requires_vllm_config +class TestFusedMMInputNormInputHandling: + def test_non_contiguous_input_matches_reference(self): + set_random_seed(0) + base = torch.randint( + 0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE + ) + non_contig = base[..., 0] + assert not non_contig.is_contiguous() + + _check_against_reference(non_contig, torch.float32) + + +# =========================================================================== +# Raw kernel entry point +# =========================================================================== +@requires_accelerator +@requires_triton +class TestFusedMMInputNormKernel: + @pytest.mark.parametrize( + "N, C, L", + [ + (1, 1, 1), + (1, 3, 1), + (3, 3, 7), # tile straddles channel boundaries + (2, 8, 63), + (4, 3, 1000), + ], + ) + def test_channel_boundary_crossing(self, N: int, C: int, L: int): + """The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile + can straddle a channel boundary and gather weight/bias per lane. + These shapes force that path by keeping ``L`` well below the block.""" + set_random_seed(0) + x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE) + w = torch.randn(C, dtype=torch.float32, device=_DEVICE) + b = torch.randn(C, dtype=torch.float32, device=_DEVICE) + + out = torch.empty_like(x) + fused_mm_input_norm_triton(x, out, w, b) + + expected = x * w.view(1, C, 1) + b.view(1, C, 1) + torch.testing.assert_close(out, expected) + + def test_non_contiguous_inputs_materialized(self): + """Read-only tensors are made contiguous internally.""" + N, C, L = 3, 3, 100 + set_random_seed(0) + base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE) + x = base[..., 0] + assert not x.is_contiguous() + w = torch.randn(C, dtype=torch.float32, device=_DEVICE) + b = torch.randn(C, dtype=torch.float32, device=_DEVICE) + + out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE) + fused_mm_input_norm_triton(x, out, w, b) + + expected = x * w.view(1, C, 1) + b.view(1, C, 1) + torch.testing.assert_close(out, expected) + + def test_rejects_mismatched_output_shape(self): + """``outputs`` must be shaped exactly like ``inputs``.""" + N, C, L = 3, 3, 100 + x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE) + w = torch.randn(C, dtype=torch.float32, device=_DEVICE) + b = torch.randn(C, dtype=torch.float32, device=_DEVICE) + + for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)): + with pytest.raises(AssertionError): + fused_mm_input_norm_triton( + x, + torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE), + w, + b, + ) + + +# =========================================================================== +# Construction / configuration +# =========================================================================== +@requires_vllm_config +class TestFusedMMInputNormConstruction: + """Weight/bias buffer semantics at init time.""" + + def test_identity_config_buffers(self): + """Numerically identity parameters still build plain weight/bias + buffers; the identity shortcut lives in ``from_model_config``.""" + norm = FusedMMInputNorm( + image_mean=[0.0, 0.0, 0.0], + image_std=[1.0, 1.0, 1.0], + rescale_factor=1.0, + ) + torch.testing.assert_close(norm.weight, torch.ones(3)) + torch.testing.assert_close(norm.bias, torch.zeros(3)) + + def test_non_identity_buffers(self): + channel = 3 + norm = FusedMMInputNorm( + image_mean=_RGB_MEAN, + image_std=_RGB_STD, + rescale_factor=_RGB_RESCALE, + channel=channel, + ) + assert norm.weight.shape == (channel,) + assert norm.bias.shape == (channel,) + + mean = torch.tensor(_RGB_MEAN, dtype=torch.float32) + std = torch.tensor(_RGB_STD, dtype=torch.float32) + torch.testing.assert_close(norm.weight, _RGB_RESCALE / std) + torch.testing.assert_close(norm.bias, -mean / std) + + def test_is_identity_params(self): + """The numeric-identity check mirrors the old allclose detection.""" + assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity + assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity + + @pytest.mark.parametrize( + ("image_mean", "image_std", "rescale_factor"), + [ + ([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0), + ([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255), + ], + ) + def test_buffers_initialized_on_default_device( + self, + image_mean: list[float], + image_std: list[float], + rescale_factor: float, + ): + """Buffers are built on CPU first, then moved to the caller's + default device, so init never touches the accelerator.""" + # The meta device gives the CPU-only test shard the same non-CPU + # default-device semantics without requiring a CUDA build. + default_device = "cuda" if torch.cuda.is_available() else "meta" + with torch.device(default_device): + norm = FusedMMInputNorm(image_mean, image_std, rescale_factor) + + assert norm.weight.device.type == default_device + assert norm.bias.device.type == default_device diff --git a/tests/model_executor/layers/test_pooler_methods.py b/tests/model_executor/layers/test_pooler_methods.py index 4196b99820..2abf3d7280 100644 --- a/tests/model_executor/layers/test_pooler_methods.py +++ b/tests/model_executor/layers/test_pooler_methods.py @@ -9,7 +9,7 @@ from unittest.mock import patch import numpy as np import pytest import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import PoolerConfig, set_current_vllm_config from vllm.model_executor.layers.pooler import PoolingParamsUpdate @@ -315,7 +315,7 @@ def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type): model_config=SimpleNamespace( pooler_config=pooler_config, head_dtype=None, - hf_config=PretrainedConfig(num_labels=3), + hf_config=PreTrainedConfig(num_labels=3), ), ) classifier = torch.nn.Linear(4, 3) diff --git a/tests/model_executor/layers/test_rocm_unquantized_gemm.py b/tests/model_executor/layers/test_rocm_unquantized_gemm.py index 66b4af7727..491cfa5435 100644 --- a/tests/model_executor/layers/test_rocm_unquantized_gemm.py +++ b/tests/model_executor/layers/test_rocm_unquantized_gemm.py @@ -17,9 +17,10 @@ if current_platform.is_cuda(): from vllm.model_executor.layers import utils -def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch): +@pytest.mark.parametrize("m", [1, 128]) +def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m): x = torch.randn(1, 64, dtype=torch.float16) - weight = torch.randn(128, 64, dtype=torch.float16) + weight = torch.randn(m, 64, dtype=torch.float16) monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False) monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True) @@ -42,6 +43,20 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch): assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("n", range(1, 6)) +def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n): + torch.manual_seed(0) + x = torch.randn(n, 2048, device="cuda", dtype=dtype) + weight = torch.randn(1, 2048, device="cuda", dtype=dtype) + + out = utils.rocm_unquantized_gemm_impl(x, weight, None) + ref = torch.nn.functional.linear(x, weight, None) + + torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2) + + def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch): x = torch.randn(64, 4, dtype=torch.float16).t() weight = torch.randn(128, 64, dtype=torch.float16) @@ -149,11 +164,12 @@ def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2) -def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch): +@pytest.mark.parametrize("m", [1, 128]) +def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m): # wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must # fall back to torch.nn.functional.linear. x = torch.randn(6, 64, dtype=torch.float16) - weight = torch.randn(128, 64, dtype=torch.float16) + weight = torch.randn(m, 64, dtype=torch.float16) monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False) monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True) diff --git a/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py b/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py new file mode 100644 index 0000000000..210cd28d6f --- /dev/null +++ b/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for --mm-encoder-only safetensors shard filtering.""" + +import json +import os +import tempfile + +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + +from vllm.model_executor.model_loader.weight_utils import ( + filter_mm_encoder_only_safetensors_files, + resolve_mm_encoder_only_lm_prefixes, +) +from vllm.model_executor.models.utils import WeightsMapper + + +def _write_index(folder: str, weight_map: dict[str, str]) -> None: + with open(os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME), "w") as f: + json.dump({"weight_map": weight_map}, f) + + +def _qwen25_mapper() -> WeightsMapper: + """Qwen2.5-VL dual layout mapper (nested + flat catch-all).""" + return WeightsMapper( + orig_to_new_prefix={ + "model.language_model.": "language_model.model.", + "model.visual.": "visual.", + "lm_head.": "language_model.lm_head.", + "model.": "language_model.model.", + } + ) + + +def _qwen3_mapper() -> WeightsMapper: + return WeightsMapper( + orig_to_new_prefix={ + "model.visual.": "visual.", + "lm_head.": "language_model.lm_head.", + "model.language_model.": "language_model.model.", + } + ) + + +def _molmo_mapper() -> WeightsMapper: + return WeightsMapper( + orig_to_new_prefix={ + "model.vision_backbone.": "vision_backbone.", + "model.transformer.blocks.": "model.layers.", + "model.transformer.ln_f.": "model.norm.", + "model.transformer.mlp.down_proj.": "lm_head.", + } + ) + + +def test_keeps_vision_shards_drops_language_only(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000003.safetensors"), + os.path.join(folder, "model-00002-of-000003.safetensors"), + os.path.join(folder, "model-00003-of-000003.safetensors"), + ] + for path in files: + open(path, "wb").close() + + _write_index( + folder, + { + "language_model.layers.0.weight": "model-00001-of-000003.safetensors", + "language_model.layers.1.weight": "model-00002-of-000003.safetensors", + "vision_tower.blocks.0.weight": "model-00003-of-000003.safetensors", + "mm_projector.weight": "model-00003-of-000003.safetensors", + }, + ) + + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == [files[2]] + + +def test_keeps_mixed_shard_with_vision_and_lm(): + with tempfile.TemporaryDirectory() as folder: + mixed = os.path.join(folder, "model-00001-of-000001.safetensors") + open(mixed, "wb").close() + _write_index( + folder, + { + "language_model.embed.weight": "model-00001-of-000001.safetensors", + "vision_tower.patch.weight": "model-00001-of-000001.safetensors", + }, + ) + + kept = filter_mm_encoder_only_safetensors_files( + [mixed], + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == [mixed] + + +def test_no_index_returns_unchanged(): + with tempfile.TemporaryDirectory() as folder: + files = [os.path.join(folder, "model.safetensors")] + open(files[0], "wb").close() + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == files + + +def test_resolve_returns_module_prefixes_without_hf_hardcode(): + """No global Qwen HF nest list — just vLLM module prefixes.""" + prefixes = resolve_mm_encoder_only_lm_prefixes(["language_model"]) + assert prefixes == ("language_model.",) + # With Qwen mapper, still module prefixes (keys classified via mapper). + assert resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen25_mapper() + ) == ("language_model.",) + + +def test_resolve_empty_language_model_names_fail_closed(): + """Missing/empty ``_language_model_names`` → None (no language_model. default).""" + assert resolve_mm_encoder_only_lm_prefixes(None) is None + assert resolve_mm_encoder_only_lm_prefixes([]) is None + assert resolve_mm_encoder_only_lm_prefixes(()) is None + + +def test_resolve_skips_filter_for_shared_hf_root(): + """Shared HF root (Molmo-shaped mapper) → fail-closed, not magic name list.""" + assert ( + resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper()) + is None + ) + # Without a mapper, identity layout: module prefix is returned as-is. + assert resolve_mm_encoder_only_lm_prefixes(["model"]) == ("model.",) + assert resolve_mm_encoder_only_lm_prefixes(["model."]) == ("model.",) + + +def test_resolve_keeps_llm_prefix_without_model_broadening(): + prefixes = resolve_mm_encoder_only_lm_prefixes(["llm"]) + assert prefixes == ("llm.",) + + +def test_qwen3_nested_index_skips_lm_keeps_visual(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000002.safetensors"), + os.path.join(folder, "model-00002-of-000002.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.language_model.layers.0.weight": ( + "model-00001-of-000002.safetensors" + ), + "lm_head.weight": "model-00001-of-000002.safetensors", + "model.visual.blocks.0.weight": "model-00002-of-000002.safetensors", + }, + ) + prefixes = resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen3_mapper() + ) + assert prefixes is not None + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + prefixes, + weights_mapper=_qwen3_mapper(), + ) + assert kept == [files[1]] + + +def test_qwen25_flat_index_skips_lm_keeps_visual(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000002.safetensors"), + os.path.join(folder, "model-00002-of-000002.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.layers.0.self_attn.q_proj.weight": ( + "model-00001-of-000002.safetensors" + ), + "model.embed_tokens.weight": "model-00001-of-000002.safetensors", + "model.norm.weight": "model-00001-of-000002.safetensors", + "lm_head.weight": "model-00001-of-000002.safetensors", + "visual.blocks.0.weight": "model-00002-of-000002.safetensors", + }, + ) + prefixes = resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen25_mapper() + ) + assert prefixes is not None + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + prefixes, + weights_mapper=_qwen25_mapper(), + ) + assert kept == [files[1]] + + +def test_molmo_shaped_index_not_filtered_when_deny_disabled(): + """Shared-root LM attr → resolve None; callers leave the file list alone.""" + assert ( + resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper()) + is None + ) + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "vision.safetensors"), + os.path.join(folder, "lm.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.vision_backbone.patch.weight": "vision.safetensors", + "model.transformer.blocks.0.weight": "lm.safetensors", + }, + ) + # Document why raw HF startswith("model.") is unsafe without fail-closed: + # whole-shard deny would also drop vision. + if_applied_naively = filter_mm_encoder_only_safetensors_files( + files, folder, SAFE_WEIGHTS_INDEX_NAME, ("model.",) + ) + assert if_applied_naively == [] diff --git a/tests/model_executor/model_loader/test_registry.py b/tests/model_executor/model_loader/test_registry.py index 95b797bb51..a795497a11 100644 --- a/tests/model_executor/model_loader/test_registry.py +++ b/tests/model_executor/model_loader/test_registry.py @@ -79,7 +79,7 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path): # Control: load_format="hf" still picks up .pt weights via fallback. (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00") loader = DefaultModelLoader(LoadConfig(load_format="hf")) - _, files, use_safetensors = loader._prepare_weights( + _, files, use_safetensors, _ = loader._prepare_weights( str(tmp_path), None, None, diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index 155957160f..85ddea3946 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner): "Qwen/Qwen3-0.6B", "inference-optimization/Qwen3-0.6B-debug-multiply", "inference-optimization/Qwen3-0.6B-debug-add", - "fp8", + "fp8_per_tensor", ), pytest.param( "inference-optimization/DeepSeek-V3-debug-empty", "inference-optimization/DeepSeek-V3-debug-multiply", "inference-optimization/DeepSeek-V3-debug-add", - "fp8", + "fp8_per_tensor", marks=[pytest.mark.slow_test], ), pytest.param( @@ -1192,7 +1192,7 @@ def test_online_quantize_reload( if current_platform.device_count() < tp_size: pytest.skip(reason="Not enough GPU devices") - if quantization == "fp8" and _fp8_reload_unsupported(): + if quantization == "fp8_per_tensor" and _fp8_reload_unsupported(): pytest.skip(reason="Requires FP8 support") with vllm_runner( diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index eb672bc35b..2280fff8b7 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -7,14 +7,11 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve identical outputs. """ -import glob -import os import shutil import subprocess import sys import tempfile import threading -import time from dataclasses import dataclass, field from typing import Any @@ -24,8 +21,6 @@ from vllm import SamplingParams from vllm.assets.image import ImageAsset from vllm.platforms import current_platform -DAEMON_TIMEOUT_S = 600 - class WeightCacheDaemon: """Context manager running the real weight cache daemon as a subprocess.""" @@ -35,18 +30,14 @@ class WeightCacheDaemon: model: str, tp_size: int, extra_args: list[str] | None = None, - num_groups: int = 1, ): # Short base path: Unix socket paths are limited to ~107 characters. self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_") - self.tp_size = tp_size - # Each daemon group (target plus cached drafts) binds one socket per - # local rank. - self.num_sockets = tp_size * num_groups self._cmd = [ sys.executable, "-m", - "vllm.model_executor.model_loader.weight_cache.daemon", + "vllm.entrypoints.cli.main", + "preload", "--model", model, "--tensor-parallel-size", @@ -57,18 +48,12 @@ class WeightCacheDaemon: *(extra_args or []), ] self._proc: subprocess.Popen | None = None - self._lines: list[str] = [] def __enter__(self) -> "WeightCacheDaemon": self._proc = subprocess.Popen( self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True ) threading.Thread(target=self._drain_stderr, daemon=True).start() - try: - self._wait_ready(DAEMON_TIMEOUT_S) - except Exception: - self._stop() - raise return self def __exit__(self, *exc_info) -> None: @@ -76,32 +61,7 @@ class WeightCacheDaemon: def _drain_stderr(self) -> None: assert self._proc is not None and self._proc.stderr is not None - for line in self._proc.stderr: - self._lines.append(line) - - def _logs(self) -> str: - return "".join(self._lines) - - def _wait_ready(self, timeout_s: float) -> None: - assert self._proc is not None - # Each rank binds its socket only once the model is fully cached. - # Poll for the socket files rather than a log line: model loading can - # pull in JIT compilers that swap the process's stderr and swallow - # everything logged afterwards, making log-based readiness flaky. - pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock") - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - if self._proc.poll() is not None: - raise RuntimeError( - f"Weight cache daemon exited with {self._proc.returncode}:\n" - f"{self._logs()}" - ) - if len(glob.glob(pattern)) >= self.num_sockets: - return - time.sleep(1.0) - raise TimeoutError( - f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}" - ) + self._proc.stderr.read() def _stop(self) -> None: assert self._proc is not None @@ -121,9 +81,6 @@ class ModelCase: images: list | None = None llm_kwargs: dict[str, Any] = field(default_factory=dict) daemon_args: list[str] = field(default_factory=list) - # Daemon groups the launcher starts; 2 when a cached draft group joins the - # target group (MTP/EAGLE speculative decoding). - daemon_groups: int = 1 def generate( @@ -203,7 +160,6 @@ QWEN_MTP_CASE = ModelCase( "--speculative-config", '{"method": "mtp", "num_speculative_tokens": 1}', ], - daemon_groups=2, ) @@ -242,7 +198,6 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase): case.model, tp_size=1, extra_args=case.daemon_args, - num_groups=case.daemon_groups, ) as d: warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False) # Warm restart: a second engine lifetime against the same daemon. @@ -251,3 +206,116 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase): assert cold_outputs == baseline_outputs assert warm_outputs == baseline_outputs assert restart_outputs == baseline_outputs + + +def _parallel(**kw): + from types import SimpleNamespace + + base = dict( + tensor_parallel_size=1, + pipeline_parallel_size=1, + data_parallel_size=1, + data_parallel_size_local=1, + data_parallel_rank=0, + nnodes=1, + node_rank=0, + ) + base.update(kw) + return SimpleNamespace(**base) + + +def test_daemon_places_tp_and_dp_ranks_on_local_gpus(): + """Each node serves a contiguous global-rank block (DP-major), so without + DP local GPU i is TP rank r*local+i, and single-node DP launchers offset + by --data-parallel-start-rank.""" + from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks + + tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1) + assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)] + + dp_third_node = _parallel( + data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8 + ) + assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)] + + dp_tp = _parallel( + tensor_parallel_size=2, + data_parallel_size=4, + data_parallel_size_local=2, + data_parallel_rank=2, + ) + assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)] + + # DP + nnodes: node-local TP replicas + dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2) + assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)] + dp_tp_node1 = _parallel( + tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1 + ) + assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)] + + # DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes) + tp_span_node1 = _parallel( + tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1 + ) + assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)] + tp_span_node2 = _parallel( + tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2 + ) + assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)] + + # DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes) + dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2) + assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)] + + +def test_daemon_rejects_unmappable_parallelism(): + from vllm.model_executor.model_loader.weight_cache.daemon import ( + _reject_unsupported_parallelism, + ) + + _reject_unsupported_parallelism( + _parallel( + data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12 + ) + ) + # DP combines with --nnodes when the world size divides evenly. + _reject_unsupported_parallelism( + _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2) + ) + with pytest.raises(ValueError, match="pipeline"): + _reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2)) + with pytest.raises(ValueError, match="evenly divide"): + _reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2)) + with pytest.raises(ValueError, match="evenly divide"): + _reject_unsupported_parallelism( + _parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4) + ) + with pytest.raises(ValueError, match="exceeds"): + _reject_unsupported_parallelism( + _parallel( + data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13 + ) + ) + + +def test_weight_cache_key_distinguishes_dp_ranks(): + from dataclasses import replace + + from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey + + key = WeightCacheKey( + checkpoint="ckpt", + model_arch="Arch", + tp_size=1, + tp_rank=0, + dtype="bf16", + quantization=None, + quant_config_hash="h", + revision=None, + vllm_version="v", + dp_size=16, + dp_rank=3, + ) + assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"] + assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"] diff --git a/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py b/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py index f18780cf6b..014a9ca25d 100644 --- a/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py +++ b/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py @@ -7,6 +7,7 @@ import torch from vllm.model_executor.layers import utils from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_default_torch_dtype @pytest.fixture(scope="module") @@ -89,3 +90,25 @@ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch): expected_prepacked, ) ] + + +@pytest.mark.usefixtures("_mock_zentorch_linear_unary") +@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32]) +def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype( + monkeypatch, weight_dtype +): + monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True) + + layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype) + loading_dtype = ( + torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16 + ) + with set_default_torch_dtype(loading_dtype): + utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True) + + assert layer.weight.numel() == 0 + assert layer.weight.dtype is weight_dtype + + x = torch.randn(4, 16, dtype=weight_dtype) + output = layer.cpu_linear(x, layer.weight, None) + assert not output.isnan().any() diff --git a/tests/model_executor/test_jit_warmup_triton_launcher.py b/tests/model_executor/test_jit_warmup_triton_launcher.py index 725a383833..a92303dcf2 100644 --- a/tests/model_executor/test_jit_warmup_triton_launcher.py +++ b/tests/model_executor/test_jit_warmup_triton_launcher.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import threading from dataclasses import dataclass from typing import Any import pytest import torch +from vllm import envs from vllm.model_executor.warmup import jit_warmup_triton_helper from vllm.model_executor.warmup.jit_warmup import ( + JitWarmupRegistry, WarmupChoices, WarmupIntRange, kernel_launcher, @@ -116,6 +119,105 @@ def test_triton_launcher_supports_compile_and_runtime_adapters() -> None: assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})] +@pytest.mark.skipif( + not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"), + reason="Requires CUDA Triton async compilation", +) +def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None: + from triton.runtime._async_compile import active_mode + + monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2) + owner = _TestTritonKernel() + owner.kernel = _FakeTritonKernel() + + def fail(): + raise RuntimeError("warmup failed") + + def warmup(**kwargs): + active_mode.get().submit(kwargs["second"], fail, lambda result: None) + + monkeypatch.setattr(owner.kernel, "warmup", warmup) + with pytest.raises(RuntimeError, match="warmup failed"): + owner.compile_many(owner.CompileKey(value=i) for i in range(2)) + + assert active_mode.get() is None + owner("runtime", 3, None) + assert len(owner.kernel.runtime_calls) == 1 + + +def test_autotuning_warmup_stays_serial(monkeypatch) -> None: + owner = _TestTritonKernel() + owner.kernel = _FakeTritonKernel() + owner._run_autotune = True + monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4) + + def unexpected_async(*args): + pytest.fail("Autotuning must not use asynchronous compilation") + + monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False) + owner.compile_many(owner.CompileKey(value=i) for i in range(2)) + assert len(owner.kernel.runtime_calls) == 2 + + +@triton.jit +def _warmup_store_kernel(out, VALUE: tl.constexpr): + tl.store(out, VALUE) + + +@pytest.mark.skipif( + not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"), + reason="Requires CUDA Triton async compilation", +) +def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial( + monkeypatch, +) -> None: + from triton.runtime._async_compile import active_mode + + monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2) + + def warmup_inputs(): + return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2)) + + @triton_kernel_dispatcher_with_warmup( + kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs + ) + def store(out, value): + return (1,), dict(VALUE=value) + + store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler. + caller = threading.get_ident() + barrier = threading.Barrier(2, timeout=10) + compiled_on = [] + compile_kernel = _warmup_store_kernel.compile + + def record_compile(*args, **kwargs): + if threading.get_ident() != caller: + barrier.wait() + result = compile_kernel(*args, **kwargs) + compiled_on.append(threading.get_ident()) + return result + + monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile) + registry = JitWarmupRegistry(None) + with registry.activate(): + store.register_warmup() + registry.warmup() + + assert len(compiled_on) == 2 + assert all(thread != caller for thread in compiled_on) + assert active_mode.get() is None + out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type) + for value in (1, 2): + store(out, value) + assert out.item() == value + assert len(compiled_on) == 2 + + # A specialization first seen during inference uses the normal blocking JIT. + store(out, 3) + assert out.item() == 3 + assert compiled_on[2:] == [caller] + + def test_triton_launcher_supports_cpu_function_wrappers() -> None: calls: list[tuple[Any, ...]] = [] diff --git a/tests/model_executor/test_qwen3_omni.py b/tests/model_executor/test_qwen3_omni.py index 71c74863e9..e96dae8186 100644 --- a/tests/model_executor/test_qwen3_omni.py +++ b/tests/model_executor/test_qwen3_omni.py @@ -8,7 +8,7 @@ from unittest.mock import Mock, patch import pytest import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import ParallelConfig from vllm.multimodal.processing import InputProcessingContext @@ -52,7 +52,7 @@ def print_input_ids(input_ids): @pytest.fixture def mock_qwen3_omni_config(): """Create a mock Qwen3OmniMoeThinker config.""" - config = Mock(spec=PretrainedConfig) + config = Mock(spec=PreTrainedConfig) # Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json config.audio_token_id = 151675 # <|audio_pad|> config.video_token_id = 151656 # <|video_pad|> @@ -144,6 +144,11 @@ def test_qwen3_omni_get_updates_use_audio_in_video( # Create a mock context mock_ctx = Mock(spec=InputProcessingContext) mock_ctx.tokenizer = mock_tokenizer + # `model_config` is an instance attribute, so it is not covered by the + # spec; the data parser reads it for `allow_missing_mm_embeddings`. + mock_ctx.model_config = SimpleNamespace( + multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False) + ) # Create processing info info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx) diff --git a/tests/model_executor/test_triton_dispatcher.py b/tests/model_executor/test_triton_dispatcher.py new file mode 100644 index 0000000000..909230c40f --- /dev/null +++ b/tests/model_executor/test_triton_dispatcher.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the apply-time Triton kernel override registry. + +The registry must let out-of-tree platforms replace kernels defined as +plain ``@triton.jit`` functions, patching every hold-site (defining +module, from-import copies, and JIT warmup owners) so that existing call +sites dispatch to the override without any core-side decoration. +""" + +import sys +import types +from typing import Any + +import pytest + +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + _DecoratedTritonJitKernel, +) +from vllm.triton_utils import tl, triton +from vllm.triton_utils.dispatcher import ( + KernelOverride, + register_kernels, +) + + +class _FakeTritonKernel: + """Stands in for a Triton JITFunction (works without Triton installed).""" + + arg_names = ("output_ptr", "input_ptr", "replace_from", "MAX_NUM_TOKENS") + + def __getitem__(self, grid: Any) -> Any: + def launch(*args: Any, **kwargs: Any) -> Any: + return ("default", grid, kwargs) + + return launch + + +@pytest.fixture +def _fake_module(monkeypatch: pytest.MonkeyPatch) -> Any: + """Expose a fake defining module holding the kernel and its warmup owner.""" + import types + + kernel = _FakeTritonKernel() + owner = _DecoratedTritonJitKernel( + kernel, + lambda: {}, + lambda output, input, replace_from: ( # noqa: ARG005 + (1,), + dict(MAX_NUM_TOKENS=8), + ), + ) + module = types.ModuleType("vllm.tests.fake_kernel_home") + module.expand_kernel = kernel # type: ignore[attr-defined] + module._expand = owner # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, module.__name__, module) + return module + + +def test_override_patches_defining_module_and_warmup_owner( + _fake_module: Any, +) -> None: + calls: list[tuple[Any, ...]] = [] + + def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None): + calls.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS)) + return "cpu" + + register_kernels( + { + "vllm.tests.fake_kernel_home.expand_kernel": cpu_impl, + } + ) + + overridden = _fake_module.expand_kernel + assert isinstance(overridden, KernelOverride) + # Launch with kernel-arg names like the warmup launch path does. + assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu" + assert calls == [(1, 2, 0, None)] + + # The warmup owner's captured kernel is patched too, and its cached + # argument-name introspection is invalidated. + assert _fake_module._expand.kernel is overridden + + +def test_override_forwards_positionally_for_mismatched_names( + _fake_module: Any, +) -> None: + calls: list[tuple[Any, ...]] = [] + + def cpu_impl(output, input_val, replace_from, MAX_NUM_TOKENS=None): + calls.append((output, input_val, replace_from, MAX_NUM_TOKENS)) + return "cpu" + + register_kernels( + { + "vllm.tests.fake_kernel_home.expand_kernel": cpu_impl, + } + ) + overridden = _fake_module.expand_kernel + assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu" + assert calls == [(1, 2, 0, None)] + + +def test_override_covers_from_import_copies(_fake_module: Any) -> None: + import types + + importer = types.ModuleType("vllm.tests.fake_kernel_importer") + importer.expand_kernel = _fake_module.expand_kernel # type: ignore[attr-defined] + sys.modules["vllm.tests.fake_kernel_importer"] = importer + + def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None): + return "cpu" + + register_kernels( + { + "vllm.tests.fake_kernel_home.expand_kernel": cpu_impl, + } + ) + assert isinstance(importer.expand_kernel, KernelOverride) + del sys.modules["vllm.tests.fake_kernel_importer"] + + +def test_override_owner_launch_goes_through_impl(_fake_module: Any) -> None: + launched: list[Any] = [] + + def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None): + launched.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS)) + return "cpu" + + register_kernels( + { + "vllm.tests.fake_kernel_home.expand_kernel": cpu_impl, + } + ) + owner = _fake_module._expand + # In-place kernels return None from launch; assert on the recorded call. + owner(1, 2, 0) + assert launched == [(1, 2, 0, 8)] + + +def test_register_kernel_auto_imports_defining_module( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Registering a kernel whose defining module is not imported yet must + import it on the spot and apply the override.""" + kernel_name = "vllm.v1.sample.rejection_sampler.expand_kernel" + module_name = kernel_name.rpartition(".")[0] + calls: list[tuple[Any, ...]] = [] + + def cpu_impl( + output_ptr, + input_ptr, + cu_num_tokens_ptr, + replace_from, + replace_to, + MAX_NUM_TOKENS=None, + ): + calls.append((output_ptr, replace_from, MAX_NUM_TOKENS)) + return "cpu" + + # Ensure the module is NOT imported before registration. Drop it from + # sys.modules; a fresh import creates new kernel/owner objects, which + # is what a cold-start platform process would see. + saved = sys.modules.pop(module_name, None) + # Re-run the registry patch against the original module afterwards so + # other tests that already imported it keep working. + try: + assert module_name not in sys.modules + register_kernels({kernel_name: cpu_impl}) + assert module_name in sys.modules + finally: + if saved is None: + # First import in the whole session: keep the fresh module. + saved = sys.modules[module_name] + else: + # Restore the original module object and re-apply the override + # to its (pre-existing) kernel so both generations agree. + sys.modules[module_name] = saved + register_kernels({kernel_name: cpu_impl}) + + overridden = saved.expand_kernel + assert isinstance(overridden, KernelOverride) + overridden[(2,)]( + output_ptr=1, + input_ptr=2, + cu_num_tokens_ptr=3, + replace_from=0, + replace_to=0, + MAX_NUM_TOKENS=8, + ) + assert calls == [(1, 0, 8)] + # The warmup owner captured at the fresh import must be patched too + # (when the module existed before, there are two module generations). + fresh = sys.modules[module_name] + if fresh is not saved: + assert fresh._expand.kernel is overridden + + +def test_real_triton_kernel_override_end_to_end() -> None: + """Override a real @triton.jit kernel through the registry.""" + if not triton.__class__.__module__.startswith("triton"): + pytest.skip("Triton is not installed") + + @triton.jit + def _add_one_kernel(x_ptr, out_ptr, BLOCK: tl.constexpr): # pragma: no cover + offsets = tl.arange(0, BLOCK) + tl.store(out_ptr + offsets, tl.load(x_ptr + offsets) + 1) + + module = types.ModuleType("vllm.tests.real_jit_kernel_home") + module._add_one_kernel = _add_one_kernel # type: ignore[attr-defined] + sys.modules[module.__name__] = module + + def cpu_impl(x_ptr, out_ptr, BLOCK=None): # noqa: ARG001 + return "cpu" + + register_kernels({"vllm.tests.real_jit_kernel_home._add_one_kernel": cpu_impl}) + assert module._add_one_kernel.arg_names == ( # type: ignore[attr-defined] + "x_ptr", + "out_ptr", + "BLOCK", + ) diff --git a/tests/model_executor/test_weight_utils.py b/tests/model_executor/test_weight_utils.py index 3dea587075..512f518a05 100644 --- a/tests/model_executor/test_weight_utils.py +++ b/tests/model_executor/test_weight_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import tempfile +from unittest.mock import Mock import huggingface_hub.constants import pytest @@ -9,6 +10,7 @@ from huggingface_hub.utils import LocalEntryNotFoundError from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf, + drop_checkpoint_cache, maybe_remap_kv_scale_name, ) @@ -292,5 +294,18 @@ class TestKvCacheScaleMapper: ) +@pytest.mark.parametrize("fails", [False, True]) +def test_drop_checkpoint_cache_releases_local_weight_files( + tmp_path, monkeypatch, fails +): + """Evict only weights; cache eviction errors must not prevent loading.""" + for name in ("weights.safetensors", "config.json"): + (tmp_path / name).touch() + fadvise = Mock(side_effect=OSError("cache eviction failed") if fails else None) + monkeypatch.setattr("os.posix_fadvise", fadvise) + drop_checkpoint_cache(str(tmp_path)) + fadvise.assert_called_once() + + if __name__ == "__main__": test_download_weights_from_hf() diff --git a/tests/models/kimi_k3/test_attn_res.py b/tests/models/kimi_k3/test_attn_res.py index 574f7a5925..366abe0221 100644 --- a/tests/models/kimi_k3/test_attn_res.py +++ b/tests/models/kimi_k3/test_attn_res.py @@ -17,6 +17,26 @@ EPS = 1e-5 attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res") +def _on_rocm_below_10() -> bool: + if not current_platform.is_rocm(): + return False + from vllm.platforms.rocm import get_rocm_version + + return (get_rocm_version() or (0,)) < (10,) + + +# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD +# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only +# the num_blocks > 0 loop contains it. ROCm serves AttnRes from +# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost. +_OLD_ROCM = _on_rocm_below_10() + + +def _skip_on_old_rocm(num_blocks: int) -> None: + if _OLD_ROCM and num_blocks > 0: + pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select") + + def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor: storage = torch.randn( *shape[:-1], @@ -79,6 +99,7 @@ def test_attn_res( ): if backend == "nvidia" and not current_platform.is_device_capability_family(100): pytest.skip("NVIDIA AttnRes requires the SM100 family") + _skip_on_old_rocm(num_blocks) prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding) delta = ( @@ -133,6 +154,7 @@ def test_attn_res( @pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1)) def test_attn_res_block_counts(num_blocks: int): + _skip_on_old_rocm(num_blocks) prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) blocks = torch.randn( 1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 @@ -169,6 +191,7 @@ def test_attn_res_block_counts(num_blocks: int): def test_attn_res_without_output_norm(): + _skip_on_old_rocm(MAX_BLOCKS) prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) delta = torch.randn_like(prefix) blocks = torch.randn( diff --git a/tests/models/kimi_k3/test_kda.py b/tests/models/kimi_k3/test_kda.py index f03d2cdcae..eaf4b69b40 100644 --- a/tests/models/kimi_k3/test_kda.py +++ b/tests/models/kimi_k3/test_kda.py @@ -22,6 +22,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_upd from vllm.model_executor.layers.mamba.ops.gather_initial_states import ( gather_initial_states, ) +from vllm.models.kimi_k3.amd.ops.third_party.kda import ( + fused_recurrent_kda as fused_recurrent_kda_amd, +) from vllm.models.kimi_k3.amd.ops.third_party.kda import ( fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd, ) @@ -44,10 +47,12 @@ from vllm.models.kimi_k3.nvidia.ops.third_party.kda import ( chunk_kda, chunk_kda_with_fused_gate, fused_kda_gate, - fused_recurrent_kda, fused_recurrent_kda_fwd, fused_recurrent_kda_packed_decode, ) +from vllm.models.kimi_k3.nvidia.ops.third_party.kda import ( + fused_recurrent_kda as fused_recurrent_kda_nvidia, +) from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd from vllm.utils.flashinfer import flashinfer_fused_kda_decode @@ -66,6 +71,10 @@ PACKED_DECODE_IMPLS = { "nvidia": fused_recurrent_kda_packed_decode, "amd": fused_recurrent_kda_packed_decode_amd, } +SPEC_DECODE_IMPLS = { + "nvidia": fused_recurrent_kda_nvidia, + "amd": fused_recurrent_kda_amd, +} def test_kda_warmup_skips_missing_metadata(monkeypatch): @@ -468,17 +477,51 @@ def test_packed_kda_decode_correctness( @pytest.mark.parametrize( - ("H", "fuse_gate"), - [(12, True), (12, False), (12, None), (96, None)], + ("impl", "H", "fuse_gate", "num_seqs", "query_len"), + [ + *[ + pytest.param( + impl, + H, + fuse_gate, + 3, + 3, + id=f"{impl}-H{H}-fuse-{fuse_gate}", + ) + for impl in SPEC_DECODE_IMPLS + for H, fuse_gate in [ + (12, True), + (12, False), + (12, None), + (96, None), + ] + ], + pytest.param("amd", 12, True, 1, 1, id="amd-single-token"), + pytest.param("amd", 12, True, 1, 8, id="amd-single-sequence-short"), + pytest.param( + "amd", + 96, + None, + 1, + 8, + id="amd-single-sequence-many-heads", + ), + pytest.param("amd", 12, True, 2, 4, id="amd-uniform-two-sequences"), + pytest.param("amd", 12, True, 4, 4, id="amd-uniform-four-sequences"), + pytest.param("amd", 12, True, 8, 7, id="amd-uniform-eight-sequences"), + ], ) @pytest.mark.parametrize("lower_bound", [-5.0, None]) @torch.inference_mode() def test_kda_spec_decode_correctness( H: int, fuse_gate: bool | None, + num_seqs: int, + query_len: int, lower_bound: float | None, + impl: str, ): - num_seqs, query_len, D = 3, 3, 128 + D = 128 T = num_seqs * query_len torch.manual_seed(1234) @@ -522,10 +565,8 @@ def test_kda_spec_decode_correctness( dtype=torch.int32, device=DEVICE, ).view(num_seqs, query_len) - num_accepted_tokens = torch.tensor( - [1, 2, 3], - dtype=torch.int32, - device=DEVICE, + num_accepted_tokens = ( + torch.arange(num_seqs, dtype=torch.int32, device=DEVICE) % query_len + 1 ) state_storage = 0.01 * torch.randn( T + 1, @@ -578,7 +619,12 @@ def test_kda_spec_decode_correctness( expected = torch.cat(expected_outputs, dim=1) actual_state = state.clone() - actual, _ = fused_recurrent_kda( + extra_args = ( + {"uniform_sequence_length": query_len} + if impl == "amd" and num_seqs in (2, 4, 8) + else {} + ) + actual, _ = SPEC_DECODE_IMPLS[impl]( q=q, k=k, v=v, @@ -593,6 +639,7 @@ def test_kda_spec_decode_correctness( num_accepted_tokens=num_accepted_tokens, out=output, fuse_gate=fuse_gate, + **extra_args, ) assert actual.data_ptr() == output.data_ptr() diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index fa810217b1..6268cb9bcc 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = { "prefill_state_indices", "prefill_has_initial_state", "spec_sequence_masks", + "uniform_spec_sequence_length", "flashinfer_prefill_query_start_loc", "flashinfer_prefill_seq_order", } @@ -409,6 +410,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): "num_speculative_tokens", "full_cuda_graph", "is_prefilling", + "expected_uniform_spec_sequence_length", ), [ pytest.param( @@ -417,6 +419,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [False, False], + 3, id="pure-spec-decode", ), pytest.param( @@ -425,6 +428,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [True, False, False], + 3, id="mixed-prefill-and-spec-decode", ), pytest.param( @@ -433,6 +437,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 0, False, [False, False], + None, id="regular-decode", ), pytest.param( @@ -441,8 +446,18 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [False, False], + None, id="no-scheduled-draft-tokens", ), + pytest.param( + BatchSpec(seq_lens=[50, 30], query_lens=[3, 2]), + [2, 1], + 2, + False, + [False, False], + None, + id="ragged-spec-decode", + ), ], ) def test_kimi_k3_kda_metadata_matches_shared_gdn( @@ -451,6 +466,7 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn( num_speculative_tokens: int, full_cuda_graph: bool, is_prefilling: list[bool], + expected_uniform_spec_sequence_length: int | None, ): kwargs: dict[str, torch.Tensor] = {} if num_decode_draft_tokens is not None: @@ -483,6 +499,9 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn( assert isinstance(actual, KimiK3KDAMetadata) _assert_matches_shared_gdn(reference, actual) + assert ( + reference.uniform_spec_sequence_length == expected_uniform_spec_sequence_length + ) def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata(): diff --git a/tests/models/kimi_k3/test_latent_moe_tail.py b/tests/models/kimi_k3/test_latent_moe_tail.py index 5fde9ff86d..96876e736c 100644 --- a/tests/models/kimi_k3/test_latent_moe_tail.py +++ b/tests/models/kimi_k3/test_latent_moe_tail.py @@ -15,6 +15,7 @@ from tests.utils import ( multi_process_parallel, ) from vllm.distributed import get_tp_group +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import ( TrtLlmMxfp4ExpertsMonolithic, ) @@ -43,12 +44,11 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup( hidden_dim = LATENT_SIZE hidden_dim_unpadded = LATENT_SIZE experts_per_token = 16 - defer_moe_finalize = False + _defer_moe_finalize = False defer_moe_finalize_max_num_tokens = -1 - - @property - def use_deferred_moe_finalize(self) -> bool: - return self.defer_moe_finalize + defer_moe_finalize = FusedMoEConfig.defer_moe_finalize + limit_deferred_moe_finalize = FusedMoEConfig.limit_deferred_moe_finalize + use_deferred_moe_finalize = FusedMoEConfig.use_deferred_moe_finalize moe_config = FakeMoEConfig() quant_method = SimpleNamespace( @@ -99,7 +99,7 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup( latent_moe_runner.LatentMoERunner() - assert moe_config.defer_moe_finalize + assert moe_config.use_deferred_moe_finalize assert moe_config.defer_moe_finalize_max_num_tokens == 128 assert initialized_with["experts_per_token"] == 16 diff --git a/tests/models/language/generation/test_gemma.py b/tests/models/language/generation/test_gemma.py index 5cffe6fb19..a8fe4ab951 100644 --- a/tests/models/language/generation/test_gemma.py +++ b/tests/models/language/generation/test_gemma.py @@ -7,7 +7,8 @@ import numpy as np import pytest import torch -from vllm.config import VllmConfig +from vllm.config import CompilationConfig, VllmConfig +from vllm.config.compilation import CompilationMode from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.models import gemma from vllm.model_executor.models.gemma3n import ( @@ -18,10 +19,72 @@ from vllm.model_executor.models.gemma4 import ( Gemma4ForCausalLM, _gemma4_layer_weights_mapper, ) +from vllm.model_executor.models.gemma4_dspark import ( + Gemma4DSparkForCausalLM, + Gemma4DSparkModel, +) MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"] +@pytest.mark.usefixtures("dist_init") +@pytest.mark.parametrize( + "enabled,has_weights,with_markov", + [ + pytest.param(True, True, False, id="hidden-only"), + pytest.param(True, True, True, id="with-markov"), + pytest.param(True, False, True, id="missing-weights"), + pytest.param(False, True, True, id="disabled-head"), + ], +) +def test_gemma4_dspark_loads_confidence_head( + monkeypatch, enabled, has_weights, with_markov +) -> None: + """Use checkpoint confidence parameters, or disable an unavailable head.""" + config = SimpleNamespace( + vocab_size=64, + hidden_size=8, + target_layer_ids=[0, 1], + num_hidden_layers=0, + rms_norm_eps=1e-6, + markov_rank=4, + enable_confidence_head=enabled, + confidence_head_with_markov=with_markov, + ) + vllm_config = SimpleNamespace( + compilation_config=CompilationConfig(mode=CompilationMode.NONE), + model_config=SimpleNamespace(dtype=torch.bfloat16), + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(hf_config=config), + ), + ) + monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None) + model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config)) + width = config.hidden_size + (config.markov_rank if with_markov else 0) + weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16 + bias = torch.tensor([-0.5], dtype=torch.bfloat16) + loaded = model.load_weights( + [("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)] + if has_weights + else [] + ) + if not (enabled and has_weights): + assert model.model.confidence_head is None + assert not loaded + return + + assert loaded == { + "model.confidence_head.proj.weight", + "model.confidence_head.proj.bias", + } + assert model.model.confidence_head.proj.weight.dtype == torch.float32 + hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16) + markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16) + inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden + expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1) + torch.testing.assert_close(model.compute_confidence(hidden, markov), expected) + + @pytest.mark.cpu_test @pytest.mark.usefixtures("dist_init") def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None: @@ -164,7 +227,9 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None: ) as llm: if model == "google/gemma-3-4b-it": normalizers = llm.llm.collective_rpc( - lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501 + lambda self: ( + self.model_runner.model.language_model.model.normalizer.cpu().item() + ) # noqa: E501 ) config = llm.llm.llm_engine.model_config.hf_config.text_config else: diff --git a/tests/models/language/generation/test_hybrid.py b/tests/models/language/generation/test_hybrid.py index b77670956a..1b90429feb 100644 --- a/tests/models/language/generation/test_hybrid.py +++ b/tests/models/language/generation/test_hybrid.py @@ -73,6 +73,7 @@ def _set_conv_state_layout(monkeypatch, layout: str) -> None: def test_models( hf_runner, vllm_runner, + gpu_memory_cleared, example_prompts, monkeypatch, model: str, diff --git a/tests/models/language/pooling/test_jina_embeddings_v5.py b/tests/models/language/pooling/test_jina_embeddings_v5.py index fbfebcc5ce..21c82e1419 100644 --- a/tests/models/language/pooling/test_jina_embeddings_v5.py +++ b/tests/models/language/pooling/test_jina_embeddings_v5.py @@ -14,7 +14,7 @@ from types import SimpleNamespace from typing import cast import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import ModelConfig from vllm.model_executor.models.config import ( @@ -23,7 +23,7 @@ from vllm.model_executor.models.config import ( ) -def _model_config(hf_config: PretrainedConfig) -> ModelConfig: +def _model_config(hf_config: PreTrainedConfig) -> ModelConfig: """Minimal stand-in for ModelConfig; only hf_config is read.""" return cast(ModelConfig, SimpleNamespace(hf_config=hf_config)) @@ -42,7 +42,7 @@ def test_encoder_backbone_enables_bidirectional_attention(): EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder implementation. """ - hf_config = PretrainedConfig(is_decoder=False) + hf_config = PreTrainedConfig(is_decoder=False) JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config)) @@ -55,13 +55,13 @@ def test_supported_decoder_backbone_is_accepted(): `-small` omits `is_decoder` entirely, so an absent attribute has to be treated as a decoder. The first assertion pins that assumption: if - PretrainedConfig ever gains an `is_decoder=False` default, this fails here + PreTrainedConfig ever gains an `is_decoder=False` default, this fails here rather than silently rejecting a supported checkpoint. """ - absent = PretrainedConfig() + absent = PreTrainedConfig() assert not hasattr(absent, "is_decoder") JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(absent)) JinaEmbeddingsV5ModelConfig.verify_and_update_model_config( - _model_config(PretrainedConfig(is_decoder=True)) + _model_config(PreTrainedConfig(is_decoder=True)) ) diff --git a/tests/models/multimodal/generation/test_pixtral.py b/tests/models/multimodal/generation/test_pixtral.py index 9be8f8efab..2c509b2a7e 100644 --- a/tests/models/multimodal/generation/test_pixtral.py +++ b/tests/models/multimodal/generation/test_pixtral.py @@ -6,15 +6,8 @@ from typing import TYPE_CHECKING, Any import pytest import torch -from mistral_common.multimodal import download_image -from mistral_common.protocol.instruct.chunk import ImageURLChunk -from mistral_common.protocol.instruct.request import ChatCompletionRequest -from mistral_common.tokens.tokenizers.mistral import MistralTokenizer -from mistral_common.tokens.tokenizers.multimodal import image_from_chunk -from transformers import AutoProcessor -from vllm import SamplingParams, TextPrompt, TokensPrompt -from vllm.inputs import MultiModalDataBuiltins +from vllm import SamplingParams from vllm.logprobs import Logprob, SampleLogprobs from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata from vllm.platforms import current_platform @@ -56,59 +49,6 @@ def _create_msg_format(urls: list[str]) -> list[dict[str, Any]]: ] -def _create_msg_format_hf(urls: list[str]) -> list[dict[str, Any]]: - return [ - { - "role": "user", - "content": [ - { - "type": "text", - "content": PROMPT, - }, - *({"type": "image", "image": download_image(url)} for url in urls), - ], - } - ] - - -def _create_engine_inputs(urls: list[str]) -> TokensPrompt: - msg = _create_msg_format(urls) - - tokenizer = MistralTokenizer.from_model("pixtral") - - request = ChatCompletionRequest(messages=msg) # type: ignore[type-var] - tokenized = tokenizer.encode_chat_completion(request) - - engine_inputs = TokensPrompt(prompt_token_ids=tokenized.tokens) - - images = [] - for chunk in request.messages[0].content: - if isinstance(chunk, ImageURLChunk): - images.append(image_from_chunk(chunk)) - - mm_data = MultiModalDataBuiltins(image=images) - engine_inputs["multi_modal_data"] = mm_data - - return engine_inputs - - -def _create_engine_inputs_hf(urls: list[str]) -> TextPrompt: - msg = _create_msg_format_hf(urls) - - tokenizer = AutoProcessor.from_pretrained("mistral-community/pixtral-12b") - prompt = tokenizer.apply_chat_template(msg) - - images = [] - for chunk in msg[0]["content"]: - if chunk["type"] == "image": - images.append(chunk["image"]) - - mm_data = MultiModalDataBuiltins(image=images) - engine_inputs = TextPrompt(prompt=prompt, multi_modal_data=mm_data) - - return engine_inputs - - SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5) LIMIT_MM_PER_PROMPT = dict(image=4) diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index ae3f376e1f..1a7931082a 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -6,7 +6,6 @@ typically specific to a small subset of models. """ import types -from pathlib import PosixPath import numpy as np import numpy.typing as npt @@ -29,7 +28,7 @@ from vllm.logprobs import SampleLogprobs from vllm.platforms import current_platform from vllm.utils.collection_utils import is_list_of -from .....conftest import HfRunner, ImageAsset, ImageTestAssets +from .....conftest import HfRunner, ImageTestAssets from .types import RunnerOutput @@ -48,17 +47,6 @@ def blip2_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutp return hf_output_ids, hf_output_str, out_logprobs -def qwen_vllm_to_hf_output( - vllm_output: RunnerOutput, model: str -) -> tuple[list[int], str, SampleLogprobs | None]: - """Sanitize vllm output [qwen models] to be comparable with hf output.""" - output_ids, output_str, out_logprobs = vllm_output - - hf_output_str = output_str + "<|endoftext|>" - - return output_ids, hf_output_str, out_logprobs - - def qwen2_vllm_to_hf_output( vllm_output: RunnerOutput, model: str ) -> tuple[list[int], str, SampleLogprobs | None]: @@ -239,37 +227,6 @@ def get_llava_embeddings(image_assets: ImageTestAssets): return [asset.image_embeds for asset in image_assets] -####### Prompt path encoders for models that need models on disk -def qwen_prompt_path_encoder( - tmp_path: PosixPath, prompt: str, assets: list[ImageAsset] | ImageTestAssets -) -> str: - """Given a temporary dir path, export one or more image assets into the - tempdir & replace its contents with the local path to the string so that - the HF version of Qwen-VL can resolve the path and load the image in its - forward() call. - - Args: - tmp_path: Tempdir for test under consideration. - prompt: Prompt with image placeholders. - assets: list of image assets whose len equals the num placeholders. - - """ - # Ensure that the number of placeholders matches the number of assets; - # If this is not true, the test is probably written incorrectly. - assert prompt.count("") == len(assets) - - # Replace the placeholders with local paths to the exported assets - for asset in assets: - image_tmp_path = tmp_path / f"{asset.name}.jpg" - asset.pil_image.save(image_tmp_path) - prompt = prompt.replace( - "", - f"{image_tmp_path}", - 1, - ) - return prompt - - ####### Model-specific HuggingFace runner patchers def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner: """Patches and returns an instance of the HfRunner to use for GLM4.""" diff --git a/tests/models/multimodal/pooling/test_jinavl_reranker.py b/tests/models/multimodal/pooling/test_jinavl_reranker.py index 05433e7269..dca2403e35 100644 --- a/tests/models/multimodal/pooling/test_jinavl_reranker.py +++ b/tests/models/multimodal/pooling/test_jinavl_reranker.py @@ -14,11 +14,6 @@ from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam from ....conftest import HfRunner, VllmRunner -pytestmark = pytest.mark.skip( - reason="jinaai/jina-reranker-m0 custom code is incompatible with " - "transformers v5 (missing all_tied_weights_keys)" -) - MODELS = ["jinaai/jina-reranker-m0"] MM_PROCESSOR_KWARGS = { @@ -29,8 +24,8 @@ MM_PROCESSOR_KWARGS = { LIMIT_MM_PER_PROMPT = {"image": 2} CHECKPOINT_TO_HF_MAPPER = { - "visual.": "model.visual.", - "model.": "model.language_model.", + r"^visual\.": "model.visual.", + r"^model\.(?!language_model\.|visual\.)": "model.language_model.", } HANDELSBLATT_IMAGE_URL = ( diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index f223145d98..c3adccb27d 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -22,19 +22,19 @@ from unittest.mock import MagicMock import numpy as np import pytest import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from tests.models.registry import HF_EXAMPLE_MODELS from vllm.config.multimodal import MultiModalDummyOptions -class MockAudioFlamingo3Config(PretrainedConfig): +class MockAudioFlamingo3Config(PreTrainedConfig): model_type = "audioflamingo3" def __init__(self, **kwargs): super().__init__(**kwargs) - self.audio_config = PretrainedConfig() - self.text_config = PretrainedConfig() + self.audio_config = PreTrainedConfig() + self.text_config = PreTrainedConfig() class MockAudioFlamingo3Processor: diff --git a/tests/models/multimodal/processing/test_bailing_moe_v3_vl.py b/tests/models/multimodal/processing/test_bailing_moe_v3_vl.py index 0ebc61909f..5ac47190dd 100644 --- a/tests/models/multimodal/processing/test_bailing_moe_v3_vl.py +++ b/tests/models/multimodal/processing/test_bailing_moe_v3_vl.py @@ -8,7 +8,7 @@ import pytest import torch from PIL import Image from tokenizers import Tokenizer, decoders, models, pre_tokenizers -from transformers import PreTrainedTokenizerFast +from transformers import TokenizersBackend from vllm.model_executor.models.bailing_moe_v3_vl import ( IMAGE_PLACEHOLDER, @@ -34,7 +34,7 @@ def processor(): add_prefix_space=False, use_regex=False ) backend.decoder = decoders.ByteLevel() - tokenizer = PreTrainedTokenizerFast( + tokenizer = TokenizersBackend( tokenizer_object=backend, additional_special_tokens=[ "<|vision_start|>", diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index fa23956017..20a5121e60 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -1,13 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json from collections.abc import Set as AbstractSet from functools import partial import numpy as np import pytest +import torch from PIL import Image +from tests.models.utils import build_model_context from vllm.config import ModelConfig from vllm.config.multimodal import ( AudioDummyOptions, @@ -16,10 +19,15 @@ from vllm.config.multimodal import ( MultiModalDummyOptions, VideoDummyOptions, ) +from vllm.distributed.ec_transfer.ec_connector.utils import ( + PlaceholderMetadataResolver, + collect_ec_item_metadata, +) +from vllm.entrypoints.chat_utils import _get_embeds_data from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache -from vllm.multimodal.inputs import batched_tensors_equal +from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext from vllm.platforms import current_platform from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config @@ -193,6 +201,79 @@ def get_token_prompt( return inputs.prompt +@pytest.mark.parametrize( + "model_id,durations", + [ + ("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]), + ("nvidia/audio-flamingo-3-hf", [31.0]), + ("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]), + ("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]), + ("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]), + ], +) +def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch): + """EC metadata preserves the raw-audio prompt without running the HF processor.""" + ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)}) + mm_config = ctx.model_config.multimodal_config + mm_config.enable_mm_embeds = True + mm_config.allow_missing_mm_embeddings = True + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + sample_rate = processor.info.get_feature_extractor().sampling_rate + audios = [ + np.zeros(int(duration * sample_rate), dtype=np.float32) + for duration in durations + ] + prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)}) + raw = processor( + prompt, + mm_items=processor.info.parse_mm_data({"audio": audios}), + hf_processor_mm_kwargs={}, + ) + features = [ + MultiModalFeatureSpec( + data=item, + modality="audio", + identifier=str(index), + mm_position=position, + ) + for index, (item, position) in enumerate( + zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"]) + ) + ] + published = collect_ec_item_metadata( + features, PlaceholderMetadataResolver(ctx.model_config) + ) + metadata = json.loads( + json.dumps([entry["metadata"] for entry in published.values()]) + ) + assert len(metadata) == len(audios) + assert all(metadata) + audio_data = _get_embeds_data("audio", metadata, processor) + + def unexpected_preprocessing(*args, **kwargs): + pytest.fail("The EC consumer must not preprocess audio") + + monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing) + remote = processor( + prompt, + mm_items=processor.info.parse_mm_data({"audio": audio_data}), + hf_processor_mm_kwargs={}, + ) + assert remote["prompt_token_ids"] == raw["prompt_token_ids"] + assert remote["mm_placeholders"] == raw["mm_placeholders"] + # Omni also consumes these lengths when constructing M-RoPE positions. + if "audio_feature_lengths" in audio_data: + assert torch.equal( + remote["mm_kwargs"].get_data()["audio_feature_lengths"], + raw["mm_kwargs"].get_data()["audio_feature_lengths"], + ) + + mm_config.allow_missing_mm_embeddings = False + ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + with pytest.raises(ValueError, match="should contain the fields"): + ordinary_processor.info.parse_mm_data({"audio": audio_data}) + + def random_vision_chunk( rng: np.random.RandomState, min_wh: int, diff --git a/tests/models/multimodal/processing/test_h2ovl.py b/tests/models/multimodal/processing/test_h2ovl.py index 3ba256f3c7..0c2e2ff3a9 100644 --- a/tests/models/multimodal/processing/test_h2ovl.py +++ b/tests/models/multimodal/processing/test_h2ovl.py @@ -6,7 +6,7 @@ from collections.abc import Mapping import pytest from PIL import Image -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.image import rescale_image_size @@ -17,7 +17,7 @@ from ...utils import build_model_context def _get_expected_num_patches( - config: PretrainedConfig, + config: PreTrainedConfig, image: Image.Image, num_imgs: int, min_num: int, diff --git a/tests/models/multimodal/processing/test_internvl.py b/tests/models/multimodal/processing/test_internvl.py index 7954dd6b50..ab5fbe9a01 100644 --- a/tests/models/multimodal/processing/test_internvl.py +++ b/tests/models/multimodal/processing/test_internvl.py @@ -6,7 +6,7 @@ from collections.abc import Mapping import pytest from PIL import Image -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.image import rescale_image_size @@ -17,7 +17,7 @@ from ...utils import build_model_context def _get_expected_num_patches( - config: PretrainedConfig, + config: PreTrainedConfig, image: Image.Image, num_imgs: int, min_num: int, diff --git a/tests/models/multimodal/processing/test_nemotron_vl.py b/tests/models/multimodal/processing/test_nemotron_vl.py index be5c222fd2..b1c1e11892 100644 --- a/tests/models/multimodal/processing/test_nemotron_vl.py +++ b/tests/models/multimodal/processing/test_nemotron_vl.py @@ -6,7 +6,7 @@ from collections.abc import Mapping import pytest from PIL import Image -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.image import rescale_image_size @@ -17,7 +17,7 @@ from ...utils import build_model_context def _get_expected_num_patches( - config: PretrainedConfig, + config: PreTrainedConfig, image: Image.Image, num_imgs: int, min_num: int, diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 51f65188d7..5ea538555e 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -8,10 +8,11 @@ import pytest import torch from PIL import Image -from vllm.model_executor.models.vision import FusedInputNorm +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import batched_tensors_equal +from vllm.platforms import current_platform from ....conftest import ImageTestAssets from ...utils import build_model_context @@ -65,45 +66,6 @@ def test_jina_vl_processing_order() -> None: ] -@pytest.mark.parametrize( - ("image_mean", "image_std", "rescale_factor", "is_identity"), - [ - ([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0, True), - ([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255, False), - ], -) -def test_fused_input_norm_initialization_on_device( - monkeypatch: pytest.MonkeyPatch, - image_mean: list[float], - image_std: list[float], - rescale_factor: float, - is_identity: bool, -): - """Identity detection must not synchronize the default device.""" - original_allclose = torch.allclose - - def cpu_allclose(input: torch.Tensor, other: torch.Tensor, *args, **kwargs): - assert input.device.type == "cpu" - assert other.device.type == "cpu" - return original_allclose(input, other, *args, **kwargs) - - monkeypatch.setattr(torch, "allclose", cpu_allclose) - # Exercise the real accelerator when available. The meta device gives the - # CPU-only test shard the same non-CPU default-device semantics without - # requiring a CUDA-enabled PyTorch build. - default_device = "cuda" if torch.cuda.is_available() else "meta" - with torch.device(default_device): - input_norm = FusedInputNorm(image_mean, image_std, rescale_factor) - - assert input_norm.is_identity is is_identity - if is_identity: - assert input_norm.weight is None - assert input_norm.bias is None - else: - assert input_norm.weight.device.type == default_device - assert input_norm.bias.device.type == default_device - - @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) @pytest.mark.parametrize( ("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"), @@ -240,16 +202,20 @@ def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None: assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2) +@pytest.mark.usefixtures("default_vllm_config") @pytest.mark.parametrize( "model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"] ) @pytest.mark.parametrize("num_imgs", [1, 2]) def test_mm_device_do_normalize( - image_assets: ImageTestAssets, - model_id: str, - num_imgs: int, -): - """Ensure that enable mm_device_do_normalize yields the correct result.""" + image_assets: ImageTestAssets, model_id: str, num_imgs: int +) -> None: + """Device-side normalisation must reproduce the on-CPU processor result. + + Runs on any platform: the CPU platform exercises the ``forward_native`` + fallback, accelerators exercise the fused kernel. + """ + device = current_platform.device_type ctx = build_model_context( model_id, limit_mm_per_prompt={"image": num_imgs}, @@ -280,9 +246,15 @@ def test_mm_device_do_normalize( ].get_data()["pixel_values"] ctx.model_config.multimodal_config.mm_device_do_normalize = True - input_norm = FusedInputNorm.from_model_config(ctx.model_config) + input_norm = build_mm_input_norm(ctx.model_config).to(device) + + # With normalisation disabled, the processor emits raw uint8 pixels, + # matching the production mm_device_do_normalize path. + assert pixel_values_without_normalize.dtype == torch.uint8 pixel_values_do_input_norm = input_norm( - pixel_values_without_normalize.to(dtype), dtype + pixel_values_without_normalize.to(device), dtype ) - torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm) + torch.testing.assert_close( + pixel_values_with_normalize.to(device), pixel_values_do_input_norm + ) diff --git a/tests/models/multimodal/processing/test_smolvlm.py b/tests/models/multimodal/processing/test_smolvlm.py index f27c4eb7b8..02181d5d6f 100644 --- a/tests/models/multimodal/processing/test_smolvlm.py +++ b/tests/models/multimodal/processing/test_smolvlm.py @@ -17,6 +17,7 @@ from ...utils import build_model_context [ ({"max_image_size": {"longest_edge": 384}}, 1377), ({"max_image_size": {"longest_edge": 768}}, 405), + ({"do_image_splitting": False}, 81), ], ) @pytest.mark.parametrize("num_imgs", [1, 2]) @@ -63,6 +64,14 @@ def test_processor_override( # Ensure the placeholders format are correct hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) + num_patches = processor.info.get_num_patches( + image_width=dummy_image.width, + image_height=dummy_image.height, + processor=hf_processor, + mm_kwargs=hf_processor_mm_kwargs, + ) + assert num_patches == expected_toks_per_img // hf_processor.image_seq_len + hf_processed_inputs = hf_processor( text=prompt, images=mm_data["image"], diff --git a/tests/models/quantization/conftest.py b/tests/models/quantization/conftest.py new file mode 100644 index 0000000000..261f7934f4 --- /dev/null +++ b/tests/models/quantization/conftest.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared BF16 baselines for the quantization comparison tests. + +Several tests here quantize the same model and compare against an +unquantized run of it. Computing that reference once per session avoids +booting an identical engine for every quantization scheme under test. + +Only the outputs are cached; each engine is still torn down as usual, so +tests keep the isolation ``VllmRunner.__exit__`` provides. +""" + +import pytest + + +@pytest.fixture(scope="session") +def quant_baseline_logprobs(vllm_runner): + """Greedy logprobs for an unquantized model, computed once per config.""" + cache: dict = {} + + def baseline( + model: str, + prompts: list[str], + *, + max_model_len: int, + max_tokens: int, + num_logprobs: int, + ): + key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts)) + if key not in cache: + with vllm_runner( + model, + max_model_len=max_model_len, + enforce_eager=True, + ) as vllm_model: + cache[key] = vllm_model.generate_greedy_logprobs( + prompts, max_tokens, num_logprobs + ) + return cache[key] + + return baseline diff --git a/tests/models/quantization/test_fp8_per_channel.py b/tests/models/quantization/test_fp8_per_channel.py index f003ccd5f8..ef11b17968 100644 --- a/tests/models/quantization/test_fp8_per_channel.py +++ b/tests/models/quantization/test_fp8_per_channel.py @@ -37,6 +37,7 @@ NUM_LOG_PROBS = 8 @pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"]) def test_fp8_per_channel_logprobs( vllm_runner, + quant_baseline_logprobs, example_prompts, model: str, monkeypatch: pytest.MonkeyPatch, @@ -52,14 +53,13 @@ def test_fp8_per_channel_logprobs( with monkeypatch.context() as m: m.setenv("TOKENIZERS_PARALLELISM", "true") - with vllm_runner( + baseline_outputs = quant_baseline_logprobs( model, + example_prompts, max_model_len=MAX_MODEL_LEN, - enforce_eager=True, - ) as vllm_model: - baseline_outputs = vllm_model.generate_greedy_logprobs( - example_prompts, MAX_TOKENS, NUM_LOG_PROBS - ) + max_tokens=MAX_TOKENS, + num_logprobs=NUM_LOG_PROBS, + ) with vllm_runner( model, diff --git a/tests/models/quantization/test_mxfp8.py b/tests/models/quantization/test_mxfp8.py index 0463468eae..06b2b3cccd 100644 --- a/tests/models/quantization/test_mxfp8.py +++ b/tests/models/quantization/test_mxfp8.py @@ -42,6 +42,7 @@ NUM_LOG_PROBS = 8 @pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"]) def test_mxfp8_logprobs( vllm_runner, + quant_baseline_logprobs, example_prompts, model: str, monkeypatch: pytest.MonkeyPatch, @@ -56,14 +57,13 @@ def test_mxfp8_logprobs( with monkeypatch.context() as m: m.setenv("TOKENIZERS_PARALLELISM", "true") - with vllm_runner( + baseline_outputs = quant_baseline_logprobs( model, + example_prompts, max_model_len=MAX_MODEL_LEN, - enforce_eager=True, - ) as vllm_model: - baseline_outputs = vllm_model.generate_greedy_logprobs( - example_prompts, MAX_TOKENS, NUM_LOG_PROBS - ) + max_tokens=MAX_TOKENS, + num_logprobs=NUM_LOG_PROBS, + ) with vllm_runner( model, diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index e7d0e72d8a..85928d293b 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -13,17 +13,24 @@ from torch.nn import functional as F import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module -import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module +import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module +from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ( ModelOptMixedPrecisionConfig, ModelOptNvFp4Config, ) +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig +from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer +from vllm.models.qwen4_exp.amd.ple_layer import ( + Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD, +) from vllm.models.qwen4_exp.common.ple import ( PLEShardOverlap, compute_ple_shard_overlap, copy_ple_embedding_shard_, ) +from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpNGramEmbedding, Qwen4ExpPLEDeviceEmbedding, @@ -33,6 +40,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpPLEUnquantizedEmbeddingMethod, ) from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer +from vllm.utils.torch_utils import weak_ref_tensor from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionMetadata, ) @@ -396,7 +404,6 @@ def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None: def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None: prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" quant_config = Fp8Config( - is_checkpoint_fp8_serialized=True, ignored_layers=[], weight_block_size=[128, 128], ) @@ -422,12 +429,11 @@ def test_ple_embedding_rejects_unsupported_quantization_configs() -> None: with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"): Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix) - dynamic_fp8_config = Fp8Config( - is_checkpoint_fp8_serialized=False, - ignored_layers=[], + online_fp8_config = OnlineQuantizationConfig( + QuantizationConfigArgs(linear="fp8_per_tensor") ) - with pytest.raises(NotImplementedError, match="serialized FP8"): - Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix) + with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"): + Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix) def test_ple_embedding_respects_modelopt_exclusion() -> None: @@ -885,6 +891,40 @@ def test_fused_ngram_ids_correctness( assert torch.all((actual >= offsets) & (actual < offsets + sizes)) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused PLE needs CUDA") +def test_ngram_prefetch_ids_outlive_start_prefetch() -> None: + """Eager breaks hand the side-stream lookup weak refs, so the ids must stay + valid after start_prefetch returns rather than be reused from the pool.""" + device = torch.device("cuda") + query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device) + input_ids = torch.arange(20, 25, dtype=torch.int32, device=device) + ngram_context = torch.tensor([[11, 12], [13, 14]], dtype=torch.int32, device=device) + params = _ngram_hash_params(device, ngram_context.shape[1]) + module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding) + nn.Module.__init__(module) + for name in ("layer_multipliers", "ngram_heads_vocab_sizes", "ngram_heads_offsets"): + module.register_buffer(name, params[name]) + module.eos_token_id = params["eos_token_id"] + module.heads_per_ngram = params["heads_per_ngram"] + num_heads = params["ngram_heads_vocab_sizes"].numel() + module._prefetch_ids = torch.empty(8, num_heads, dtype=torch.long, device=device) + prefetched: list[torch.Tensor] = [] + module.ngram_embedding = SimpleNamespace( + supports_prefetch=True, + start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)), + ) + expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params) + + # A fresh pool makes the next same-size allocation reuse any freed ids, + # like a later CUDA graph segment would. + pool = torch.cuda.MemPool() + with torch.cuda.use_mem_pool(pool): + module.start_prefetch(None, input_ids, query_start_loc, ngram_context) + torch.full_like(expected, -1) + + assert torch.equal(prefetched[0], expected) + + def _short_conv_dilated_decode_pytorch( x_d: torch.Tensor, conv_state: torch.Tensor, @@ -1715,3 +1755,211 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None: expected_normed = grouped_norm(expected_gated, norm_conv) assert torch.equal(gated, expected_gated) torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2) + + +def _build_amd_ngram_embedding( + monkeypatch: pytest.MonkeyPatch, + *, + device: str, + cpu_offload: bool, + fp8_checkpoint: bool, +) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]: + """Construct and load a small AMD embedding through its checkpoint loader.""" + _mock_etp_group(monkeypatch) + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + + monkeypatch.setattr( + amd_ple_layer, + "get_current_vllm_config", + lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)), + ) + config = Qwen4ExpTextConfig( + ngram_size=3, + heads_per_ngram=1, + ngram_vocab_size_base=5, + make_ngram_vocab_size_divisible_by=8, + split_ngram_parts=2, + ple_embed_dim=6, + eos_token_id=0, + vocab_size=64, + ) + with torch.device(device): + module = amd_ple_layer.Qwen4ExpNGramEmbedding( + config, + embedding_dim=config.ple_embed_dim, + ple_dense_layer_id=0, + max_total_tokens=4, + max_num_reqs=2, + prefix="test.ple_embedding", + layer_name="test.ple", + quant_config=( + Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None + ), + params_dtype=torch.bfloat16, + ) + # Only the outer owner is a stub; the embedding constructor and loader run. + layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD) + nn.Module.__init__(layer) + layer.ple_embedding = module + + monkeypatch.setattr( + amd_ple_layer, + "get_forward_context", + lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}), + ) + + embedding = module.ngram_embedding + loaded_weight = ( + torch.arange(embedding.org_vocab_size * embedding.embedding_dim) + .reshape(embedding.org_vocab_size, embedding.embedding_dim) + .to(embedding.weight.dtype) + ) + shard_size = (embedding.org_vocab_size + 1) // 2 + weights = [ + (f"ngram_embedding.shard_{i}.weight", shard) + for i, shard in enumerate(loaded_weight.split(shard_size)) + ] + expected_loaded = {"ngram_embedding.weight"} + if fp8_checkpoint: + weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25]))) + expected_loaded.add("ngram_embedding.weight_scale") + assert module.load_weights(weights) == expected_loaded + embedding.quant_method.process_weights_after_loading(embedding) + return module, loaded_weight + + +@pytest.mark.parametrize( + "device,cpu_offload", + [ + pytest.param("cpu", False, id="cpu-resident"), + pytest.param( + "cuda:0", + False, + id="gpu-resident", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA or ROCm" + ), + ), + pytest.param( + "cuda:0", + True, + id="gpu-offloaded", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA or ROCm" + ), + ), + ], +) +def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale( + monkeypatch: pytest.MonkeyPatch, + device: str, + cpu_offload: bool, +) -> None: + """AMD checkpoint loading must accept and apply the global FP8 scale.""" + module, loaded_weight = _build_amd_ngram_embedding( + monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True + ) + embedding = module.ngram_embedding + assert embedding.weight.dtype == torch.float8_e4m3fn + assert embedding.weight.device == torch.device("cpu" if cpu_offload else device) + if cpu_offload: + assert embedding.weight.is_pinned() + assert embedding.weight_scale.device == torch.device(device) + torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float()) + dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16) + torch.testing.assert_close( + dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0 + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) +def test_amd_pinned_embedding_output_written_under_compile( + monkeypatch: pytest.MonkeyPatch, + fp8_checkpoint: bool, +) -> None: + """The AMD pinned lookup op must survive aot_autograd/Inductor. + + The op writes via the declared-mutated ``output``. Without + ``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup + never runs, and the returned ``output`` stays untouched. + """ + module, loaded_weight = _build_amd_ngram_embedding( + monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint + ) + + ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") + output = torch.zeros( + ngram_ids.shape[0], + module.embedding_dim, + dtype=module.ngram_embedding.weight.dtype, + device="cuda:0", + ) + + def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor: + torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( + ids, out, module.layer_name + ) + return out + + torch.compile(lookup, fullgraph=True)(ngram_ids, output) + torch.cuda.current_stream().synchronize() + + expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) + torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) +def test_amd_pinned_embedding_output_written_under_cudagraph_capture( + monkeypatch: pytest.MonkeyPatch, + fp8_checkpoint: bool, +) -> None: + """The AMD pinned lookup op must be capture-safe. + + It launches a single kernel on the current stream, so a cudagraph capture + reproduces the lookup into ``output`` without cross-stream work. + """ + module, loaded_weight = _build_amd_ngram_embedding( + monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint + ) + + ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") + output = torch.zeros( + ngram_ids.shape[0], + module.embedding_dim, + dtype=module.ngram_embedding.weight.dtype, + device="cuda:0", + ) + + def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor: + torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( + ids, out, module.layer_name + ) + return out + + graph = torch.cuda.CUDAGraph() + warmup_stream = torch.cuda.Stream(device="cuda:0") + warmup_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup_stream): + for _ in range(3): + lookup(ngram_ids, output) + torch.cuda.current_stream().wait_stream(warmup_stream) + + with torch.cuda.graph(graph): + lookup(ngram_ids, output) + for _ in range(2): + ngram_ids.copy_(ngram_ids.flip(-1) + 3) + output.fill_(float("nan")) + graph.replay() + torch.cuda.current_stream().synchronize() + + expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) + torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) diff --git a/tests/models/registry.py b/tests/models/registry.py index 6a0fe8318e..45d1a0a4f5 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -7,7 +7,7 @@ from typing import Any, Literal import pytest from packaging.version import Version -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers import __version__ as TRANSFORMERS_VERSION from vllm.config.model import ModelDType, TokenizerMode @@ -1233,7 +1233,7 @@ _MULTIMODAL_EXAMPLE_MODELS = { # NemotronH layers are constructed via `hybrid_override_pattern` use_original_num_layers=True, hf_overrides={ - "vision_config": PretrainedConfig( + "vision_config": PreTrainedConfig( args={ "min_num_patches": 1, "max_num_patches": 12, @@ -1502,6 +1502,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env max_num_seqs=32, ), + # LiLiCorr checkpoints are not published yet. + "LiLiCorrDraftModel": _HfExamplesInfo( + "Qwen/Qwen3-8B", + speculative_model="LiLiCorrDraftModel", + is_available_online=False, + use_original_num_layers=True, + ), "DFlash2DraftModel": _HfExamplesInfo( "Qwen/Qwen3.8-27B", speculative_model="z-lab/Qwen3.8-27B-DFlash2", diff --git a/tests/models/test_adapters.py b/tests/models/test_adapters.py index 7547dbd4ef..62442e332e 100644 --- a/tests/models/test_adapters.py +++ b/tests/models/test_adapters.py @@ -4,7 +4,7 @@ import pytest import torch -from transformers import Gemma3Config, PretrainedConfig, Qwen2Config +from transformers import Gemma3Config, PreTrainedConfig, Qwen2Config from vllm.model_executor.models.adapters import ( _create_pooling_model_cls, @@ -266,14 +266,14 @@ def test_resolve_num_labels_defaults_when_undeclared(): config = _composite_config() assert ( _resolve_num_labels(config, config.get_text_config()) - == PretrainedConfig().num_labels + == PreTrainedConfig().num_labels ) def test_resolve_num_labels_declared_on_outer_config(): """Multimodal checkpoints keep id2label/problem_type on the top-level config.""" config = _composite_config(outer_labels=20) - assert config.get_text_config().num_labels == PretrainedConfig().num_labels + assert config.get_text_config().num_labels == PreTrainedConfig().num_labels assert _resolve_num_labels(config, config.get_text_config()) == 20 diff --git a/tests/models/test_deepseek_v4_vl_rocm.py b/tests/models/test_deepseek_v4_vl_rocm.py index 098e5185fc..036efbaa42 100644 --- a/tests/models/test_deepseek_v4_vl_rocm.py +++ b/tests/models/test_deepseek_v4_vl_rocm.py @@ -16,6 +16,132 @@ pytestmark = pytest.mark.skipif( ) +@pytest.mark.parametrize("hash_routing", [False, True], ids=["regular", "hash"]) +@pytest.mark.parametrize("vision", [False, True], ids=["text", "vision"]) +@torch.inference_mode() +def test_rocm_moe_routing_and_shared_experts_match_reference( + hash_routing, vision, monkeypatch, default_vllm_config, dist_init +) -> None: + """AMD V4 routing, clamped experts and shared output agree with PyTorch.""" + from vllm.forward_context import set_forward_context + from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE + from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID + from vllm.utils.torch_utils import set_default_torch_dtype + from vllm.v1.worker.workspace import ( + init_workspace_manager, + is_workspace_manager_initialized, + ) + + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "0") + default_vllm_config.kernel_config.moe_backend = "triton" + config = SimpleNamespace( + hidden_size=512, + n_routed_experts=8, + num_experts_per_tok=2, + moe_intermediate_size=512, + swiglu_limit=0.05, + norm_topk_prob=True, + scoring_func="sqrtsoftplus", + routed_scaling_factor=1.5, + num_hash_layers=int(hash_routing), + vocab_size=IMAGE_SENTINEL_BASE_ID + 8, + topk_method="noaux_tc", + vision_n_layers=int(vision), + n_shared_experts=1, + hidden_act="silu", + ) + torch.manual_seed(42) + with torch.device("cuda"), set_default_torch_dtype(torch.bfloat16): + moe = DeepseekV4MoE( + SimpleNamespace( + model_config=SimpleNamespace(hf_config=config), + quant_config=None, + parallel_config=ParallelConfig(), + ), + prefix="model.layers.0.ffn", + ) + for parameter in moe.parameters(): + if parameter.is_floating_point(): + parameter.normal_(std=0.04) + hidden = torch.randn(17, config.hidden_size) * 0.1 + input_ids = torch.arange(17, dtype=torch.int32) + if vision: + # Include both ends of the five-token sentinel interval and the + # first ordinary token above it. + input_ids[1:4] = torch.tensor( + [ + IMAGE_SENTINEL_BASE_ID, + IMAGE_SENTINEL_BASE_ID + 4, + IMAGE_SENTINEL_BASE_ID + 5, + ], + dtype=torch.int32, + ) + if moe.gate.tid2eid is not None: + rows = torch.arange(config.vocab_size) + moe.gate.tid2eid.copy_(torch.stack((rows % 8, (rows + 3) % 8), dim=1)) + if moe.gate.e_score_correction_bias is not None: + moe.gate.e_score_correction_bias.copy_(torch.arange(8) * 0.1) + if moe.gate.bias_vl is not None: + moe.gate.bias_vl.copy_(torch.arange(8) * -0.1) + + scores = torch.nn.functional.softplus( + hidden.float() @ moe.gate.weight.float().T + ).sqrt() + bias = moe.gate.e_score_correction_bias + selection_scores = scores if bias is None else scores + bias + expected_ids = selection_scores.topk(2, dim=-1).indices + if hash_routing: + expected_ids = moe.gate.tid2eid[input_ids.long()].long() + if vision: + image_mask = (input_ids >= IMAGE_SENTINEL_BASE_ID) & ( + input_ids < IMAGE_SENTINEL_BASE_ID + 5 + ) + expected_ids[image_mask] = ( + (scores + moe.gate.bias_vl).topk(2, dim=-1).indices[image_mask] + ) + expected_weights = scores.gather(1, expected_ids) + expected_weights *= config.routed_scaling_factor / expected_weights.sum( + dim=-1, keepdim=True + ) + router_logits, _ = moe.gate(hidden) + weights, ids = moe.experts.router.select_experts( + hidden, router_logits, topk_indices_dtype=torch.int32, input_ids=input_ids + ) + torch.testing.assert_close(ids.long(), expected_ids) + torch.testing.assert_close(weights, expected_weights, rtol=1e-5, atol=1e-6) + + def expert_reference(x, gate_up_weight, down_weight): + gate, up = ( + torch.nn.functional.linear(x, gate_up_weight).float().chunk(2, dim=-1) + ) + gate = gate.clamp(max=config.swiglu_limit) + up = up.clamp(-config.swiglu_limit, config.swiglu_limit) + activated = (torch.nn.functional.silu(gate) * up).to(x.dtype) + return torch.nn.functional.linear(activated, down_weight).float() + + routed = moe.experts.routed_experts + expected = torch.zeros_like(hidden, dtype=torch.float32) + for expert_id in range(config.n_routed_experts): + token, slot = torch.where(expected_ids == expert_id) + expected[token] += ( + expert_reference( + hidden[token], routed.w13_weight[expert_id], routed.w2_weight[expert_id] + ) + * expected_weights[token, slot, None] + ) + shared = moe.shared_experts + expected += expert_reference( + hidden, shared.gate_up_proj.weight, shared.down_proj.weight + ) + moe.experts._quant_method.process_weights_after_loading(routed) + if not is_workspace_manager_initialized(): + init_workspace_manager(torch.device("cuda:0")) + with set_forward_context(None, default_vllm_config, num_tokens=hidden.shape[0]): + actual = moe(hidden, input_ids) + torch.testing.assert_close(actual.float(), expected, rtol=0.02, atol=5e-5) + + def test_rocm_packed_kv_cache_auto_uses_ds_mla_layout() -> None: from vllm.config import CacheConfig from vllm.models.deepseek_v4.attention import _resolve_dsv4_kv_cache_dtype diff --git a/tests/models/test_qwen3_5_mtp_config.py b/tests/models/test_qwen3_5_mtp_config.py index 6cf54dbc95..885995b906 100644 --- a/tests/models/test_qwen3_5_mtp_config.py +++ b/tests/models/test_qwen3_5_mtp_config.py @@ -5,7 +5,7 @@ from typing import Any import pytest -from transformers import AutoConfig, PretrainedConfig +from transformers import AutoConfig, PreTrainedConfig from vllm.config.speculative import SpeculativeConfig @@ -15,19 +15,19 @@ _CHECKPOINTS = { } -def _mtp_config(model_type: str) -> PretrainedConfig: +def _mtp_config(model_type: str) -> PreTrainedConfig: """Create a top-level MTP configuration with mtp_num_hidden_layers.""" kwargs: dict[str, Any] = { "model_type": model_type, "architectures": ["SomeArch"], "mtp_num_hidden_layers": 1, } - return PretrainedConfig(**kwargs) + return PreTrainedConfig(**kwargs) def _multimodal_wrapper_mtp_config( model_type: str, mtp_layers: int = 1 -) -> PretrainedConfig: +) -> PreTrainedConfig: """Download a multimodal wrapper config via AutoConfig.from_pretrained and configure mtp_num_hidden_layers in text_config. @@ -36,7 +36,7 @@ def _multimodal_wrapper_mtp_config( - MoE (qwen3_5_moe): Qwen/Qwen3.6-35B-A3B """ repo = _CHECKPOINTS[model_type] - config: PretrainedConfig = AutoConfig.from_pretrained(repo) + config: PreTrainedConfig = AutoConfig.from_pretrained(repo) text_config = config.get_text_config() text_config.mtp_num_hidden_layers = mtp_layers return config diff --git a/tests/models/test_registry.py b/tests/models/test_registry.py index b7b1339786..d269aeec13 100644 --- a/tests/models/test_registry.py +++ b/tests/models/test_registry.py @@ -88,70 +88,68 @@ def test_registry_imports(model_arch): assert supports_multimodal(model_cls) -@create_new_process_for_each_test() @pytest.mark.parametrize( - "model_arch,is_mm,init_cuda,score_type", + "model_arch,is_mm,score_type", [ - ("LlamaForCausalLM", False, False, "bi-encoder"), - ("LlavaForConditionalGeneration", True, True, "bi-encoder"), - ("DeepseekV41ForCausalLM", True, False, "bi-encoder"), - ("BertForSequenceClassification", False, False, "cross-encoder"), - ("RobertaForSequenceClassification", False, False, "cross-encoder"), - ("XLMRobertaForSequenceClassification", False, False, "cross-encoder"), - ("GteNewModel", False, False, "bi-encoder"), - ("GteNewForSequenceClassification", False, False, "cross-encoder"), - ("HF_ColBERT", False, False, "late-interaction"), + ("LlamaForCausalLM", False, "bi-encoder"), + ("LlavaForConditionalGeneration", True, "bi-encoder"), + ("DeepseekV41ForCausalLM", True, "bi-encoder"), + ("BertForSequenceClassification", False, "cross-encoder"), + ("RobertaForSequenceClassification", False, "cross-encoder"), + ("XLMRobertaForSequenceClassification", False, "cross-encoder"), + ("GteNewModel", False, "bi-encoder"), + ("GteNewForSequenceClassification", False, "cross-encoder"), + ("HF_ColBERT", False, "late-interaction"), ], ) -def test_registry_model_property(model_arch, is_mm, init_cuda, score_type): +def test_registry_model_property(model_arch, is_mm, score_type): model_info = ModelRegistry._try_inspect_model_cls(model_arch) assert model_info is not None assert model_info.supports_multimodal is is_mm assert model_info.score_type == score_type - if init_cuda and current_platform.is_cuda_alike(): - assert not torch.cuda.is_initialized() - ModelRegistry._try_load_model_cls(model_arch) - if not torch.cuda.is_initialized(): - warnings.warn( - "This model no longer initializes CUDA on import. " - "Please test using a different one.", - stacklevel=2, - ) - - -@create_new_process_for_each_test() @pytest.mark.parametrize( - "model_arch,is_pp,init_cuda", + "model_arch,is_pp", [ # TODO(woosuk): Re-enable this once the MLP Speculator is supported # in V1. - # ("MLPSpeculatorPreTrainedModel", False, False), - ("DeepseekV2ForCausalLM", True, False), - ("Qwen2VLForConditionalGeneration", True, True), + # ("MLPSpeculatorPreTrainedModel", False), + ("DeepseekV2ForCausalLM", True), + ("Qwen2VLForConditionalGeneration", True), ], ) -def test_registry_is_pp(model_arch, is_pp, init_cuda): +def test_registry_is_pp(model_arch, is_pp): model_info = ModelRegistry._try_inspect_model_cls(model_arch) assert model_info is not None assert model_info.supports_pp is is_pp - if init_cuda and current_platform.is_cuda_alike(): - assert not torch.cuda.is_initialized() - - ModelRegistry._try_load_model_cls(model_arch) - if not torch.cuda.is_initialized(): - warnings.warn( - "This model no longer initializes CUDA on import. " - "Please test using a different one.", - stacklevel=2, - ) - @create_new_process_for_each_test() +@pytest.mark.parametrize( + "model_arch", + ["LlavaForConditionalGeneration", "Qwen2VLForConditionalGeneration"], +) +def test_registry_inspect_does_not_init_cuda(model_arch): + """Inspecting a model must not import it: these architectures initialize + CUDA at import time, so the inspection path has to stay out-of-process.""" + if not current_platform.is_cuda_alike(): + pytest.skip("Requires a CUDA-like platform") + + assert ModelRegistry._try_inspect_model_cls(model_arch) is not None + assert not torch.cuda.is_initialized() + + ModelRegistry._try_load_model_cls(model_arch) + if not torch.cuda.is_initialized(): + warnings.warn( + "This model no longer initializes CUDA on import. " + "Please test using a different one.", + stacklevel=2, + ) + + @pytest.mark.parametrize( "model_arch,supported", [ diff --git a/tests/models/test_vision.py b/tests/models/test_vision.py index 94edce45bf..8d2a7d8a75 100644 --- a/tests/models/test_vision.py +++ b/tests/models/test_vision.py @@ -13,7 +13,6 @@ from vllm.distributed.parallel_state import ( initialize_model_parallel, ) from vllm.model_executor.models.vision import ( - FusedInputNorm, get_load_balance_assignment, resolve_visual_encoder_outputs, run_dp_sharded_mrope_vision_model, @@ -488,62 +487,3 @@ def test_simple_mrope_vision_model_spatial_merge(spatial_merge_size: int): assert output.shape[0] == expected_output_patches assert output.shape[1] == vision_model.out_hidden_size - - -def _reference_input_norm( - pixel_values: torch.Tensor, - image_mean: list[float], - image_std: list[float], - rescale_factor: float, - channel: int, - out_dtype: torch.dtype, -) -> torch.Tensor: - """Straightforward per-channel affine: (x * rescale - mean) / std.""" - patches, size = pixel_values.shape - patch_size = size // channel - mean = torch.tensor(image_mean, dtype=torch.float32).view(1, channel, 1) - std = torch.tensor(image_std, dtype=torch.float32).view(1, channel, 1) - x = pixel_values.to(torch.float32).view(patches, channel, patch_size) - x = (x * rescale_factor - mean) / std - return x.view(patches, size).to(out_dtype) - - -@pytest.mark.parametrize("num_patches", [1, 37, 70000]) -def test_fused_input_norm_matches_reference(num_patches: int): - """FusedInputNorm must equal the plain affine, including for num_patches - above the cuDNN batch-norm grid limit (~65535) that previously raised - CUDNN_STATUS_INTERNAL_ERROR (issue #51717).""" - channel = 3 - patch_size = 14 * 14 - image_mean = [0.48145466, 0.4578275, 0.40821073] - image_std = [0.26862954, 0.26130258, 0.27577711] - rescale_factor = 1.0 / 255.0 - - set_random_seed(0) - pixel_values = torch.randint( - 0, 256, (num_patches, channel * patch_size), dtype=torch.float32 - ) - - norm = FusedInputNorm( - image_mean=image_mean, - image_std=image_std, - rescale_factor=rescale_factor, - channel=channel, - ) - assert not norm.is_identity - - out = norm(pixel_values, visual_dtype=torch.float32) - expected = _reference_input_norm( - pixel_values, image_mean, image_std, rescale_factor, channel, torch.float32 - ) - torch.testing.assert_close(out, expected) - - -def test_fused_input_norm_identity_passthrough(): - """The identity configuration returns the input unchanged (cast only).""" - norm = FusedInputNorm.identity() - assert norm.is_identity - - pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32) - out = norm(pixel_values, visual_dtype=torch.bfloat16) - torch.testing.assert_close(out, pixel_values.to(torch.bfloat16)) diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py index 153f7863bd..256f83649a 100644 --- a/tests/models/transformers/fusers/test_rms_norm.py +++ b/tests/models/transformers/fusers/test_rms_norm.py @@ -50,6 +50,55 @@ class WeightlessRMSNorm(RMSNorm): return self._rms(x.to(torch.float32)).to(x.dtype) +class Gemma4RMSNorm(RMSNorm): + """`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`). + + Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep + Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not* + zero-centered, so this fuses to a plain `RMSNorm`. + """ + + def _rms(self, x): + mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon + return x * torch.pow(mean_squared, -0.5) + + def forward(self, x): + return (self._rms(x.float()) * self.weight.float()).type_as(x) + + +class PowOperatorRMSNorm(Gemma4RMSNorm): + """The `v ** -0.5` operator spelling of the same reciprocal square root.""" + + def _rms(self, x): + return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5 + + +class KwargPowRMSNorm(Gemma4RMSNorm): + """The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt.""" + + def _rms(self, x): + mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon + return x * torch.pow(mean_squared, exponent=-0.5) + + +class KwargBasePowRMSNorm(Gemma4RMSNorm): + """The fully functional spelling `torch.pow(input=v, exponent=-0.5)`.""" + + def _rms(self, x): + mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon + return x * torch.pow(input=mean_squared, exponent=-0.5) + + +class WeightlessGemma4RMSNorm(Gemma4RMSNorm): + """Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter.""" + + def __init__(self, hidden: int = 16, eps: float = 1e-6): + super().__init__(hidden, eps, weight=False) + + def forward(self, x): + return self._rms(x.float()).type_as(x) + + class LayerNorm(RMSNorm): """An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast).""" @@ -156,6 +205,11 @@ class UntraceableGatedRMSNorm(RMSNorm): (RMSNorm, 1e-5, False), (GemmaRMSNorm, 1e-6, True), (WeightlessRMSNorm, 1e-6, False), + (Gemma4RMSNorm, 1e-6, False), + (PowOperatorRMSNorm, 1e-6, False), + (KwargPowRMSNorm, 1e-6, False), + (KwargBasePowRMSNorm, 1e-6, False), + (WeightlessGemma4RMSNorm, 1e-6, False), (LayerNorm, 1e-6, False), (torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op ], @@ -188,6 +242,10 @@ def test_gated_rms_norm_is_not_fused(cls): (RMSNorm, "RMSNorm", False), (GemmaRMSNorm, "GemmaRMSNorm", True), (WeightlessRMSNorm, "RMSNorm", False), + (Gemma4RMSNorm, "RMSNorm", False), + (KwargPowRMSNorm, "RMSNorm", False), + (KwargBasePowRMSNorm, "RMSNorm", False), + (WeightlessGemma4RMSNorm, "RMSNorm", False), ], ) def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config): @@ -213,6 +271,25 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0) +@pytest.mark.parametrize( + "cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm] +) +def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config): + """Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square + root: the fused norm must reproduce the unfused forward, not just replace it. + `pow` and `rsqrt` take different paths to the same value, so compare at float32 + tolerance rather than bit-for-bit.""" + torch.manual_seed(0) + module = cls(16) + with torch.no_grad(): + module.weight.copy_(torch.randn(16)) + built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config) + with torch.no_grad(): + built.weight.copy_(module.weight) + x = torch.randn(4, 16) + torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6) + + def test_weightless_norm_has_no_hidden_size(default_vllm_config): """A weightless norm states no hidden size, and the model's (LM) hidden size would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index fd0d980def..fa5e14eddd 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -13,7 +13,7 @@ from typing import Any import pytest import torch import torch.nn as nn -from transformers import AutoConfig, AutoModel, PretrainedConfig +from transformers import AutoConfig, AutoModel, PreTrainedConfig from vllm.config import ModelConfig, VllmConfig from vllm.model_executor.models.interfaces import SupportsMultiModal @@ -524,7 +524,7 @@ def replace_vocab_embeddings(model, **config_kwargs): stub = nn.Module() stub.model = model - stub.config = PretrainedConfig( + stub.config = PreTrainedConfig( vocab_size=VOCAB_SIZE, num_positions=NUM_POSITIONS, **config_kwargs ) embeddings = Base._vocab_embeddings(stub) diff --git a/tests/models/utils.py b/tests/models/utils.py index e4cc5e673b..003ef1a078 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -8,7 +8,7 @@ from typing import Any import torch import torch.nn.functional as F -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption from vllm.config.pooler import SequencePoolingType, TokenPoolingType @@ -446,12 +446,12 @@ def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs): def dummy_hf_overrides( - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, *, model_arch: str = "", exist_overrides: dict[str, Any] | None = None, use_original_num_layers: bool = False, -) -> PretrainedConfig: +) -> PreTrainedConfig: """Dummy HF overrides function used to create dummy model with only minimum nums of layer. """ diff --git a/tests/multimodal/test_parse.py b/tests/multimodal/test_parse.py index 2649f80777..c20677fc25 100644 --- a/tests/multimodal/test_parse.py +++ b/tests/multimodal/test_parse.py @@ -15,6 +15,45 @@ from vllm.multimodal.parse import ( H, W = 480, 640 +class AudioMetadataParser(MultiModalDataParser): + embedding_fields = { + "audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"}, + } + + +@pytest.mark.parametrize("allow_missing", [False, True]) +def test_audio_metadata_requires_ec_consumer(allow_missing): + parser = AudioMetadataParser(allow_missing_mm_embeddings=allow_missing) + data = {"audio_num_tokens": torch.tensor([[3], [5]])} + if not allow_missing: + with pytest.raises(ValueError, match="audio_embeds"): + parser.parse_mm_data({"audio": data}) + else: + items = parser.parse_mm_data({"audio": data})["audio"] + assert len(items) == 2 + assert items.get(1)["audio_num_tokens"].item() == 5 + assert items.get_processor_data() == {} + + +@pytest.mark.parametrize("counts", [[0], [-1], [1.5], [True], [[1, 2]]]) +def test_audio_metadata_rejects_invalid_token_counts(counts): + parser = AudioMetadataParser(allow_missing_mm_embeddings=True) + with pytest.raises(ValueError, match="positive integer"): + parser.parse_mm_data({"audio": {"audio_num_tokens": torch.tensor(counts)}}) + + +def test_audio_metadata_checks_supplied_embedding_lengths(): + parser = AudioMetadataParser() + data = { + "audio_num_tokens": torch.tensor([3, 5]), + "audio_embeds": [torch.zeros(3, 8), torch.zeros(5, 8)], + } + assert len(parser.parse_mm_data({"audio": data})["audio"]) == 2 + data["audio_num_tokens"] = torch.tensor([3, 4]) + with pytest.raises(ValueError, match="does not match"): + parser.parse_mm_data({"audio": data}) + + @pytest.mark.parametrize( "image", [ diff --git a/tests/parser/engine/replay_harness.py b/tests/parser/engine/replay_harness.py index 5eec8d213a..a33121705d 100644 --- a/tests/parser/engine/replay_harness.py +++ b/tests/parser/engine/replay_harness.py @@ -34,6 +34,7 @@ class Sample: tools: list[dict] | None = None chat_template_kwargs: dict | None = None prompt_token_ids: list[int] | None = None + content_lstrip: str | None = None @dataclass diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index 2bc4338785..5cd48dd870 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -99,6 +99,10 @@ def _discover_pairings() -> list[_PairingInfo]: # terminal matching; combined-parser replay coverage lives in # test_inkling.py. continue + if cfg.name == "granite": + # Granite has a JSON-array tool body with no TOOL_END terminal; + # its replay coverage lives in test_granite.py. + continue parser_cls = type( f"_Delegating{engine_cls.__name__}", diff --git a/tests/parser/engine/test_granite.py b/tests/parser/engine/test_granite.py new file mode 100644 index 0000000000..ddc7e2bfe0 --- /dev/null +++ b/tests/parser/engine/test_granite.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Granite parser. + +Granite emits tool calls as a JSON array following a marker +(``<|tool_call|>`` for 3.0, ```` for 3.1) with no closing +terminal; the ``]`` ends the region and surrounding prose is content. The +engine's ``tool_call_body_array`` mode splits the array into one call per +element and ``_granite_arg_converter`` carves each element's ``arguments``. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite +from vllm.parser.granite import GraniteParser, _granite_arg_converter + +TOOL_TOKEN = "<|tool_call|>" +TOOL_STRING = "" + +_GRANITE_VOCAB = {TOOL_TOKEN: 49154} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_GRANITE_VOCAB) + + +@pytest.fixture +def parser(mock_tokenizer): + return GraniteParser(mock_tokenizer) + + +def _tokenize(text: str) -> list[tuple[int, str]]: + """Tokenize like the real stream: ``<|tool_call|>`` is one special token, + plain text becomes one token per character.""" + tokens: list[tuple[int, str]] = [] + i = 0 + while i < len(text): + if text.startswith(TOOL_TOKEN, i): + tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN)) + i += len(TOOL_TOKEN) + else: + tokens.append((ord(text[i]), text[i])) + i += 1 + return tokens + + +def _stream(parser, request, text: str, chunk_size: int): + tokens = _tokenize(text) + results = [] + previous_text = "" + previous_token_ids: list[int] = [] + for start in range(0, len(tokens), chunk_size): + batch = tokens[start : start + chunk_size] + delta_text = "".join(t for _, t in batch) + delta_token_ids = [tid for tid, _ in batch] + current_text = previous_text + delta_text + current_token_ids = previous_token_ids + delta_token_ids + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=tuple(previous_token_ids), + current_token_ids=tuple(current_token_ids), + delta_token_ids=tuple(delta_token_ids), + request=request, + ) + results.append(delta) + previous_text = current_text + previous_token_ids = current_token_ids + finish = parser.finish_streaming() + if finish is not None: + results.append(finish) + return results + + +def _collect_content(results) -> str: + return "".join(d.content for d in results if d and d.content) + + +def _collect_names(results) -> list[str]: + names: dict[int, str] = {} + for d in results: + if not (d and d.tool_calls): + continue + for tc in d.tool_calls: + if tc.function and tc.function.name: + names[tc.index] = names.get(tc.index, "") + tc.function.name + return [names[i] for i in sorted(names)] + + +def _collect_args(results) -> dict[int, str]: + args: dict[int, str] = {} + for d in results: + if not (d and d.tool_calls): + continue + for tc in d.tool_calls: + if tc.function and tc.function.arguments: + args[tc.index] = args.get(tc.index, "") + tc.function.arguments + return args + + +class TestArgConverter: + def test_complete_wrapper(self): + raw = '{"name": "get_weather", "arguments": {"city": "SF"}}' + assert _granite_arg_converter(raw, False) == '{"city": "SF"}' + + def test_arguments_before_name(self): + raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}' + assert _granite_arg_converter(raw, False) == '{"city": "SF"}' + + def test_partial_before_arguments(self): + assert _granite_arg_converter('{"name": "get_w', True) == "" + + def test_partial_inside_arguments(self): + raw = '{"name": "x", "arguments": {"a": 1' + assert _granite_arg_converter(raw, True) == '{"a": 1' + + def test_prefix_stability(self): + # Each growing prefix must extend the previous converter output. + full = '{"name": "x", "arguments": {"city": "Tokyo"}}' + prev = "" + for i in range(1, len(full) + 1): + out = _granite_arg_converter(full[:i], True) + assert out.startswith(prev) or prev.startswith(out) or out == "" + if out: + prev = out + + def test_missing_arguments_defaults(self): + assert _granite_arg_converter('{"name": "x"}', False) == "{}" + assert _granite_arg_converter('{"name": "x"}', True) == "" + + def test_non_object_arguments_rejected(self): + with pytest.raises(ValueError, match="JSON object"): + _granite_arg_converter('{"name": "x", "arguments": [1]}', False) + + +class TestNonStreaming: + def test_plain_text(self, parser, mock_request): + reasoning, content, tools = parser.parse("Just a reply.", mock_request) + assert reasoning is None + assert content == "Just a reply." + assert tools is None + + def test_single_tool_token_marker(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + _, content, tools = parser.parse(text, mock_request) + assert content is None + assert [t.name for t in tools] == ["get_weather"] + assert json.loads(tools[0].arguments) == {"city": "SF"} + + def test_single_tool_string_marker(self, parser, mock_request): + # Granite 3.1 uses the plain-text ```` marker. + text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]' + _, content, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["get_time"] + assert json.loads(tools[0].arguments) == {} + + def test_parallel_calls_one_array(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, ' + f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]' + ) + _, _, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["a", "b"] + assert json.loads(tools[0].arguments) == {"x": 1} + assert json.loads(tools[1].arguments) == {"y": [1, 2]} + + def test_surrounding_text_is_content(self, parser, mock_request): + text = ( + "Let me check.\n" + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + _, content, tools = parser.parse(text, mock_request) + assert content == "Let me check." + assert [t.name for t in tools] == ["get_weather"] + + def test_arguments_before_name(self, parser, mock_request): + text = ( + f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]' + ) + _, _, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["get_weather"] + assert json.loads(tools[0].arguments) == {"city": "SF"} + + def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request): + # A non-array body after the marker (malformed) yields no tool call. + text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}' + _, _, tools = parser.parse(text, mock_request) + assert not tools + + +class TestStreaming: + @pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11]) + def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size): + text = ( + f'{TOOL_TOKEN} [{{"name": "get_weather", ' + f'"arguments": {{"city": "Tokyo"}}}}, ' + f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]' + ) + results = _stream(parser, mock_request, text, chunk_size) + assert _collect_names(results) == ["get_weather", "get_time"] + args = _collect_args(results) + assert json.loads(args[0]) == {"city": "Tokyo"} + assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"} + + @pytest.mark.parametrize("chunk_size", [1, 3, 7]) + def test_surrounding_text_streaming(self, parser, mock_request, chunk_size): + text = ( + "Let me check.\n" + f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]' + ) + results = _stream(parser, mock_request, text, chunk_size) + assert _collect_names(results) == ["get_weather"] + assert "Let me check." in _collect_content(results) + + def test_incomplete_array_at_eos(self, parser, mock_request): + # Truncated mid-element must not raise and must not leak the marker. + text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1' + results = _stream(parser, mock_request, text, 3) + assert TOOL_TOKEN not in _collect_content(results) + + +class TestReplay: + @pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id) + def test_replay_scenarios(self, scenario): + # ``_build_granite`` replays the sample through GraniteParser at + # chunk_size=1 and asserts reasoning/content/tool_calls (validate=True). + _build_granite(scenario) diff --git a/tests/parser/engine/test_inkling.py b/tests/parser/engine/test_inkling.py index 9026d548df..1ce653f8b1 100644 --- a/tests/parser/engine/test_inkling.py +++ b/tests/parser/engine/test_inkling.py @@ -23,8 +23,13 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionToolsParam, FunctionDefinition, ) +from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.engine.events import EventType from vllm.parser.engine.parser_engine_config import ParserState +from vllm.parser.engine.registered_adapters import ( + InklingParserReasoningAdapter, + InklingParserToolAdapter, +) from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine from vllm.parser.inkling import InklingParser, _inkling_arg_converter, inkling_config from vllm.parser.parser_manager import ParserManager @@ -166,6 +171,11 @@ def _delegating(mock_tokenizer, tools=None): return parser_cls(mock_tokenizer, tools or []) +class _TwoPassInklingParser(DelegatingParser): + reasoning_parser_cls = InklingParserReasoningAdapter + tool_parser_cls = InklingParserToolAdapter + + def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids): """Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size`` tokens per delta; return ``(content, reasoning, ordered tool names, @@ -759,6 +769,30 @@ class TestDelegatingTwoPass: assert TOOL_JSON not in content assert END_MESSAGE not in content + def test_function_name_not_in_content_after_reasoning( + self, mock_tokenizer, mock_request + ): + name = "search_documents_by_keyword" + tools = [_function_tool(name)] + mock_request.tools = tools + text = ( + f"{THINK_START} context.{END_MESSAGE}{MSG_MODEL}" + f"{name}{_tool_block(name, '{}')}" + ) + + content, reasoning, names, args = _stream_delegating( + _TwoPassInklingParser(mock_tokenizer, tools), + mock_request, + text, + chunk_size=12, + prompt_token_ids=self.GEN_PROMPT, + ) + + assert reasoning == " context." + assert content == "" + assert names == [name] + assert json.loads(args[0]) == {} + @pytest.mark.parametrize("opener", [TEXT_START, TOOL_TEXT, TOOL_ERROR]) @pytest.mark.parametrize("chunk_size", [1, 3, 64]) def test_visible_text_then_tool_streaming( diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index fee74f8d2c..cc8fef483f 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -78,6 +78,11 @@ def _discover_parsers() -> list[_ParserInfo]: # Inkling opts out of token-id terminal matching and has typed # structural blocks; its replay coverage lives in test_inkling.py. continue + if cfg.name == "granite": + # Granite has a JSON-array tool body with no TOOL_END terminal, so + # it does not fit this token-terminal harness; its replay coverage + # lives in test_granite.py. + continue tool_end = cfg.token_id_terminals.get("TOOL_END") if not tool_end: raise RuntimeError( @@ -322,7 +327,18 @@ _TOOL_CALL_SAMPLES = [ def _tool_suppression_expectations( sample, think_end: str, tool_start: str, *, include_tool_block: bool ) -> tuple[str, str]: - """Expected (reasoning, content) when tool calls are not extracted. + reasoning, content = _raw_tool_suppression_expectations( + sample, think_end, tool_start, include_tool_block=include_tool_block + ) + if sample.content_lstrip: + content = content.lstrip(sample.content_lstrip) + return reasoning, content + + +def _raw_tool_suppression_expectations( + sample, think_end: str, tool_start: str, *, include_tool_block: bool +) -> tuple[str, str]: + """Expected (reasoning, content) from the raw token text. With ``include_tool_block=True`` (skip_tool_parsing / reasoning adapter first pass), tool terminal text is preserved as content so diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 36e347fc57..5b405e6dae 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -15,7 +15,7 @@ from __future__ import annotations import functools import json -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any from tests.parser.engine.replay_harness import ( @@ -34,6 +34,8 @@ from vllm.parser.engine.registered_adapters import ( DeepSeekV41Parser, Gemma4Parser, Glm47MoeParser, + GraniteParser, + GraniteThinkingParser, InklingParser, KimiK2Parser, MinimaxM2Parser, @@ -593,6 +595,30 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample: ) +def _build_granite_thinking(scenario: Scenario, validate: bool = True) -> Sample: + """Granite 4.2: the Nemotron V3 grammar plus leading-newline stripping on + content after ````.""" + sample = _build_qwen3( + scenario, + name="granite_thinking_parser", + parser_cls=GraniteThinkingParser, + strip_trailing_ws=True, + validate=False, + ) + sample = replace( + sample, + expected_content=( + sample.expected_content.lstrip("\n") + if sample.expected_content + else sample.expected_content + ), + content_lstrip="\n", + ) + if validate: + _validate_sample(sample, GraniteThinkingParser) + return sample + + # ── Seed-OSS (Qwen3 XML grammar with Seed wrapper tokens) ──────────── _SEED_OSS_VOCAB: dict[str, int] = { @@ -1042,6 +1068,89 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample: return sample +# ── Granite (JSON-array tool bodies, no reasoning) ─────────────────────── + +_GRANITE_VOCAB: dict[str, int] = { + "<|tool_call|>": 49154, +} + + +def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.content: + # Granite has no reasoning; prose is plain content preceding the marker. + segs.append((scenario.content, False)) + if scenario.tool_calls: + segs.append(("<|tool_call|>", True)) + payload = json.dumps( + [ + {"name": tc.name, "arguments": tc.arguments} + for tc in scenario.tool_calls + ], + ensure_ascii=False, + separators=(", ", ": "), + ) + segs.append((" " + payload, False)) + return segs + + +def _granite_expected_content(scenario: Scenario) -> str | None: + if scenario.tool_calls: + if not scenario.content: + return None + return scenario.content.strip() or None + return scenario.content + + +def _build_granite(scenario: Scenario, validate: bool = True) -> Sample: + sample = _make_sample( + sample_id=f"granite-{scenario.id}", + description=scenario.description, + vocab=_GRANITE_VOCAB, + segments=_granite_segments(scenario), + expected_reasoning=None, + expected_content=_granite_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, GraniteParser) + return sample + + +# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not +# apply; these exercise the JSON-array tool body (single, parallel, surrounding +# text) instead. +_GRANITE_SCENARIOS: list[Scenario] = [ + Scenario( + id="single-tool", + description="Single tool call", + tool_calls=[_READ_TOOL], + ), + Scenario( + id="parallel-tools", + description="Parallel tool calls in one JSON array", + tool_calls=[_BASH_TOOL, _WEATHER_TOOL], + ), + Scenario( + id="complex-json-args", + description="Tool call with nested objects, arrays, numbers, booleans", + tool_calls=[_COMPLEX_TOOL], + ), + Scenario( + id="content-only", + description="Plain content response without tool calls", + content="Hello! How can I help you today?", + ), + Scenario( + id="surrounding-text", + description="Prose content preceding the tool call", + content="Let me check the weather.", + tool_calls=[_WEATHER_TOOL], + ), +] + + # ── Registry and public API ────────────────────────────────────────── _BUILDERS: dict[str, Any] = { @@ -1049,8 +1158,10 @@ _BUILDERS: dict[str, Any] = { "deepseek_v4": _build_deepseek_v4, "deepseek_v41": functools.partial(_build_deepseek_v4, v41=True), "gemma4": _build_gemma4, + "granite": _build_granite, "minimax_m2": _build_minimax_m2, "nemotron_v3": _build_nemotron_v3, + "granite_thinking_parser": _build_granite_thinking, "seed_oss": _build_seed_oss, "glm47_moe": _build_glm47_moe, "kimi_k2": _build_kimi_k2, @@ -1065,23 +1176,3 @@ def build_samples(model: str) -> tuple[Sample, ...]: builder = _BUILDERS[model] scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None) - - -def build_sample(model: str, scenario: Scenario) -> Sample | None: - """Build a single sample for one model + scenario.""" - return _BUILDERS[model](scenario) - - -def build_scaling_sample( - model: str, token_count: int, validate: bool = False -) -> Sample: - """Build a sample with approximately *token_count* tokens.""" - sentence = "The quick brown fox jumps over the lazy dog. " - text = sentence * (token_count // 10 + 1) - scenario = Scenario( - id=f"scaling-{token_count}", - description=f"Scaling test with ~{token_count} tokens", - reasoning=text, - tool_calls=[_READ_TOOL], - ) - return _BUILDERS[model](scenario, validate=validate) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 86f157c768..3d6fff04bf 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -873,6 +873,67 @@ class TestProcessChunk: ] +class TestCountReasoningTokens: + def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer): + """Chat usage must report the same count as the Responses API path.""" + token_ids = get_model_output_tokens( + [ + assistant("Let me check the weather.", "analysis"), + tool_call("functions.get_weather", '{"location": "SF"}'), + ] + ) + count = harmony_parser.count_reasoning_tokens(token_ids) + + fresh = HarmonyParser(gpt_oss_tokenizer) + assert count == fresh.process_chunk(token_ids).reasoning_token_count + assert count > 0 + + def test_excludes_harmony_control_tokens(self, harmony_parser): + text = "Let me think about this." + token_ids = get_model_output_tokens( + [assistant(text, "analysis"), assistant("Hi", "final")] + ) + expected = len(get_encoding().encode(text)) + assert harmony_parser.count_reasoning_tokens(token_ids) == expected + + def test_final_only_has_no_reasoning(self, harmony_parser): + token_ids = get_model_output_tokens([assistant("Hello", "final")]) + assert harmony_parser.count_reasoning_tokens(token_ids) == 0 + + def test_streaming_reuses_running_total( + self, harmony_parser, gpt_oss_tokenizer, chat_request, monkeypatch + ): + """Called per streamed chunk with all tokens so far; must not replay.""" + token_ids = get_model_output_tokens( + [assistant("Think", "analysis"), assistant("Answer", "final")] + ) + expected = HarmonyParser(gpt_oss_tokenizer).count_reasoning_tokens(token_ids) + harmony_parser.parse_delta("", token_ids[:3], chat_request, finished=False) + + def fail(): + raise AssertionError("count_reasoning_tokens replayed the output") + + monkeypatch.setattr( + "vllm.parser.harmony.get_streamable_parser_for_assistant", fail + ) + for end in range(3, len(token_ids)): + harmony_parser.process_chunk(token_ids[end : end + 1]) + harmony_parser.count_reasoning_tokens(token_ids[: end + 1]) + assert harmony_parser.count_reasoning_tokens(token_ids) == expected + + def test_does_not_disturb_stream_state(self, harmony_parser, chat_request): + """Streaming counts after each chunk, so it must not consume parser state.""" + token_ids = get_model_output_tokens( + [assistant("Think", "analysis"), assistant("Answer", "final")] + ) + harmony_parser.count_reasoning_tokens(token_ids) + + reasoning, content, _ = harmony_parser.parse( + "", chat_request, model_output_token_ids=token_ids + ) + assert (reasoning, content) == ("Think", "Answer") + + class TestAdjustRequest: REQUEST_TEXT = "Hello" TOOL_TYPE = "function" diff --git a/tests/platforms/test_cpu.py b/tests/platforms/test_cpu.py index fd270315eb..9c7f0b188b 100644 --- a/tests/platforms/test_cpu.py +++ b/tests/platforms/test_cpu.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os from types import SimpleNamespace import pytest import torch -from vllm.config import CacheConfig, VllmConfig +from vllm.config import CacheConfig, KVTransferConfig, VllmConfig from vllm.platforms.cpu import CpuPlatform @@ -44,6 +45,7 @@ def _cpu_config( architecture=architecture or model_type, has_inner_state=True, use_mla=False, + hf_config=SimpleNamespace(model_type=model_type), hf_text_config=SimpleNamespace( model_type=model_type, layer_types=layer_types, @@ -150,3 +152,74 @@ def test_cpu_accelerated_gdn_dtype_policy( CpuPlatform.check_and_update_config(config) assert cache_config.mamba_ssm_cache_dtype == expected_dtype + + +@pytest.mark.parametrize( + ( + "connector", + "extra_config", + "explicit_layout", + "avx512_bf16_supported", + "expected_layout", + ), + [ + pytest.param("NixlConnector", None, None, True, "DS", id="nixl"), + pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"), + pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"), + pytest.param( + "MultiConnector", + { + "connectors": [ + { + "kv_connector": "ExampleConnector", + "kv_connector_extra_config": {}, + }, + { + "kv_connector": "NixlConnector", + "kv_connector_extra_config": {}, + }, + ] + }, + None, + True, + "DS", + id="multi-nixl", + ), + pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"), + pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"), + pytest.param( + "OffloadingConnector", None, None, False, None, id="non-nixl-no-avx" + ), + ], +) +def test_cpu_conv_state_layout_selection( + monkeypatch: pytest.MonkeyPatch, + connector: str, + extra_config: dict | None, + explicit_layout: str | None, + avx512_bf16_supported: bool, + expected_layout: str | None, +) -> None: + layout_env = "VLLM_SSM_CONV_STATE_LAYOUT" + monkeypatch.setattr( + "torch.cpu._is_avx512_bf16_supported", + lambda: avx512_bf16_supported, + ) + + kv_transfer_config = KVTransferConfig( + kv_connector=connector, + kv_connector_extra_config=extra_config or {}, + kv_role="kv_both", + ) + config = _cpu_config( + CacheConfig(mamba_ssm_cache_dtype="float32"), + model_type="qwen3_5", + resolved_dtype="float32", + ) + config.kv_transfer_config = kv_transfer_config + monkeypatch.delenv(layout_env, raising=False) + if explicit_layout is not None: + monkeypatch.setenv(layout_env, explicit_layout) + + CpuPlatform.check_and_update_config(config) + assert os.environ.get(layout_env) == expected_layout diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index 2bc463b593..f2eeb9e9d9 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -146,6 +146,21 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = { }, } +# These checkpoints fail during XPU graph capture, so run them eagerly there. +XPU_EAGER_ONLY_MODELS = { + "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", + "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", + "Intel/Qwen3-8B-w2g64-for-ut", + "INCModel/Qwen3-30B-A3B-12L-W4A16-test", +} + + +def _runner_kwargs(model: str) -> dict[str, Any]: + kwargs = dict(MODEL_RUNNER_KWARGS.get(model, {})) + if current_platform.is_xpu() and model in XPU_EAGER_ONLY_MODELS: + kwargs["enforce_eager"] = True + return kwargs + @pytest.mark.skipif( not ( @@ -157,7 +172,7 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = { ) @pytest.mark.parametrize("model", MODELS + QWEN3_AUTOROUND_MODELS) def test_auto_round_model(vllm_runner, model): - with vllm_runner(model, **MODEL_RUNNER_KWARGS.get(model, {})) as llm: + with vllm_runner(model, **_runner_kwargs(model)) as llm: output = llm.generate_greedy(["The capital of France is"], max_tokens=8) assert output @@ -1234,7 +1249,7 @@ def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None: kernel = DummyKernel() monkeypatch.setattr( "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel", - lambda: kernel, + lambda weight_shape: kernel, ) monkeypatch.setattr( "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter", @@ -1273,7 +1288,7 @@ def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None: def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None: monkeypatch.setattr( "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel", - lambda: object(), + lambda weight_shape: object(), ) scheme = INCMxfp8LinearScheme() @@ -1638,19 +1653,6 @@ def test_wna16_linear_gptq_unsupported_config_raises() -> None: INCWNA16LinearScheme(make_layer_config(sym=False)) -def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None: - monkeypatch.setattr(current_platform, "is_xpu", lambda: True) - monkeypatch.setattr(current_platform, "is_cpu", lambda: False) - - with pytest.raises(NotImplementedError, match="unsupported config"): - INCWna16Scheme().get_linear_method( - make_config(weight_bits=2, sym=False), - object(), - "layer", - make_layer_config(bits=2, sym=False), - ) - - def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None: config = make_config(extra_config={"layer": {"bits": 16}}) layer = object.__new__(LinearBase) diff --git a/tests/quantization/test_configs.py b/tests/quantization/test_configs.py index 674d57b84a..cca7d4b6b0 100644 --- a/tests/quantization/test_configs.py +++ b/tests/quantization/test_configs.py @@ -5,20 +5,11 @@ Run `pytest tests/quantization/test_configs.py --forked`. """ -from dataclasses import dataclass - import pytest from vllm.config import ModelConfig from vllm.platforms import current_platform - -@dataclass -class ModelPair: - model_marlin: str - model_gptq: str - - # Model Id // Quantization Arg // Expected Type MODEL_ARG_EXPTYPES = [ # AUTOGPTQ diff --git a/tests/quantization/test_cpu_offload.py b/tests/quantization/test_cpu_offload.py index 151b5d97dd..f5237efa5a 100644 --- a/tests/quantization/test_cpu_offload.py +++ b/tests/quantization/test_cpu_offload.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Expanded quantized model tests for CPU offloading -# Base tests: tests/basic_correctness/test_cpu_offload.py +# Base tests: tests/basic_correctness/cpu_offload/test_cpu_offload.py import pytest diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 65e119b083..e391d516da 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -5,11 +5,9 @@ Run `pytest tests/quantization/test_fp8.py --forked`. """ -import logging from types import SimpleNamespace import pytest -import regex as re import torch from tests.quantization.utils import ( @@ -22,9 +20,6 @@ from vllm.config.cache import CacheConfig from vllm.config.kernel import KernelConfig from vllm.config.model import ModelConfig from vllm.forward_context import set_forward_context -from vllm.model_executor.kernels.linear.scaled_mm import ( - MarlinFP8ScaledMMLinearKernel, -) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention.attention import ( set_default_quant_scales, @@ -32,15 +27,11 @@ from vllm.model_executor.layers.attention.attention import ( from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, - Fp8KVCacheMethod, Fp8LinearMethod, Fp8MoEMethod, ) from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod -from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineLinearMethod, -) from vllm.model_executor.layers.quantization.utils import flashinfer_utils from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( prepare_fp8_moe_layer_for_fi, @@ -237,6 +228,85 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales( ] +def test_deepseek_v41_vl_exposes_quant_mappings_on_class(): + """``configure_quant_config`` reads both mappings off the class, before + ``__init__`` builds the instance mapper. Without them a Quark config's + per-layer keys never match this wrapper's ``language_model.``-rooted + prefixes and every attention shard falls back to the global spec.""" + from vllm.models.deepseek_v41.amd.vl_model import DeepseekV41ForCausalLM + + assert "fused_wqa_wkv" in DeepseekV41ForCausalLM.packed_modules_mapping + mapper = DeepseekV41ForCausalLM.hf_to_vllm_mapper.get_rename_mapper() + assert mapper.apply_list(["layers.0.attn.wq_a"]) == [ + "language_model.model.layers.0.attn.wq_a" + ] + + +def test_deepseek_v41_engram_scale_accepts_quark_name(): + """Quark exports name the engram scale ``embed.weight_scale``; the + ``\\.scale$`` rules only match a literal ``.scale`` suffix, so without an + explicit rule the tensor is never routed and loading fails.""" + from vllm.models.deepseek_v41.amd import model as model_module + + mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale") + weight = torch.empty(0) + mapped = [ + name + for name, _ in mapper.apply( + [ + ("layers.1.engram.embed.weight_scale", weight), + ("layers.1.engram.embed.scale", weight), + ] + ) + ] + assert mapped == [ + "model.layers.1.engram.embed_tokens.weight_scale_inv", + "model.layers.1.engram.embed_tokens.weight_scale_inv", + ] + + +def test_deepseek_v41_declines_quark_configs(): + """``from_config`` rewrites a Quark config into a single global FP8 scheme, + which would discard the per-layer specs of a mixed-precision export (MXFP4 + experts + 2-D block MXFP8 attention in DeepSeek-V4.1-Flash). QuarkConfig + knows how to dispatch each scheme, so it must handle every Quark export.""" + from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config + + hf_config = SimpleNamespace(model_type="deepseek_v41") + mxfp4_global = { + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32} + }, + "quant_method": "quark", + } + + assert ( + DeepseekV4FP8Config.override_quantization_method( + mxfp4_global, None, hf_config=hf_config + ) + is None + ) + + mixed = { + **mxfp4_global, + "layer_quant_config": { + "layers.0.attn.wkv": { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + } + } + }, + } + assert ( + DeepseekV4FP8Config.override_quantization_method( + mixed, None, hf_config=hf_config + ) + is None + ) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA") @pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu]) @pytest.mark.parametrize( @@ -531,187 +601,6 @@ def test_model_load_and_run( model(input_ids, positions, None) -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) -@pytest.mark.parametrize( - "force_marlin", [True, False] if current_platform.is_cuda() else [False] -) -@pytest.mark.parametrize( - "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] -) -def test_online_quantization( - vllm_runner, - kv_cache_dtype: str, - force_marlin: bool, - use_rocm_aiter: bool, - monkeypatch, -) -> None: - if use_rocm_aiter: - monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") - - # `LLM.apply_model` requires pickling a function. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - kwargs = {} - if force_marlin: - kwargs["linear_backend"] = "marlin" - kwargs["moe_backend"] = "marlin" - - model_dtype = "auto" - if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90): - # FA3 requires BF16 output when the query input is FP8. - model_dtype = "bfloat16" - - with vllm_runner( - "facebook/opt-125m", - quantization="fp8", - dtype=model_dtype, - enforce_eager=True, - kv_cache_dtype=kv_cache_dtype, - **kwargs, - ) as llm: - - def check_model(model): - fc1 = model.model.decoder.layers[0].fc1 - assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod) - if kv_cache_dtype == "fp8": - attn = model.model.decoder.layers[0].self_attn.attn - assert isinstance(attn.quant_method, Fp8KVCacheMethod) - assert attn._k_scale == 1.0 - assert attn._v_scale == 1.0 - - if current_platform.is_cuda() or current_platform.is_xpu(): - if current_platform.supports_fp8() and not force_marlin: - # For GPUs with hardware support, we keep weights in fp8 - assert fc1.weight.dtype == torch.float8_e4m3fn - assert not isinstance( - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel - ) - else: - # For GPUs without hardware support, we pack the fp8 weights - # for weight-only quantization using Marlin kernels - assert fc1.weight.dtype == torch.int32 - assert isinstance( - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel - ) - elif current_platform.is_rocm(): - if current_platform.supports_fp8() and not force_marlin: - # For GPUs with hardware support, we keep weights in fp8 - assert fc1.weight.dtype == current_platform.fp8_dtype() - else: # unsupported ROCm platform - pytest.skip( - "Skip `test_load_fp16_model`. " - "It only runs on ROCm platform with FP8 compute." - " e.g. MI300X and above." - ) - else: # unsupported platform - pytest.skip( - "Skip `test_load_fp16_model`. " - "It only runs on CUDA and ROCm platform." - ) - - llm.apply_model(check_model) - - outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4) - print(outputs[0][1]) - - -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -def test_online_quant_peak_mem( - vllm_runner, - caplog_mp_spawn, - monkeypatch, -) -> None: - # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because: - # 1. it covers both Linear and MoE paths - # 2. it is already used by other tests in CI, so adding it here - # does not increase disk space for CI runners - # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base` - # which I think is the smallest MoE model in vLLM (2.5 GiB bf16, - # 1.3 GiB fp8), but could not as adding one more model makes CI - # run out of disk space. - model_name = "allenai/OLMoE-1B-7B-0125-Instruct" - - # Force spawn to ensure caplog_mp_spawn works consistently - # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores) - monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") - - with ( - caplog_mp_spawn(logging.DEBUG) as log_holder, - vllm_runner( - model_name, - quantization="fp8", - enforce_eager=True, - ) as llm, - ): - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4) - print(outputs[0][1]) - - log_text = log_holder.text - - # Parse memory usage from captured logs - model_memory_gib = None - peak_memory_gib = None - for line in log_text.splitlines(): - if model_memory_gib is None: - match = re.search(r"Model loading took ([\d.]+) GiB memory", line) - if match: - model_memory_gib = float(match.group(1)) - if peak_memory_gib is None: - match = re.search( - r"Peak GPU memory after loading weights: ([\d.]+) GiB", line - ) - if match: - peak_memory_gib = float(match.group(1)) - - assert model_memory_gib is not None, "Could not find model loading memory log" - assert peak_memory_gib is not None, "Could not find peak memory log" - print(f"GPU memory used after loading weights: {model_memory_gib} GiB") - print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB") - - # model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant - # uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB) - expected_model_memory_gib = 6.7 - - # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06 - # GiB, which is 1.36x above model_memory_gib. A slightly higher number is - # expected as when we load and quantize weights in a streaming fashion we - # need to have individual weights in bf16 + fp8 alive at the same time. - expected_peak_memory_gib = expected_model_memory_gib * 1.4 - - assert model_memory_gib < expected_model_memory_gib, ( - f"{model_memory_gib=} higher than {expected_model_memory_gib}" - ) - assert peak_memory_gib < expected_peak_memory_gib, ( - f"{peak_memory_gib=} higher than {expected_peak_memory_gib}" - ) - - -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -def test_online_quant_load_format_dummy( - vllm_runner, - monkeypatch, - caplog, -) -> None: - with vllm_runner( - "ibm-granite/granite-3.0-1b-a400m-base", - quantization="fp8", - enforce_eager=True, - load_format="dummy", - ) as llm: - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4) - print(outputs[0][1]) - - @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", @@ -799,7 +688,6 @@ def test_scaled_fp8_quant(dtype) -> None: ) @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod]) # FP8 weight reloading does not support online quantization -@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False @pytest.mark.parametrize("weight_block_size", [None, [128, 128]]) # any postprocessing that is applied to the weights such as padding and repacking # (excluding device sharding) must also be applied to the reloaded weights @@ -809,7 +697,6 @@ def test_scaled_fp8_quant(dtype) -> None: def test_fp8_reloading( default_vllm_config, method_cls, - is_checkpoint_fp8_serialized, weight_block_size, use_marlin, dist_init, @@ -820,9 +707,6 @@ def test_fp8_reloading( # we set fp8_backend to None, which sidestepped the issue. monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0") - if is_checkpoint_fp8_serialized is False: - pytest.skip("FP8 weight reloading does not support online quantization") - if method_cls is Fp8MoEMethod and weight_block_size is None: pytest.skip( "FP8 Tensor weight reloading does not support fusing w13_weight_scale. " @@ -835,7 +719,6 @@ def test_fp8_reloading( layer_size = 128 if weight_block_size is not None else 1 with torch.device(f"{DEVICE_TYPE}:0"): config = Fp8Config( - is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, weight_block_size=weight_block_size, ) diff --git a/tests/quantization/test_gptq_v2.py b/tests/quantization/test_gptq_v2.py deleted file mode 100644 index a10088f403..0000000000 --- a/tests/quantization/test_gptq_v2.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests whether vllm correctly load and run gptq_v2 format checkpoints. - -Run `pytest tests/quantization/test_gptq_v2.py --forked`. - -Note: 2/3-bit GPTQ models are no longer supported after the consolidation -to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported. -""" - -import pytest -import torch -from transformers import AutoTokenizer - -from vllm import SamplingParams -from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod - -# A dummy small model quantized by GPTQModel, stored in GPTQ v2 format -# Note: This is a 2-bit model which is no longer supported with Marlin kernels -MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"] - -# Generate multiple sequences for testing, because an 1.7B 2-bit model -# cannot always generate normal texts. -N_SEQ = 5 - - -@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation") -@pytest.mark.parametrize("model_id", MODELS) -def test_model_load(vllm_runner, model_id, monkeypatch): - # `LLM.apply_model` requires pickling a function. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm: - - def check_model(model_id): - for name, submodule in model_id.named_modules(): - # Could check more modules if necessary - if name == "model_id.layers.0.self_attn.qkv_proj": - assert isinstance(submodule.quant_method, AutoGPTQLinearMethod) - # Just break since currently we only check 1 module - break - - # Check if gptq_v2 format is correctly loaded - llm.apply_model(check_model) - - -@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation") -@pytest.mark.parametrize("model_id", MODELS) -def test_model_inference(vllm_runner, model_id): - # Prepare prompt to test the model's generation result. - prompt = "What is the meaning of life?" - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": prompt}, - ] - tokenizer = AutoTokenizer.from_pretrained(model_id) - text = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - enable_thinking=False, # If thinking model, set it to false - ) - sampling_params = SamplingParams( - n=N_SEQ, - max_tokens=128, - temperature=0.7, - top_p=0.8, - top_k=20, - min_p=0, - presence_penalty=2, - ) - - with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm: - # Generate a response to verify inference correctness - output = llm.generate(text, sampling_params) - - # Make sure the output exists - assert output - assert output[0][1] - assert len(output[0][1]) == N_SEQ - - def has_normal_char_distribution(texts, min_len): - for text in texts: - # Response too short - if len(text) < min_len: - return False - - # Basic ratio checks - letters = sum(c.isalpha() for c in text) - spaces = sum(c.isspace() for c in text) - total = len(text) - - letter_ratio = letters / total - space_ratio = spaces / total - - # At least 1 normal text should exist within output sequences - # Normal text should be mostly letters with reasonable spacing - # Some magic numbers, could be adjusted - if 0.5 <= letter_ratio <= 0.9 and 0.01 <= space_ratio <= 0.3: - return True - # No sequence contains normal text, output might be broken - return False - - # Apply some simple checks for giberish output - # Print the output sequences if failed - assert has_normal_char_distribution(output[0][1], 5), output[0][1] diff --git a/tests/quantization/test_lm_head.py b/tests/quantization/test_lm_head.py index 16f8fd8d38..b89be3d055 100644 --- a/tests/quantization/test_lm_head.py +++ b/tests/quantization/test_lm_head.py @@ -5,14 +5,26 @@ Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`. """ +from types import SimpleNamespace + import pytest import torch +from compressed_tensors.quantization import preset_name_to_scheme from tests.quantization.utils import load_model_without_vllm_runner +from vllm.model_executor.layers.linear import ColumnParallelLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 + CompressedTensorsConfig, +) +from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, UnquantizedEmbeddingMethod, ) +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_humming PROMPT = "On the surface of Mars, we found" @@ -44,3 +56,65 @@ def test_lm_head( assert isinstance(lm_head_layer.quant_method, AutoGPTQLinearMethod) else: assert isinstance(lm_head_layer.quant_method, UnquantizedEmbeddingMethod) + + +@pytest.mark.skipif( + not current_platform.is_cuda() or not has_humming(), reason="requires Humming/CUDA" +) +@pytest.mark.parametrize( + "preset,quant_format", + [ + ("FP8_DYNAMIC", "float-quantized"), + ("NVFP4A16", "nvfp4-pack-quantized"), + ("W4A16", "pack-quantized"), + ("W4A16_NVFP4", None), + ], +) +@pytest.mark.parametrize("bias", [False, True]) +@torch.inference_mode() +def test_quantized_lm_head_matches_linear( + dist_init, default_vllm_config, preset, quant_format, bias +): + """An explicitly quantized head must produce the same logits as a linear layer.""" + default_vllm_config.model_config = SimpleNamespace( + dtype=torch.bfloat16, head_dtype=None + ) + default_vllm_config.kernel_config.linear_backend = "humming" + if quant_format is None: + quant_config = ModelOptNvFp4Config( + quant_method=preset, + is_checkpoint_nvfp4_serialized=True, + kv_cache_quant_algo=None, + exclude_modules=[], + ) + else: + scheme = preset_name_to_scheme(preset, targets=["Linear", "lm_head"]) + quant_config = CompressedTensorsConfig.from_config( + {"config_groups": {"group_0": scheme.model_dump()}, "format": quant_format} + ) + with torch.device("cuda"): + kwargs = dict( + bias=bias, + params_dtype=torch.bfloat16, + quant_config=quant_config, + disable_tp=True, + ) + head = ParallelLMHead(500, 256, prefix="lm_head", **kwargs) + linear = ColumnParallelLinear(256, 512, prefix="proj", **kwargs) + for name, param in linear.named_parameters(): + if "scale" in name: + param.fill_(1.0) + elif name == "weight_shape": + param.copy_(torch.tensor([512, 256])) + elif param.is_floating_point(): + param.copy_(torch.randn(param.shape, dtype=torch.float32)) + else: + param.random_(0, 127) + head.load_state_dict(linear.state_dict()) + for layer in (head, linear): + layer.quant_method.process_weights_after_loading(layer) + x = torch.randn(8, 256, dtype=torch.bfloat16) + expected, _ = linear(x) + actual = LogitsProcessor(500)(head, x, head.bias) + assert torch.isfinite(actual).all() + torch.testing.assert_close(actual, expected[:, :500], rtol=0, atol=0) diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index ae8548c370..d7b52723cb 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -718,49 +718,6 @@ def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls): assert isinstance(kernel, kernel_cls) -@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") -def test_modelopt_linear_exposes_humming_layer_attrs(dist_init, monkeypatch): - """``prepare_humming_linear_layer_config`` reads ``output_partition_sizes`` - and ``has_bias`` straight off the layer, so ``--linear-backend=humming`` - needs create_weights to leave both there. Nothing else sets - ``output_partition_sizes``; ``LinearBase`` sets ``has_bias`` but - ``ParallelLMHead`` does not. - """ - from vllm.config.quantization import QuantSpec - from vllm.model_executor.layers.quantization import modelopt as mo - - monkeypatch.setattr( - mo, "select_linear_kernel", lambda spec, layer, rt, **kwargs: Mock() - ) - monkeypatch.setattr(mo, "expose_input_quant_key", lambda layer, kernel: None) - - def build(layer): - method = ModelOptLinearMethod.__new__(ModelOptLinearMethod) - method.spec = QuantSpec(weight=kNvfp4Static, activation=None) - method.ctx = mo.CkptCtx(group_size=16) - method.fmt = mo.FormatScheme() - method.wkey = mo.SCHEME_FOR[kNvfp4Static] - method.akey = None - method.input_dtype = method.out_dtype = torch.bfloat16 - method.marlin_input_dtype = None - method.create_weights( - layer, 64, [32, 32], 64, 64, torch.bfloat16, weight_loader=Mock() - ) - - # ParallelLMHead-style: a bias slot but no has_bias attribute. - lm_head = torch.nn.Module() - lm_head.register_parameter("bias", None) - build(lm_head) - assert lm_head.output_partition_sizes == [32, 32] - assert lm_head.has_bias is False - - # LinearBase already decided has_bias; we must not overwrite it. - linear = torch.nn.Module() - linear.has_bias = True - build(linear) - assert linear.has_bias is True - - @pytest.mark.parametrize( "quant_method, expected_use_a16, act_key_is_none", [ diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 846b981478..645ad67062 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -20,6 +20,8 @@ from tests.quantization.utils import ( from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm._custom_ops import scaled_fp4_quant +from vllm.config.cache import CacheConfig +from vllm.config.kernel import KernelConfig from vllm.config.load import LoadConfig from vllm.config.model import ModelConfig from vllm.config.quantization import ( @@ -34,8 +36,12 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import ( from vllm.model_executor.kernels.linear.mxfp8.marlin import ( MarlinMxfp8LinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm import ( + MarlinFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle from vllm.model_executor.layers.linear import ( ColumnParallelLinear, LinearBase, @@ -124,6 +130,81 @@ PARTIALLY_PREQUANTIZED_MODEL_NAME = ( ) +@pytest.mark.parametrize( + "overrides", + [ + None, + {"ignore": ["model.layers.0.self_attn.o_proj"]}, + { + "linear": "fp8_per_block", + "ignore": ["model.layers.0.self_attn.o_proj"], + }, + ], + ids=["defaults", "ignore", "linear-and-ignore"], +) +def test_legacy_fp8_online_quantization_uses_per_tensor_shorthand( + tmp_path, caplog, disable_log_dedup, overrides +) -> None: + """The legacy alias preserves overrides and fills in per-tensor defaults.""" + _write_minimal_llama_config(tmp_path) + model_config = ModelConfig( + model=str(tmp_path), quantization="fp8", quantization_config=overrides + ) + + result = weight_utils.get_quant_config(model_config, LoadConfig()) + + assert isinstance(result, OnlineQuantizationConfig) + assert result.args == resolve_quantization_config("fp8_per_tensor", overrides) + assert "--quantization fp8 is deprecated for online quantization" in caplog.text + assert "--quantization fp8_per_tensor instead" in caplog.text + + +@pytest.mark.parametrize( + "shorthand,method_cls", + [ + ("fp8_per_tensor", Fp8PerTensorOnlineLinearMethod), + ("fp8_per_block", Fp8PerBlockOnlineLinearMethod), + ("fp8_per_channel", Fp8PtpcOnlineLinearMethod), + ], +) +def test_qianfan_online_fp8_keeps_vision_layers_unquantized( + shorthand, method_cls +) -> None: + """Model exclusions preserve the language backbone and shared defaults.""" + from vllm.model_executor.models.qianfan_ocr import ( + QianfanOCRForConditionalGeneration, + ) + + args = resolve_quantization_config(shorthand, None) + assert args is not None + config = OnlineQuantizationConfig(args) + other_config = OnlineQuantizationConfig(args) + model = QianfanOCRForConditionalGeneration.__new__( + QianfanOCRForConditionalGeneration + ) + model._patch_quant_config( + SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)), + config, + ) + + layer = Mock(spec=LinearBase) + for prefix in ( + "vision_model.encoder.layers.0.attn.qkv", + "vision_model.encoder.layers.1.mlp.fc2", + "language_model.lm_head", + "mlp1.1", + "mlp1.3", + ): + assert config.resolve_quant_method_cls(layer, prefix) is None + + resolved = config.resolve_quant_method_cls( + layer, "language_model.model.layers.0.self_attn.qkv_proj" + ) + assert resolved is not None and resolved[-1] is method_cls + assert other_config.ignored_layers == [] + assert args.ignore == [] + + def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( monkeypatch, ) -> None: @@ -135,16 +216,29 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( method.moe_kernel = None layer = Mock() - converted_weights = tuple(object() for _ in range(8)) - convert_weights = Mock(return_value=converted_weights) - process_weights = Mock() + converted_weights = [ + tuple(torch.full((2,), value + index) for index in range(8)) + for value in (1.0, 2.0, 0.5) + ] + convert_weights = Mock(side_effect=converted_weights) + quant_config = SimpleNamespace( + g1_alphas=converted_weights[0][2].clone(), + g2_alphas=converted_weights[0][6].clone(), + ) + alpha_ptrs = (quant_config.g1_alphas.data_ptr(), quant_config.g2_alphas.data_ptr()) + observed_scales = [] + process_weights = Mock( + side_effect=lambda _: observed_scales.append( + (quant_config.g1_alphas.clone(), quant_config.g2_alphas.clone()) + ) + ) kernel = SimpleNamespace( fused_experts=SimpleNamespace( process_weights_after_loading=process_weights, ) ) make_kernel = Mock(return_value=kernel) - get_quant_config = Mock(return_value=object()) + get_quant_config = Mock(return_value=quant_config) method.get_fused_moe_quant_config = get_quant_config monkeypatch.setattr( @@ -161,14 +255,22 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( make_kernel, ) - method._setup_kernel(layer) - method._setup_kernel(layer) + for weights in converted_weights: + method._setup_kernel(layer) + # Expert post-processing must observe fresh scales, with stable storage. + torch.testing.assert_close(observed_scales[-1][0], weights[2]) + torch.testing.assert_close(observed_scales[-1][1], weights[6]) + assert method.moe_quant_config is quant_config + assert ( + quant_config.g1_alphas.data_ptr(), + quant_config.g2_alphas.data_ptr(), + ) == alpha_ptrs assert method.moe_kernel is kernel - assert convert_weights.call_count == 2 + assert convert_weights.call_count == 3 make_kernel.assert_called_once() get_quant_config.assert_called_once() - assert process_weights.call_count == 2 + assert process_weights.call_count == 3 def _fully_quantized_quark_config() -> QuarkConfig: @@ -503,38 +605,156 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: ModelConfig(model=str(tmp_path), quantization="fp8_per_channel") +@pytest.mark.parametrize("per_token_activation", [False, True]) +@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16]) +def test_nvfp4_one_sided_sizes_dispatched_activations( + monkeypatch, per_token_activation: bool, input_dtype: torch.dtype +) -> None: + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.all2all_utils import ( + flashinfer_one_sided_dispatch_layout, + ) + from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig + from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import ( + TrtLlmNvFp4ExpertsModular, + ) + + config = make_dummy_moe_config(hidden_dim=6144, in_dtype=input_dtype) + config.moe_parallel_config.dp_size = 2 + config.moe_parallel_config.ep_size = 2 + config.moe_parallel_config.use_ep = True + config.moe_parallel_config.all2all_backend = "flashinfer_nvlink_one_sided" + expert_quant_config = FusedMoEQuantConfig.make("nvfp4") + + def check_dispatch_layout(*, quant_config, input_dtype, **kwargs): + assert quant_config is expert_quant_config + layout = flashinfer_one_sided_dispatch_layout( + config.hidden_dim, quant_config, input_dtype=input_dtype + ) + if per_token_activation: + assert input_dtype == config.in_dtype + assert ( + layout.x_bytes_per_token == config.hidden_dim * config.in_dtype.itemsize + ) + assert layout.x_sf_bytes_per_token == 0 + else: + assert input_dtype is None + assert layout.x_bytes_per_token == config.hidden_dim // 2 + assert layout.x_sf_bytes_per_token == config.hidden_dim // 16 + assert expert_quant_config.quant_dtype == "nvfp4" + raise RuntimeError("dispatch layout verified before allocation") + + monkeypatch.setattr( + nvfp4_oracle, "maybe_make_prepare_finalize", check_dispatch_layout + ) + with pytest.raises( + RuntimeError, match="dispatch layout verified before allocation" + ): + nvfp4_oracle.make_nvfp4_moe_kernel( + moe_quant_config=expert_quant_config, + moe_config=config, + experts_cls=TrtLlmNvFp4ExpertsModular, + backend=nvfp4_oracle.NvFp4MoeBackend.FLASHINFER_TRTLLM, + per_token_activation=per_token_activation, + ) + + +@pytest.mark.parametrize( + "input_dtype", + [torch.float32, torch.float8_e4m3fn, torch.float8_e5m2, torch.int8], +) +def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtype): + from vllm.model_executor.layers.fused_moe.all2all_utils import ( + flashinfer_one_sided_dispatch_layout, + ) + from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig + + with pytest.raises(ValueError, match="unpacked inputs must be float16 or bfloat16"): + flashinfer_one_sided_dispatch_layout( + 6144, FusedMoEQuantConfig.make("nvfp4"), input_dtype=input_dtype + ) + + @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", ) @pytest.mark.parametrize( - ("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"), + ( + "model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls,extra_runner_kwargs" + ), [ # simple case - quantization='fp8_per_tensor' - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_tensor", None, Fp8PerTensorOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="fp8_per_tensor", + ), + # FP8 KV cache with the fp8_per_tensor shorthand. + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + {"kv_cache_dtype": "fp8"}, + id="fp8_per_tensor-fp8-kv-cache", + ), + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + {"linear_backend": "marlin", "moe_backend": "marlin"}, + marks=pytest.mark.skipif( + not current_platform.is_cuda(), + reason="Marlin online FP8 coverage is CUDA-only.", + ), + id="fp8_per_tensor-marlin", + ), + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + { + "kv_cache_dtype": "fp8", + "linear_backend": "marlin", + "moe_backend": "marlin", + }, + marks=pytest.mark.skipif( + not current_platform.is_cuda(), + reason="Marlin online FP8 coverage is CUDA-only.", + ), + id="fp8_per_tensor-fp8-kv-cache-marlin", ), # simple case - quantization='fp8_per_block' - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_block", None, Fp8PerBlockOnlineLinearMethod, Fp8PerBlockOnlineMoEMethod, + {}, + id="fp8_per_block", ), - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_channel", None, Fp8PtpcOnlineLinearMethod, Fp8PtpcOnlineMoEMethod, + {}, + id="fp8_per_channel", ), # quantization='online' with per-layer-kind overrides - ( + pytest.param( GRANITE_MODEL_NAME, "online", { @@ -543,9 +763,11 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: }, Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="per_layer_kind_overrides", ), # quantization='online' with per-layer target patterns - ( + pytest.param( GRANITE_MODEL_NAME, "online", { @@ -556,9 +778,11 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: }, Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="targets", ), # ignore with direct layer name - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_tensor", # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex @@ -566,13 +790,17 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]}, Fp8PerTensorOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="ignore", ), - ( + pytest.param( GRANITE_MODEL_NAME, "mxfp4", None, Mxfp4OnlineLinearMethod, Mxfp4OnlineMoEMethod, + {}, + id="mxfp4", ), pytest.param( PARTIALLY_PREQUANTIZED_MODEL_NAME, @@ -580,19 +808,10 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}}, Mxfp8OnlineLinearMethod, CompressedTensorsMoEMethod, + {}, id="partially_prequantized_checkpoint", ), ], - ids=[ - "fp8_per_tensor", - "fp8_per_block", - "fp8_per_channel", - "per_layer_kind_overrides", - "targets", - "ignore", - "mxfp4", - "partially_prequantized_checkpoint", - ], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] @@ -603,6 +822,7 @@ def test_online_quantization( online_quant_args: dict | None, expected_linear_cls, expected_moe_cls, + extra_runner_kwargs: dict, use_rocm_aiter: bool, monkeypatch, dist_init, @@ -629,14 +849,31 @@ def test_online_quantization( ): pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.") + runner_kwargs = dict(extra_runner_kwargs) + kv_cache_dtype = runner_kwargs.get("kv_cache_dtype", "auto") + force_marlin = extra_runner_kwargs.get("linear_backend") == "marlin" + if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90): + # FA3 requires BF16 output when the query input is FP8. + runner_kwargs["dtype"] = "bfloat16" + + vllm_config_kwargs = { + "cache_config": CacheConfig(cache_dtype=kv_cache_dtype), + "kernel_config": KernelConfig( + linear_backend=runner_kwargs.get("linear_backend", "auto"), + moe_backend=runner_kwargs.get("moe_backend", "auto"), + ), + } + if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: model, vllm_config = load_model_without_vllm_runner( model_name, + dtype=runner_kwargs.get("dtype", "bfloat16"), model_config_kwargs={ "quantization_config": resolve_quantization_config( None, online_quant_args ) }, + vllm_config_kwargs=vllm_config_kwargs, model_loader_cls=DummyModelLoader, ) assert isinstance( @@ -654,7 +891,7 @@ def test_online_quantization( else: model, vllm_config = load_model_without_vllm_runner( model_name, - dtype="bfloat16", + dtype=runner_kwargs.get("dtype", "bfloat16"), quantization=quant_scheme, model_config_kwargs={ "quantization_config": resolve_quantization_config( @@ -672,6 +909,7 @@ def test_online_quantization( "num_experts_per_tok": 2, }, }, + vllm_config_kwargs=vllm_config_kwargs, model_loader_cls=DummyModelLoader, ) @@ -699,7 +937,16 @@ def test_online_quantization( elif quant_scheme == "mxfp4": assert o_proj.weight.dtype == torch.uint8 elif current_platform.is_cuda() or current_platform.is_xpu(): - assert o_proj.weight.dtype == torch.float8_e4m3fn + if current_platform.supports_fp8() and not force_marlin: + assert o_proj.weight.dtype == torch.float8_e4m3fn + assert not isinstance( + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) + else: + assert o_proj.weight.dtype == torch.int32 + assert isinstance( + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) elif current_platform.is_rocm(): assert o_proj.weight.dtype == current_platform.fp8_dtype() else: @@ -723,6 +970,11 @@ def test_online_quantization( layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod ) + if kv_cache_dtype == "fp8": + attn = model.model.layers[0].self_attn.attn + assert attn._k_scale == 1.0 + assert attn._v_scale == 1.0 + if ( model_name == GRANITE_MODEL_NAME and isinstance(online_quant_args, dict) diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 39481456fd..54d3127af0 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config): UnquantizedFusedMoEMethod, ) - dynamic_mxfp4_config = _make_qtensor_config( + mxfp4_config = _make_qtensor_config( { "dtype": "fp4", "qscheme": "per_group", @@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config): None, exclude=["self_attn.q_proj", "mlp.down_proj"], ) - dynamic_mxfp4_config.dynamic_mxfp4_quant = True - - assert dynamic_mxfp4_config.get_quant_method_target( - "self_attn.q_proj", LinearBase - ) == (kMxfp4Static, None, QuarkLinearMethod) - attention_proj = TestLinear() - assert isinstance( - dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"), - QuarkLinearMethod, + assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, ) - assert isinstance(attention_proj.scheme, QuarkOCP_MX) - assert attention_proj.scheme.dynamic_mxfp4_quant - - assert dynamic_mxfp4_config.get_quant_method_target( - "mlp.down_proj", LinearBase - ) == (None, None, UnquantizedLinearMethod) assert isinstance( - dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), + mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"), + UnquantizedLinearMethod, + ) + + assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, + ) + assert isinstance( + mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), UnquantizedLinearMethod, ) @@ -910,6 +909,184 @@ def test_quant_method_dispatch_target(case): assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls) +def test_quant_method_dispatch_mxfp8_2d_block(default_vllm_config): + """Shape B (DeepSeek-V4.1): 32x32 per-block e8m0 routes to QuarkOCP_MX. + + Verifies that get_quant_method and get_quant_method_target agree (the + short-circuit that previously caused them to diverge has been removed). + """ + from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX + from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + "symmetric": True, + "is_dynamic": False, + "scale_type": "float8_e8m0fnu", + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "symmetric": True, + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": { + "weight": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + }, + "layer_type_quant_config": {}, + "layer_quant_config": {"layers.0.attn.wkv": mxfp8_spec}, + "exclude": [], + } + ) + + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + # get_quant_method_target now routes MXFP8 through the matcher chain. + wk, ak, mcls = config.get_quant_method_target("layers.0.attn.wkv", LinearBase) + assert wk == kMxfp8Static + assert mcls is QuarkLinearMethod + + linear = TestLinear() + method = config.get_quant_method(linear, "layers.0.attn.wkv") + assert isinstance(method, QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkOCP_MX) + assert linear.scheme.weight_quant_key == kMxfp8Static + assert linear.scheme.scale_block_rows == 32 + + # Experts still fall through to the global MXFP4 spec. + assert ( + config.get_quant_method_target("layers.0.ffn.experts", RoutedExperts)[2] + is not QuarkLinearMethod + ) + + # 128x128 per-block FP8 (DeepSeek V4) keeps its existing Quark scheme. + v4_spec = { + "weight": {**mxfp8_spec["weight"], "block_size": [128, 128]}, + "input_tensors": {**mxfp8_spec["input_tensors"], "group_size": 128}, + } + v4_config = _make_qtensor_config(v4_spec["weight"], v4_spec["input_tensors"]) + assert v4_config.get_quant_method_target("linear", LinearBase)[2] is ( + QuarkLinearMethod + ) + linear = TestLinear() + assert isinstance(v4_config.get_quant_method(linear, "linear"), QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkW8A8Fp8PerBlock) + + +def test_quant_method_dispatch_mxfp8_canonical(default_vllm_config): + """Shape A (canonical MX): per_group/group_size=32/e8m0 FP8 routes + to QuarkOCP_MX with scale_block_rows == 1. + + This covers the canonical MXFP8 spelling (no 2-D block). Validated by + synthetic dispatch test only — no canonical-MXFP8 checkpoint on hand. + """ + from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX + from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_canonical_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": mxfp8_canonical_spec, + "layer_type_quant_config": {}, + "layer_quant_config": {}, + "exclude": [], + } + ) + + wk, ak, mcls = config.get_quant_method_target("linear", LinearBase) + assert wk == kMxfp8Static + assert mcls is QuarkLinearMethod + + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + linear = TestLinear() + method = config.get_quant_method(linear, "linear") + assert isinstance(method, QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkOCP_MX) + assert linear.scheme.weight_quant_key == kMxfp8Static + assert linear.scheme.scale_block_rows == 1 + + +def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config): + """MXFP8 in a MoE config raises ValueError — experts are unsupported.""" + from vllm.model_executor.layers.quantization.quark.quark_moe import ( + QuarkOCP_MX_MoEMethod, + ) + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": mxfp8_spec, + "layer_type_quant_config": {}, + "layer_quant_config": {}, + "exclude": [], + } + ) + wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts) + assert mcls is QuarkOCP_MX_MoEMethod + # The OCP MX MoE constructor should fail loudly for MXFP8. + fake_moe_config = MagicMock() + with pytest.raises(ValueError, match="MXFP8 experts are not supported"): + QuarkOCP_MX_MoEMethod(fake_moe_config, wk, ak) + + @pytest.mark.parametrize( ("weight", "input_tensors"), [ @@ -2314,3 +2491,107 @@ def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric) ) assert torch.equal(actual, expected) + + +# --------------------------------------------------------------------------- +# override_quantization_method: Quark always delegates to QuarkConfig +# --------------------------------------------------------------------------- + +_QUARK_MXFP4_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + }, +} + +_QUARK_MXFP8_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + "scale_type": "float8_e8m0fnu", + "symmetric": True, + }, + }, +} + +_QUARK_MIXED_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + }, + "layer_quant_config": {"layers.0.attn": {"weight": {"dtype": "fp8_e4m3"}}}, +} + +_FP8_CFG = {"quant_method": "fp8"} + +_DSV4_FP8_CFG = {"quant_method": "deepseek_v4_fp8"} + + +@pytest.mark.parametrize( + "quant_config_cls", + [ + "deepseek_v41", + "deepseek_v4", + ], +) +@pytest.mark.parametrize( + "hf_quant_cfg, expect_none", + [ + (_QUARK_MXFP4_CFG, True), + (_QUARK_MXFP8_CFG, True), + (_QUARK_MIXED_CFG, True), + (_FP8_CFG, False), + (_DSV4_FP8_CFG, False), + ], + ids=[ + "quark-mxfp4", + "quark-mxfp8", + "quark-mixed", + "native-fp8", + "deepseek-v4-fp8", + ], +) +def test_quark_override_delegates_to_quark_config( + quant_config_cls, hf_quant_cfg, expect_none +): + """Any quant_method=='quark' must return None (delegate to QuarkConfig). + + Native FP8 configs should still be claimed by the model-specific config. + """ + import importlib + + if quant_config_cls == "deepseek_v41": + # Module was renamed from deepseek_v4_1 to deepseek_v41 upstream. + for mod_name in ( + "vllm.models.deepseek_v41.quant_config", + "vllm.models.deepseek_v4_1.quant_config", + ): + try: + mod = importlib.import_module(mod_name) + break + except ModuleNotFoundError: + continue + else: + pytest.skip("deepseek_v41 quant_config not found") + model_type = "deepseek_v41" + else: + mod = importlib.import_module("vllm.models.deepseek_v4.quant_config") + model_type = "deepseek_v4" + + config_cls = mod.DeepseekV4FP8Config + hf_config = SimpleNamespace(model_type=model_type) + result = config_cls.override_quantization_method( + hf_quant_cfg, None, hf_config=hf_config + ) + + if expect_none: + assert result is None, ( + f"Quark config should delegate to QuarkConfig (return None), got {result!r}" + ) + else: + assert result == "deepseek_v4_fp8", ( + f"Native FP8 should be claimed by DeepseekV4FP8Config, got {result!r}" + ) diff --git a/tests/quantization/test_rocm_moe.py b/tests/quantization/test_rocm_moe.py new file mode 100644 index 0000000000..f6f75b0c0e --- /dev/null +++ b/tests/quantization/test_rocm_moe.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm equivalents of the quantized-MoE model startup/generation contract. + +Like test_blackwell_moe.py, these tests use reduced models and dummy weights. +They validate model/backend integration, not trained-model accuracy. +""" + +import math + +import pytest + +from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform +from vllm.platforms.rocm import on_gfx950 + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm() or not on_gfx950(), + reason="ROCm FP8/MXFP4 model integration requires gfx950", +) + +COMMON_OVERRIDES = { + "num_hidden_layers": 4, + "hidden_size": 512, + "intermediate_size": 1024, + "num_attention_heads": 8, + "num_key_value_heads": 2, +} + +CASES = [ + pytest.param( + "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8", + "efe1ced428db63e7ccbcf367334596f77e9af140", + "triton", + "TRITON_MLA", + { + "n_routed_experts": 8, + "n_shared_experts": 1, + "num_experts_per_tok": 2, + "moe_intermediate_size": 512, + }, + ["--code-revision", "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11"], + id="deepseek-fp8-per-tensor-triton", + ), + pytest.param( + "Qwen/Qwen3-30B-A3B-FP8", + "d206ba732169f29bb77fbf80fc2c4b81d4d30782", + "triton", + "TRITON_ATTN", + { + "num_experts": 8, + "num_experts_per_tok": 2, + "moe_intermediate_size": 512, + "head_dim": 64, + }, + [], + id="qwen3-fp8-block-triton", + ), + pytest.param( + "openai/gpt-oss-20b", + "6cee5e81ee83917806bbde320786a8fb61efebee", + "aiter", + "TRITON_ATTN", + { + "num_local_experts": 8, + "num_experts_per_tok": 2, + "experts_per_token": 2, + "layer_types": ["sliding_attention", "full_attention"] * 2, + }, + [], + id="gptoss-mxfp4-aiter", + ), +] + + +@pytest.mark.parametrize("enforce_eager", [True, False], ids=["eager", "compiled"]) +@pytest.mark.parametrize( + "model,revision,moe_backend,attention_backend,overrides,extra_args", CASES +) +def test_rocm_quantized_moe_generation( + model, + revision, + moe_backend, + attention_backend, + overrides, + extra_args, + enforce_eager, +): + """Quantized MoE models warm up and return finite logits through serving.""" + args = [ + "--revision", + revision, + "--load-format", + "dummy", + "--trust-remote-code", + "--dtype", + "bfloat16", + "--max-model-len", + "2048", + "--max-num-batched-tokens", + "256", + "--max-num-seqs", + "8", + "--gpu-memory-utilization", + "0.02", + "--kv-cache-memory-bytes", + str(256 * 1024 * 1024), + "--moe-backend", + moe_backend, + "--attention-backend", + attention_backend, + *extra_args, + ] + if enforce_eager: + args.append("--enforce-eager") + + with RemoteOpenAIServer( + model, + args, + env_dict={"VLLM_ROCM_USE_AITER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"}, + override_hf_configs={**COMMON_OVERRIDES, **overrides}, + max_wait_seconds=1200, + ) as server: + prompts = [[1, 2, 3], [1, 2, 3, 4] * 4 + [5]] + completion = server.get_client().completions.create( + model=model, + prompt=prompts, + temperature=0, + max_tokens=2, + logprobs=1, + extra_body={"ignore_eos": True, "skip_special_tokens": False}, + ) + assert len(completion.choices) == len(prompts) + assert completion.usage is not None + assert completion.usage.prompt_tokens == sum(map(len, prompts)) + assert completion.usage.completion_tokens == 2 * len(prompts) + for choice in completion.choices: + assert choice.finish_reason == "length" + assert choice.text + assert choice.logprobs is not None + assert choice.logprobs.token_logprobs is not None + assert len(choice.logprobs.token_logprobs) == 2 + assert all( + p is not None and math.isfinite(p) + for p in choice.logprobs.token_logprobs + ) diff --git a/tests/reasoning/test_deepseekv3_reasoning_parser.py b/tests/reasoning/test_deepseekv3_reasoning_parser.py index 03867fb87b..d6bbea2f91 100644 --- a/tests/reasoning/test_deepseekv3_reasoning_parser.py +++ b/tests/reasoning/test_deepseekv3_reasoning_parser.py @@ -87,3 +87,14 @@ def test_identity_reasoning_parser_basic(tokenizer): delta_token_ids=[], ) assert result_none is None + + +@pytest.mark.parametrize("thinking,expected", [(True, 3), (False, 0)]) +def test_count_reasoning_tokens_delegates(tokenizer, thinking, expected): + parser = DeepSeekV3ReasoningParser( + tokenizer, chat_template_kwargs={"thinking": thinking} + ) + think = tokenizer.convert_tokens_to_ids(["", ""]) + token_ids = [think[0], 11, 12, 13, think[1], 14] + + assert parser.count_reasoning_tokens(token_ids) == expected diff --git a/tests/reasoning/test_granite_thinking_reasoning_parser.py b/tests/reasoning/test_granite_thinking_reasoning_parser.py new file mode 100644 index 0000000000..39ce3bf436 --- /dev/null +++ b/tests/reasoning/test_granite_thinking_reasoning_parser.py @@ -0,0 +1,643 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TypedDict + +import pytest +import regex as re + +from tests.reasoning.utils import run_reasoning_extraction +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.engine.registered_adapters import ( + GraniteThinkingParserReasoningAdapter, +) +from vllm.reasoning import ReasoningParser, ReasoningParserManager + +parser_name = "granite_thinking_parser" + + +class ReasoningCase(TypedDict): + output: str + reasoning: str | None + content: str | None + + +class GraniteThinkingTokenizer: + def __init__(self): + self._vocab = { + "": 1, + "": 2, + } + self._inv_vocab = {v: k for k, v in self._vocab.items()} + self._pattern = re.compile(r"(|)") + + def get_vocab(self) -> dict[str, int]: + return self._vocab + + def tokenize(self, text: str) -> list[str]: + tokens: list[str] = [] + for part in self._pattern.split(text): + if part: + tokens.append(part) + return tokens + + def convert_tokens_to_string(self, tokens: list[str]) -> str: + return "".join(tokens) + + def decode(self, token_ids: list[int]) -> str: + return "".join(self._inv_vocab.get(tid, f"") for tid in token_ids) + + +@pytest.fixture +def tokenizer(): + return GraniteThinkingTokenizer() + + +# ── Basic reasoning extraction (non-streaming + streaming) ─────────── + + +@pytest.mark.parametrize( + "streaming,param_dict", + [ + pytest.param( + False, + { + "output": "reasoning\nHello", + "reasoning": "reasoning", + "content": "Hello", + }, + id="leading_newline_stripped", + ), + pytest.param( + True, + { + "output": "reasoning\nHello", + "reasoning": "reasoning", + "content": "Hello", + }, + id="leading_newline_stripped_streaming", + ), + pytest.param( + False, + { + "output": "rc", + "reasoning": "r", + "content": "c", + }, + id="simple_reasoning", + ), + pytest.param( + True, + { + "output": "rc", + "reasoning": "r", + "content": "c", + }, + id="simple_reasoning_streaming", + ), + pytest.param( + False, + { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + }, + id="without_start_token", + ), + pytest.param( + True, + { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + }, + id="without_start_token_streaming", + ), + pytest.param( + False, + { + "output": "This is a reasoning sectionThis is the rest", # noqa: E501 + "reasoning": "This is a reasoning section", + "content": "This is the rest", + }, + id="with_start_token", + ), + pytest.param( + True, + { + "output": "This is a reasoning sectionThis is the rest", # noqa: E501 + "reasoning": "This is a reasoning section", + "content": "This is the rest", + }, + id="with_start_token_streaming", + ), + pytest.param( + False, + { + "output": "reasoning\n\n\nHello", + "reasoning": "reasoning", + "content": "Hello", + }, + id="multiple_leading_newlines_stripped", + ), + pytest.param( + True, + { + "output": "reasoning\n\n\nHello", + "reasoning": "reasoning", + "content": "Hello", + }, + id="multiple_leading_newlines_stripped_streaming", + ), + pytest.param( + False, + { + "output": "line1\nline2\nresult1\nresult2", + "reasoning": "line1\nline2", + "content": "result1\nresult2", + }, + id="multiline_reasoning_and_content", + ), + pytest.param( + True, + { + "output": "line1\nline2\nresult1\nresult2", + "reasoning": "line1\nline2", + "content": "result1\nresult2", + }, + id="multiline_reasoning_and_content_streaming", + ), + ], +) +def test_granite_thinking_reasoning( + tokenizer: GraniteThinkingTokenizer, + streaming: bool, + param_dict: ReasoningCase, +): + output = tokenizer.tokenize(param_dict["output"]) + model_output = [tokenizer.convert_tokens_to_string([token]) for token in output] + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( + tokenizer + ) + + reasoning, content = run_reasoning_extraction( + parser, model_output, streaming=streaming + ) + + assert reasoning == param_dict["reasoning"] + assert content == param_dict["content"] + + +# ── No content after end token ─────────────────────────────────────── + + +def test_granite_thinking_no_content_after_end_token( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning", ""], + streaming=False, + ) + + assert reasoning == "reasoning" + assert content is None + + +# ── Whitespace-only content after end token ────────────────────────── + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_granite_thinking_whitespace_only_content( + tokenizer: GraniteThinkingTokenizer, + streaming: bool, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning", "", "\n\n"], + streaming=streaming, + ) + + assert reasoning == "reasoning" + assert content is None + + +# ── Unterminated think block ───────────────────────────────────────── + + +def test_granite_thinking_unterminated_think_block( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"enable_thinking": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning only"], + request=request, + streaming=False, + ) + + assert reasoning == "reasoning only" + assert content is None + + +# ── enable_thinking=False ──────────────────────────────────────────── + + +def test_granite_thinking_disabled_moves_into_content( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"enable_thinking": False}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["This is plain content"], + request=request, + streaming=False, + ) + + assert reasoning is None + assert content == "This is plain content" + + +# ── enable_thinking=False + leading newline (§6.2 ordering) ───────── + + +def test_granite_thinking_disabled_with_leading_newline( + tokenizer: GraniteThinkingTokenizer, +): + # With enable_thinking=False, model output goes through the swap + # path: content is initially None (all text classified as + # reasoning), so lstrip doesn't fire pre-swap. After the swap, + # any leading \n in the original output is preserved. In practice, + # enable_thinking=False output has no template-injected \n, so + # this is a correctness check, not a realistic scenario. + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"enable_thinking": False}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["\nThis is plain content"], + request=request, + streaming=False, + ) + + assert reasoning is None + assert content == "\nThis is plain content" + + +# ── force_nonempty_content=True ────────────────────────────────────── + + +def test_granite_thinking_force_nonempty_content_moves_into_content( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "This is plain content"], + request=request, + streaming=False, + ) + + assert reasoning is None + assert content == "This is plain content" + + +def test_granite_thinking_force_nonempty_no_swap_when_newlines_only( + tokenizer: GraniteThinkingTokenizer, +): + # When IS present and content is newlines-only, lstrip + # removes them but the swap should NOT fire — content was present, + # just whitespace. Matches the HF plugin behavior. + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning", "", "\n\n"], + request=request, + streaming=False, + ) + + assert reasoning == "reasoning" + assert content is None + + +def test_granite_thinking_force_nonempty_swaps_when_content_absent( + tokenizer: GraniteThinkingTokenizer, +): + # When IS present but content is truly absent (zero + # characters after , e.g. max_tokens cut), swap fires. + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning", ""], + request=request, + streaming=False, + ) + + assert reasoning is None + assert content == "reasoning" + + +def test_granite_thinking_force_nonempty_keeps_real_content( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "reasoning here", "", "real answer"], + request=request, + streaming=False, + ) + + assert reasoning == "reasoning here" + assert content == "real answer" + + +# ── Truncated reasoning with thinking on ───────────────────────────── + + +def test_granite_thinking_keeps_truncated_reasoning( + tokenizer: GraniteThinkingTokenizer, +): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"enable_thinking": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["This is truncated reasoning"], + request=request, + streaming=False, + ) + + assert reasoning == "This is truncated reasoning" + assert content is None + + +# ── DelegatingParser / parse_delta streaming tests ─────────────────── + +_SPECIAL_TOKEN_IDS = {"": 1, "": 2} + + +def _token_id(token: str) -> int: + return _SPECIAL_TOKEN_IDS.get(token, 0) + + +def _make_reasoning_parser(tokenizer): + class _GraniteThinkingDelegating(DelegatingParser): + reasoning_parser_cls = GraniteThinkingParserReasoningAdapter + tool_parser_cls = None + + return _GraniteThinkingDelegating(tokenizer) + + +def _run_parse_delta(parser, tokenizer, text, request): + tokens = tokenizer.tokenize(text) + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + for i, token in enumerate(tokens): + delta = parser.parse_delta( + delta_text=token, + delta_token_ids=[_token_id(token)], + request=request, + prompt_token_ids=[] if i == 0 else None, + finished=(i == len(tokens) - 1), + ) + if delta is None: + continue + if delta.reasoning: + reasoning_parts.append(delta.reasoning) + if delta.content: + content_parts.append(delta.content) + return "".join(reasoning_parts), "".join(content_parts) + + +def test_granite_thinking_streaming_enable_thinking_false( + tokenizer: GraniteThinkingTokenizer, +): + # With enable_thinking=False, the parser (constructed without + # kwargs) starts in REASONING state. All text streams as reasoning + # AND is duplicated into content via the streaming fallback — + # matching NemotronV3 behavior. + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"enable_thinking": False}, + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta( + parser, tokenizer, "This is plain content", request + ) + + assert reasoning == "This is plain content" + assert content == "This is plain content" + + +def test_granite_thinking_streaming_strips_leading_newline( + tokenizer: GraniteThinkingTokenizer, +): + request = ChatCompletionRequest( + model="test-model", + messages=[], + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta( + parser, tokenizer, "reason\nHello", request + ) + + assert reasoning == "reason" + assert content == "Hello" + + +def test_granite_thinking_streaming_promotes_reasoning_to_content( + tokenizer: GraniteThinkingTokenizer, +): + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta(parser, tokenizer, "4", request) + + assert reasoning == "4" + assert content == "4" + + +def test_granite_thinking_streaming_no_promotion_with_real_content( + tokenizer: GraniteThinkingTokenizer, +): + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta( + parser, + tokenizer, + "reasonreal answer", + request, + ) + + assert reasoning == "reason" + assert content == "real answer" + + +def test_granite_thinking_streaming_no_promotion_without_opt_in( + tokenizer: GraniteThinkingTokenizer, +): + request = ChatCompletionRequest(model="test-model", messages=[]) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta(parser, tokenizer, "4", request) + + assert reasoning == "4" + assert content == "" + + +# ── Empty think block ──────────────────────────────────────────────── + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_granite_thinking_empty_think_block( + tokenizer: GraniteThinkingTokenizer, + streaming: bool, +): + # \nHello — empty reasoning, content after newline. + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + + reasoning, content = run_reasoning_extraction( + parser, + ["", "", "\n", "Hello"], + streaming=streaming, + ) + + assert reasoning is None or reasoning == "" + assert content == "Hello" + + +# ── Chunking-independence streaming test ───────────────────────────── + + +@pytest.mark.parametrize( + "text,expected_reasoning,expected_content", + [ + ("r\nHello", "r", "Hello"), + ("\nHello", "", "Hello"), + ], +) +def test_granite_thinking_streaming_chunking_independent( + tokenizer: GraniteThinkingTokenizer, + text: str, + expected_reasoning: str, + expected_content: str, +): + # Verify the same input produces identical results regardless + # of how it's chunked into streaming deltas. Uses the + # DelegatingParser path (_run_parse_delta) which matches the + # real serving flow. + tokens = tokenizer.tokenize(text) + request = ChatCompletionRequest(model="test-model", messages=[]) + + chunk_patterns = [ + tokens, + # Split at the boundary + [ + tokenizer.convert_tokens_to_string(tokens[: tokens.index("") + 1]), + tokenizer.convert_tokens_to_string(tokens[tokens.index("") + 1 :]), + ], + ] + + for chunks in chunk_patterns: + parser = _make_reasoning_parser(tokenizer) + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + all_tokens = [] + for chunk in chunks: + chunk_tokens = tokenizer.tokenize(chunk) + all_tokens.extend(chunk_tokens) + for i, token in enumerate(all_tokens): + delta = parser.parse_delta( + delta_text=token, + delta_token_ids=[_token_id(token)], + request=request, + prompt_token_ids=[] if i == 0 else None, + finished=(i == len(all_tokens) - 1), + ) + if delta is None: + continue + if delta.reasoning: + reasoning_parts.append(delta.reasoning) + if delta.content: + content_parts.append(delta.content) + reasoning = "".join(reasoning_parts) + content = "".join(content_parts) + assert reasoning == expected_reasoning or ( + not expected_reasoning and not reasoning + ), f"reasoning mismatch with chunks={chunks}" + assert content == expected_content, f"content mismatch with chunks={chunks}" diff --git a/tests/reasoning/test_kimi_k3_reasoning_parser.py b/tests/reasoning/test_kimi_k3_reasoning_parser.py index ea5b374e6e..45c5511128 100644 --- a/tests/reasoning/test_kimi_k3_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k3_reasoning_parser.py @@ -19,6 +19,9 @@ SEP = "<|sep|>" THINK_OPEN = f"{OPEN}think{SEP}" THINK_CLOSE = f"{CLOSE}think{SEP}" RESPONSE_OPEN = f"{OPEN}response{SEP}" +OPEN_IDS = [1, 2, 3] +CLOSE_IDS = [4, 2, 3] +RESPONSE_OPEN_IDS = [ord(ch) for ch in RESPONSE_OPEN] class DummyTokenizer: @@ -119,6 +122,54 @@ def test_is_reasoning_end_ignores_stale_close_from_prior_turn(): assert not parser.is_reasoning_end([*new_open]) +@pytest.mark.parametrize( + ("token_ids", "expected"), + [ + pytest.param( + [*OPEN_IDS, 9, 10, *CLOSE_IDS, *RESPONSE_OPEN_IDS, 11], + 2, + id="open_and_close_markers", + ), + pytest.param( + [9, 10, *CLOSE_IDS, *RESPONSE_OPEN_IDS, 11], + 2, + id="open_marker_consumed_as_generation_prefix", + ), + pytest.param([*RESPONSE_OPEN_IDS, 11, 12], 0, id="response_only"), + pytest.param([*OPEN_IDS, 9, 10], 2, id="unterminated_after_open"), + pytest.param([9, 10, 11], 3, id="unterminated_without_markers"), + pytest.param([], 0, id="empty"), + pytest.param([9, 4, 2, 10], 4, id="partial_close_marker_is_reasoning"), + pytest.param( + [*OPEN_IDS, 9, *RESPONSE_OPEN_IDS, 11], + 2 + len(RESPONSE_OPEN_IDS), + id="response_open_inside_open_think_is_reasoning", + ), + ], +) +def test_count_reasoning_tokens_matches_think_channel(token_ids, expected): + """reasoning_tokens must cover exactly what extract_reasoning labels as + reasoning: marker tokens are excluded and a consumed generation prefix + means the output starts inside the think channel.""" + parser = KimiK3ReasoningParser(DummyTokenizer()) + + assert parser.count_reasoning_tokens(token_ids) == expected + + +def test_count_reasoning_tokens_is_zero_when_thinking_disabled(): + parser = KimiK3ReasoningParser( + DummyTokenizer(), chat_template_kwargs={"thinking": False} + ) + + assert parser.count_reasoning_tokens([*OPEN_IDS, 9, 10, *CLOSE_IDS]) == 0 + + +def test_count_reasoning_tokens_through_delegating_parser(): + parser = ReasoningOnlyParser(DummyTokenizer()) + + assert parser.count_reasoning_tokens([*OPEN_IDS, 9, *CLOSE_IDS, 11]) == 1 + + def test_streaming_split_open_marker_is_held_back(): parser = KimiK3ReasoningParser(DummyTokenizer()) @@ -250,10 +301,6 @@ def test_adjust_request_keeps_xtml_markers_contiguous(): assert adjusted.spaces_between_special_tokens is False -OPEN_IDS = [1, 2, 3] -CLOSE_IDS = [4, 2, 3] - - def _reference_is_reasoning_end(input_ids: list[int]) -> bool: """Full-sequence reference for the streaming check to be measured against. diff --git a/tests/reasoning/test_step3_reasoning_parser.py b/tests/reasoning/test_step3_reasoning_parser.py new file mode 100644 index 0000000000..999d1d2645 --- /dev/null +++ b/tests/reasoning/test_step3_reasoning_parser.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.reasoning import ReasoningParserManager +from vllm.tokenizers import get_tokenizer + +pytestmark = pytest.mark.skip_global_cleanup + +REASONING_MODEL_NAME = "stepfun-ai/step3" + + +@pytest.fixture(scope="module") +def step3_parser(): + tokenizer = get_tokenizer(tokenizer_name=REASONING_MODEL_NAME) + return ReasoningParserManager.get_reasoning_parser("step3")(tokenizer) + + +def test_count_reasoning_tokens_stops_at_think_end(step3_parser): + end = step3_parser.think_end_token_id + assert step3_parser.count_reasoning_tokens([11, 12, end, 13]) == 2 + + +def test_count_reasoning_tokens_without_think_end(step3_parser): + """Step3 treats output as reasoning until appears.""" + assert step3_parser.count_reasoning_tokens([11, 12, 13]) == 3 diff --git a/tests/renderers/test_token_offsets.py b/tests/renderers/test_token_offsets.py index fc9192bb43..aad91655ba 100644 --- a/tests/renderers/test_token_offsets.py +++ b/tests/renderers/test_token_offsets.py @@ -20,7 +20,7 @@ def fast_tokenizer(): """gpt2 ships a Fast tokenizer; use it to test the offsets happy path.""" from transformers import AutoTokenizer - return AutoTokenizer.from_pretrained("openai-community/gpt2", use_fast=True) + return AutoTokenizer.from_pretrained("openai-community/gpt2") def _make_base_renderer_with(tokenizer): diff --git a/tests/samplers/conftest.py b/tests/samplers/conftest.py new file mode 100644 index 0000000000..4585e98898 --- /dev/null +++ b/tests/samplers/conftest.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm import envs +from vllm.platforms import current_platform + + +@pytest.fixture(autouse=True) +def skip_unsupported_flashinfer_sampler(): + # CI runs this suite with FlashInfer both explicitly enabled and disabled. + if ( + current_platform.is_cuda() + and envs.is_set("VLLM_USE_FLASHINFER_SAMPLER") + and envs.VLLM_USE_FLASHINFER_SAMPLER + and current_platform.num_compute_units(torch.accelerator.current_device_index()) + <= 16 + ): + pytest.skip("FlashInfer top-k masking requires more than 16 SMs") diff --git a/tests/test_config.py b/tests/test_config.py index 0720007135..bbb12e3917 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ from unittest.mock import patch import pydantic import pytest +import torch from huggingface_hub import ResolvedRevision from pydantic import ValidationError @@ -37,7 +38,7 @@ from vllm.config import ( WatermarkConfig, update_config, ) -from vllm.config.compilation import CompilationMode, CUDAGraphMode +from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig from vllm.config.kernel import IrOpPriorityConfig from vllm.config.load import LoadConfig from vllm.config.mamba import MambaBackendEnum @@ -61,16 +62,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch def original_validate_rope(config, *args, **kwargs): calls.append(config) - from transformers import PretrainedConfig + from transformers import PreTrainedConfig - monkeypatch.setattr(PretrainedConfig, "validate_rope", original_validate_rope) + monkeypatch.setattr(PreTrainedConfig, "validate_rope", original_validate_rope) _patch_hf_transformers_nested_rope_validation() nested_rope_parameters = { "full_attention": {"rope_type": "default"}, "original_max_position_embeddings": 32768, } - PretrainedConfig.validate_rope( + PreTrainedConfig.validate_rope( SimpleNamespace(rope_parameters=nested_rope_parameters) ) assert nested_rope_parameters == {"full_attention": {"rope_type": "default"}} @@ -80,7 +81,7 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch "factor": 8.0, "rope_theta": 500000.0, } - PretrainedConfig.validate_rope( + PreTrainedConfig.validate_rope( SimpleNamespace(rope_parameters=flat_rope_parameters) ) assert flat_rope_parameters == { @@ -91,6 +92,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch assert len(calls) == 2 +def test_dspark_adaptive_verification_separates_graph_cache(): + config = object.__new__(SpeculativeConfig) + config.method = "dspark" + config.draft_model_config = None + config.enable_adaptive_verification = False + fixed_hash = config.compute_hash() + config.enable_adaptive_verification = True + assert config.compute_hash() != fixed_hash + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") @@ -289,6 +300,46 @@ def test_kv_offloading_does_not_skip_dcp_interleave_validation(): VllmConfig.validate_block_size(config) +def test_nixl_dcp_check_skipped_for_submodel_config(): + # with_hf_config() builds a submodule view of the config (e.g. a + # multimodal model's text stack) whose architecture list is empty. + # Re-running VllmConfig validation on it is unsafe: the NIXL DCP check + # reads use_mla, which resolves the architecture registry. + model_config = ModelConfig("Qwen/Qwen2-VL-2B-Instruct", max_model_len=2048) + vllm_config = VllmConfig( + model_config=model_config, + device_config=DeviceConfig(device="cpu"), + kv_transfer_config=KVTransferConfig( + kv_connector="NixlConnector", + kv_role="kv_both", + ), + ) + submodel_config = vllm_config.with_hf_config(model_config.hf_text_config) + assert submodel_config.model_config.is_submodel_config + assert submodel_config.model_config.architectures == [] + assert submodel_config.model_config.use_mla is False + + +def test_nixl_dcp_check_rejects_non_mla_model_with_dcp(monkeypatch): + # Pretend the model has a single KV head so the DCP feasibility checks + # pass and the MLA-only assert is what actually fires. + monkeypatch.setattr(ModelConfig, "get_total_num_kv_heads", lambda self: 1) + with pytest.raises(ValidationError, match="only supported for MLA models"): + VllmConfig( + model_config=ModelConfig("Qwen/Qwen3-0.6B", max_model_len=2048), + device_config=DeviceConfig(device="cpu"), + parallel_config=ParallelConfig( + tensor_parallel_size=2, + decode_context_parallel_size=2, + distributed_executor_backend="mp", + ), + kv_transfer_config=KVTransferConfig( + kv_connector="NixlConnector", + kv_role="kv_both", + ), + ) + + def test_compile_config_repr_succeeds(): # setup: VllmBackend mutates the config object config = VllmConfig() @@ -509,11 +560,58 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]), attention_config=AttentionConfig(), ) + config._get_v1_model_runner_unsupported_features = lambda: [] assert VllmConfig.use_v2_model_runner.fget(config) is False finally: default_breakable_cudagraph_architectures.cache_clear() +def test_rocm_mrv1_default_yields_to_v1_unsupported_config(monkeypatch): + """The ROCm V1 default is a speed preference, not a capability claim. + + DSpark runs only on V2, so pinning DeepSeek V4 to V1 would fail config + validation instead of serving it. With nothing V1 refuses, it still holds. + """ + from vllm.platforms import current_platform + + monkeypatch.setattr(current_platform, "is_rocm", lambda: True) + monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) + monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) + + config = SimpleNamespace( + model_config=SimpleNamespace( + architectures=["DeepseekV4ForCausalLM"], is_diffusion=False + ), + attention_config=AttentionConfig(), + parallel_config=SimpleNamespace( + prefill_context_parallel_size=1, + pipeline_parallel_size=1, + enable_batch_sharded_sampling=False, + ), + scheduler_config=SimpleNamespace(async_scheduling=False), + speculative_config=None, + ) + config._dflash_needs_multi_kv_group = lambda: False + config._is_dflash_candidate_draft = lambda: False + config._get_v2_model_runner_unsupported_features = lambda: [] + # The real predicate, so the test also pins where dspark lands in it. + config._get_v1_model_runner_unsupported_features = lambda: ( + VllmConfig._get_v1_model_runner_unsupported_features(config) + ) + + assert VllmConfig.use_v2_model_runner.fget(config) is False + + config.speculative_config = SimpleNamespace( + method="dspark", enable_adaptive_verification=False + ) + assert VllmConfig.use_v2_model_runner.fget(config) is True + + # Yielding is not the same as selecting V2: the later checks still run, so + # a config neither runner can serve lands on V1 and fails validation there. + config._get_v2_model_runner_unsupported_features = lambda: ["sequence parallelism"] + assert VllmConfig.use_v2_model_runner.fget(config) is False + + @pytest.mark.parametrize( ("model", "architecture"), [ @@ -616,6 +714,70 @@ def test_breakable_cudagraph_platform_default( default_breakable_cudagraph_architectures.cache_clear() +@pytest.mark.parametrize( + "case,hidden,heads,intermediate,tp,expected", + [ + ("bf16", 2048, (16, 8), 6144, 1, True), + ("bi-off", 2048, (16, 8), 6144, 1, False), + ("opt-out", 2048, (16, 8), 6144, 1, False), + ("eager", 2048, (16, 8), 6144, 1, False), + ("sp", 2048, (16, 8), 6144, 1, False), + ("fp16", 2048, (16, 8), 6144, 1, False), + ("quantized", 2048, (16, 8), 6144, 1, False), + ("untuned", 4096, (32, 32), 11008, 1, False), + ("tp4-tuned", 4096, (8, 2), 12288, 4, True), + ("list-intermediate", 2048, (16, 8), [2048, 4096], 1, False), + ("no-table", 2048, None, 6144, 1, False), + ], +) +def test_batch_invariant_breakable_cudagraph( + monkeypatch, case, hidden, heads, intermediate, tp, expected +): + from vllm.config.vllm import default_breakable_cudagraph_architectures + from vllm.model_executor.determinism import batch_invariant_configs as bi_configs + + monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0" if case == "bi-off" else "1") + monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) + if case == "opt-out": + monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_platform, "is_rocm", lambda: False) + table = None if case == "no-table" else {(12288, 2048): None, (6144, 4096): None} + monkeypatch.setattr(current_platform, "get_device_capability", lambda: None) + monkeypatch.setattr(bi_configs, "_get_tuned_matmul_arch_family", lambda cap: "test") + monkeypatch.setattr( + bi_configs, + "_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS", + {"test": table} if table else {}, + ) + default_breakable_cudagraph_architectures.cache_clear() + config = object.__new__(VllmConfig) + config.model_config = SimpleNamespace( + architectures=["Qwen3ForCausalLM"], + enforce_eager=case == "eager", + dtype=torch.float16 if case == "fp16" else torch.bfloat16, + quantization="fp8" if case == "quantized" else None, + hf_text_config=SimpleNamespace(intermediate_size=intermediate), + get_hidden_size=lambda: hidden, + get_head_size=lambda: 128, + get_num_attention_heads=lambda pc: heads[0], + get_num_kv_heads=lambda pc: heads[1], + ) + config.parallel_config = SimpleNamespace(tensor_parallel_size=tp) + config.compilation_config = ( + CompilationConfig(pass_config=PassConfig(enable_sp=True)) + if case == "sp" + else CompilationConfig() + ) + try: + assert config._maybe_enable_breakable_cudagraph() is expected + if expected: + assert config.compilation_config.mode == CompilationMode.NONE + finally: + os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) + default_breakable_cudagraph_architectures.cache_clear() + + @pytest.mark.parametrize( ("model_type", "expected_architecture"), [ @@ -625,9 +787,9 @@ def test_breakable_cudagraph_platform_default( ], ) def test_dsa_models_select_matching_mtp(model_type, expected_architecture): - from transformers import PretrainedConfig + from transformers import PreTrainedConfig - hf_config = PretrainedConfig( + hf_config = PreTrainedConfig( architectures=["DeepseekV32ForCausalLM"], num_nextn_predict_layers=1, ) @@ -661,7 +823,8 @@ def test_v2_model_runner_supports_custom_logits_processors(): assert config._get_v2_model_runner_unsupported_features() == [] -def test_dflash2_draft_forces_v2_model_runner(): +@pytest.mark.parametrize("architecture", ["DFlash2DraftModel", "LiLiCorrDraftModel"]) +def test_dflash_candidate_draft_forces_v2_model_runner(architecture): """A DFlash2 draft must reach the V2 speculator, the only one that runs its candidate selector; on V1 it would draft as DFlash1 without raising.""" @@ -673,11 +836,15 @@ def test_dflash2_draft_forces_v2_model_runner(): ) ) - assert VllmConfig._is_dflash2_draft(config("dflash", ["DFlash2DraftModel"])) - assert not VllmConfig._is_dflash2_draft(config("dflash", ["DFlashDraftModel"])) - assert not VllmConfig._is_dflash2_draft(config("eagle", ["DFlash2DraftModel"])) - assert not VllmConfig._is_dflash2_draft(SimpleNamespace(speculative_config=None)) - assert not VllmConfig._is_dflash2_draft( + assert VllmConfig._is_dflash_candidate_draft(config("dflash", [architecture])) + assert not VllmConfig._is_dflash_candidate_draft( + config("dflash", ["DFlashDraftModel"]) + ) + assert not VllmConfig._is_dflash_candidate_draft(config("eagle", [architecture])) + assert not VllmConfig._is_dflash_candidate_draft( + SimpleNamespace(speculative_config=None) + ) + assert not VllmConfig._is_dflash_candidate_draft( SimpleNamespace( speculative_config=SimpleNamespace(method="dflash", draft_model_config=None) ) @@ -721,6 +888,8 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( ("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"), ("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"), ("FULL_DECODE_ONLY", False, "NEVER", "NONE"), + ("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"), + ("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"), ], ) def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider( @@ -1091,7 +1260,7 @@ def test_v1_model_runner_rejects_v2_only_features(): model_config=None, ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False + config._is_dflash_candidate_draft = lambda: False config._get_v1_model_runner_unsupported_features = lambda: ( VllmConfig._get_v1_model_runner_unsupported_features(config) ) @@ -1278,7 +1447,7 @@ def test_v1_model_runner_rejects_pipeline_parallelism_with_async_scheduling(): model_config=None, ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False + config._is_dflash_candidate_draft = lambda: False assert VllmConfig._get_v1_model_runner_unsupported_features(config) == [] @@ -1378,26 +1547,32 @@ def test_engram_dp_shared_memory_config_validation( @pytest.mark.parametrize( - "architecture, ple_layers, accelerator, supported", + "architecture, ple_layers, platform, supported", [ - ("DeepseekV41ForCausalLM", [1], True, True), - ("DeepseekV41ForCausalLM", [], True, False), - ("DeepseekV41ForCausalLM", [1], False, False), - ("Qwen4ExpForCausalLM", [1], True, True), - ("Qwen4ExpForConditionalGeneration", [1], True, True), - ("Qwen4ExpForCausalLM", [], True, False), - ("Qwen4ExpForCausalLM", None, True, False), - ("Qwen4ExpForCausalLM", [1], False, False), - ("LlamaForCausalLM", [1], True, False), - ("Qwen4ExpMTP", [], True, False), - (None, None, True, False), + ("DeepseekV41ForCausalLM", [1], "cuda", True), + ("DeepseekV41ForCausalLM", [1], "rocm", True), + ("DeepseekV41ForCausalLM", [], "cuda", False), + ("DeepseekV41ForCausalLM", [1], "cpu", False), + ("Qwen4ExpForCausalLM", [1], "cuda", True), + ("Qwen4ExpForCausalLM", [1], "rocm", True), + ("Qwen4ExpForConditionalGeneration", [1], "cuda", True), + ("Qwen4ExpForConditionalGeneration", [1], "rocm", True), + ("Qwen4ExpForCausalLM", [], "cuda", False), + ("Qwen4ExpForCausalLM", None, "cuda", False), + ("Qwen4ExpForCausalLM", [1], "cpu", False), + ("LlamaForCausalLM", [1], "cuda", False), + ("Qwen4ExpMTP", [], "cuda", False), + (None, None, "cuda", False), ], ) def test_engram_model_support( - monkeypatch, architecture, ple_layers, accelerator, supported + monkeypatch, architecture, ple_layers, platform, supported ): """A similarly named HF field must not enable unsupported implementations.""" - monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: accelerator) + monkeypatch.setattr(current_platform, "is_cuda", lambda: platform == "cuda") + monkeypatch.setattr( + current_platform, "is_cuda_alike", lambda: platform in ("cuda", "rocm") + ) model = ( cast( ModelConfig, @@ -2358,14 +2533,14 @@ def test_get_and_verify_max_len_with_nope_layers( max_model_len, rope_parameters, expected_max_len ): """NoPE layers do not prevent deriving or scaling the context length.""" - from transformers import PretrainedConfig + from transformers import PreTrainedConfig from vllm.config.model import _get_and_verify_max_len from vllm.transformers_utils.model_arch_config_convertor import ( ModelArchConfigConvertorBase, ) - hf_config = PretrainedConfig( + hf_config = PreTrainedConfig( max_position_embeddings=4096, original_max_position_embeddings=2048, ) @@ -2409,14 +2584,14 @@ def test_get_and_verify_max_len_yarn_is_already_scaled( for every YaRN variant, so scaling it again overstates the limit and lets requests past the end of the cos/sin cache. """ - from transformers import PretrainedConfig + from transformers import PreTrainedConfig from vllm.config.model import _get_and_verify_max_len from vllm.transformers_utils.model_arch_config_convertor import ( ModelArchConfigConvertorBase, ) - hf_config = PretrainedConfig(max_position_embeddings=32768) + hf_config = PreTrainedConfig(max_position_embeddings=32768) hf_config.rope_parameters = { "rope_type": rope_type, "factor": factor, @@ -3762,19 +3937,12 @@ def test_load_config_rejects_non_string_load_format(bad_load_format): LoadConfig(load_format=bad_load_format) -# A real Qwen3-0.6B model revision that is used in the tests below. +# A real Qwen3-0.6B model revision that is used in the test below. REVISION = "c1899de289a04d12100db370d81485cdf75e47ca" @patch("vllm.config.model.resolve_revision", return_value=ResolvedRevision(REVISION)) -def test_revision_not_resolved_when_weights_differ_from_model(mock_resolve): - model_weights = "unsloth/Qwen3-0.6B-GGUF:Q8_0" - config = ModelConfig("Qwen/Qwen3-0.6B", model_weights=model_weights) - assert config.revision is None - - -@patch("vllm.config.model.resolve_revision", return_value=ResolvedRevision(REVISION)) -def test_revision_resolved_when_weights_match_model(mock_resolve): +def test_revision_resolved_for_model(mock_resolve): model = "Qwen/Qwen3-0.6B" config = ModelConfig(model) assert isinstance(config.revision, ResolvedRevision) diff --git a/tests/test_logger.py b/tests/test_logger.py index 9dcc4a1ea8..1810eeba16 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -16,16 +16,20 @@ from uuid import uuid4 import pytest +import vllm.logger as vllm_logger +from vllm.config import LoggingConfig from vllm.logger import ( _DATE_FORMAT, _FORMAT, _configure_vllm_root_logger, _use_color, + configure_logging, enable_trace_function_call, init_logger, ) from vllm.logging_utils import NewLineFormatter from vllm.logging_utils.dump_input import prepare_object_to_dump +from vllm.utils.system_utils import decorate_logs def f1(x): @@ -50,16 +54,27 @@ def test_trace_function_call(): os.remove(path) +def test_caplog_vllm_captures_info_before_runtime_logging_is_configured(caplog_vllm): + message = "Capture this unconfigured INFO record" + logger = init_logger(f"vllm.test_logger.{uuid4()}") + + logger.info(message) + + assert message in caplog_vllm.text + + def test_default_vllm_root_logger_configuration(monkeypatch): """This test presumes that VLLM_CONFIGURE_LOGGING (default: True) and VLLM_LOGGING_CONFIG_PATH (default: None) are not configured and default behavior is activated.""" monkeypatch.setenv("VLLM_LOGGING_COLOR", "0") - _configure_vllm_root_logger() + early_logger = init_logger(f"vllm.test_logger.{uuid4()}") + configure_logging(LoggingConfig()) logger = logging.getLogger("vllm") assert logger.level == logging.INFO assert not logger.propagate + assert early_logger.isEnabledFor(logging.INFO) handler = logger.handlers[0] assert isinstance(handler, logging.StreamHandler) @@ -74,6 +89,82 @@ def test_default_vllm_root_logger_configuration(monkeypatch): assert formatter.datefmt == _DATE_FORMAT +def test_offline_llm_configures_logging_before_logging_args(monkeypatch): + import vllm.entrypoints.llm as llm_module + + class StopInitialization(Exception): + pass + + configured = False + + def configure_logging(_): + nonlocal configured + configured = True + + def log_args(_): + assert configured + raise StopInitialization + + monkeypatch.setattr(llm_module, "configure_logging_if_needed", configure_logging) + monkeypatch.setattr(llm_module, "log_non_default_args", log_args) + + with pytest.raises(StopInitialization): + llm_module.LLM(model="facebook/opt-125m") + + +def test_json_logging(monkeypatch, tmp_path): + output = io.StringIO() + logging_config = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "json": { + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": ( + "%(asctime)s %(levelname)s %(name)s %(vllm_process_name)s " + "%(process)d %(message)s" + ), + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "json", + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "vllm": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, + } + + logging_config_path = tmp_path / "logging_config.json" + logging_config_path.write_text(json.dumps(logging_config)) + + try: + with monkeypatch.context() as context: + context.setattr(sys, "stdout", output) + # Restore this module-global state when the context exits. + context.setattr(vllm_logger, "_vllm_process_info", None) + _configure_vllm_root_logger( + LoggingConfig(pylogging_config_file=str(logging_config_path)) + ) + decorate_logs("Worker_DP0") + init_logger("vllm.structured_log_probe").info("structured log probe") + + log = json.loads(output.getvalue()) + finally: + _configure_vllm_root_logger(LoggingConfig()) + + assert log["message"] == "structured log probe" + assert log["vllm_process_name"] == "Worker_DP0" + assert log["process"] == os.getpid() + + def test_use_color_force_color(monkeypatch): """FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an explicit VLLM_LOGGING_COLOR=0 take precedence over it.""" @@ -133,7 +224,7 @@ def test_logger_configuring_can_be_disabled(monkeypatch): monkeypatch.delenv("VLLM_LOGGING_CONFIG_PATH", raising=False) with patch("vllm.logger.dictConfig") as dict_config_mock: - _configure_vllm_root_logger() + configure_logging(LoggingConfig(configure_logging=False)) dict_config_mock.assert_not_called() @@ -214,9 +305,11 @@ def test_custom_logging_config_is_parsed_and_used_when_provided(monkeypatch): with NamedTemporaryFile(encoding="utf-8", mode="w") as logging_config_file: logging_config_file.write(json.dumps(valid_logging_config)) logging_config_file.flush() - monkeypatch.setenv("VLLM_LOGGING_CONFIG_PATH", logging_config_file.name) + monkeypatch.delenv("VLLM_LOGGING_CONFIG_PATH", raising=False) with patch("vllm.logger.dictConfig") as dict_config_mock: - _configure_vllm_root_logger() + configure_logging( + LoggingConfig(pylogging_config_file=logging_config_file.name) + ) dict_config_mock.assert_called_with(valid_logging_config) @@ -237,13 +330,18 @@ def test_custom_logging_config_causes_an_error_if_configure_logging_is_off(monke with NamedTemporaryFile(encoding="utf-8", mode="w") as logging_config_file: logging_config_file.write(json.dumps(valid_logging_config)) logging_config_file.flush() - monkeypatch.setenv("VLLM_LOGGING_CONFIG_PATH", logging_config_file.name) + monkeypatch.delenv("VLLM_LOGGING_CONFIG_PATH", raising=False) with pytest.raises(RuntimeError) as ex_info: - _configure_vllm_root_logger() + configure_logging( + LoggingConfig( + configure_logging=False, + pylogging_config_file=logging_config_file.name, + ) + ) assert ex_info.type is RuntimeError expected_message_snippet = ( - "VLLM_CONFIGURE_LOGGING evaluated to false, but " - "VLLM_LOGGING_CONFIG_PATH was given." + "Logging configuration is disabled, but a Python logging config " + "file was given." ) assert expected_message_snippet in str(ex_info) @@ -297,6 +395,8 @@ test_logger = init_logger("vllm.test_logger") def mp_function(**kwargs): # This function runs in a subprocess + if logging_config := kwargs.pop("logging_config", None): + configure_logging(logging_config) test_logger.warning("This is a subprocess: %s", kwargs.get("a")) test_logger.error("This is a subprocess error.") @@ -328,7 +428,13 @@ def test_caplog_mp_spawn(caplog_mp_spawn): p = ctx.Process( target=mp_function, name=f"SubProcess{1}", - kwargs={"a": "AAAA", "b": "BBBBB"}, + kwargs={ + "a": "AAAA", + "b": "BBBBB", + "logging_config": LoggingConfig( + pylogging_config_file=os.environ["VLLM_LOGGING_CONFIG_PATH"] + ), + }, ) p.start() p.join() diff --git a/tests/test_pcp_dp.py b/tests/test_pcp_dp.py new file mode 100644 index 0000000000..f6eed57a6f --- /dev/null +++ b/tests/test_pcp_dp.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch + +from vllm.config import ParallelConfig +from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager +from vllm.forward_context import DPMetadata +from vllm.v1.attention.ops.pcp import maybe_gather_mla_latent_cache_inputs + + +@pytest.mark.parametrize( + "pcp_size,sp_size,enable_ep,expected", + [ + (1, 1, True, [5, 7]), + (1, 2, True, [3, 3, 4, 4]), + (2, 1, False, [10, 14]), + (2, 1, True, [5, 5, 7, 7]), + (2, 2, True, [3, 3, 3, 3, 4, 4, 4, 4]), + ], +) +def test_dispatch_sizes_expand_pcp_before_tp(pcp_size, sp_size, enable_ep, expected): + config = ParallelConfig( + distributed_executor_backend="mp", + data_parallel_size=2, + prefill_context_parallel_size=pcp_size, + tensor_parallel_size=sp_size, + enable_expert_parallel=enable_ep, + ) + metadata = DPMetadata.make(config, 5, torch.tensor([5, 7])) + with metadata.sp_local_sizes(sp_size, pcp_size=pcp_size, use_ep=enable_ep) as sizes: + assert sizes == expected + assert metadata.local_sizes is None + assert metadata.num_tokens_across_dp_cpu.tolist() == [5, 7] + + +@pytest.mark.parametrize( + "dp_size,pcp_size,tp_size,use_ep,is_sp,expected", + [ + (2, 2, 1, True, False, "ep"), + (2, 2, 2, True, True, "ep"), + (2, 2, 2, False, False, "dp"), + (2, 1, 2, True, False, "dp"), + (1, 2, 2, True, False, "pcp"), + (2, 2, 2, True, False, None), + ], +) +def test_dispatch_reuses_existing_groups( + dp_size, pcp_size, tp_size, use_ep, is_sp, expected, monkeypatch +): + groups = { + "dp": SimpleNamespace(world_size=dp_size), + "pcp": SimpleNamespace(world_size=pcp_size), + "ep": object(), + } + for name, group in groups.items(): + monkeypatch.setattr( + f"vllm.distributed.device_communicators.all2all.get_{name}_group", + lambda group=group: group, + ) + manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager) + manager.dp_world_size = dp_size + manager.tp_group = SimpleNamespace(world_size=tp_size) + manager.use_ep = use_ep + if expected is None: + with pytest.raises(AssertionError, match="requires sequence-parallel MoE"): + manager._get_comm_group(is_sp) + else: + assert manager._get_comm_group(is_sp) is groups[expected] + + +@pytest.mark.parametrize("enable_ep", [False, True]) +def test_ag_rs_dispatch_and_combine_use_dp_pcp_sizes(monkeypatch, enable_ep): + calls = [] + local_tokens = [20, 21] if enable_ep else [20, 21, 22, 23] + + class FakeGroup: + world_size = 4 if enable_ep else 2 + rank_in_group = 2 if enable_ep else 1 + + def all_gatherv(self, tensors, dim, sizes): + calls.append(("gather", sizes)) + return [torch.tensor([10, 11, 20, 21, 22, 23]) for _ in tensors] + + def reduce_scatterv(self, tensor, dim, sizes): + calls.append(("scatter", sizes)) + return tensor[2 : 2 + len(local_tokens)] + + config = ParallelConfig( + distributed_executor_backend="mp", + data_parallel_size=2, + data_parallel_rank=1, + prefill_context_parallel_size=2, + enable_expert_parallel=enable_ep, + ) + metadata = DPMetadata.make(config, 2, torch.tensor([1, 2])) + manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager) + manager.dp_world_size = 2 + manager.use_ep = enable_ep + + manager.tp_group = SimpleNamespace(world_size=1) + for name in ("dp", "ep"): + monkeypatch.setattr( + f"vllm.distributed.device_communicators.all2all.get_{name}_group", + FakeGroup, + ) + monkeypatch.setattr( + "vllm.distributed.device_communicators.all2all.get_pcp_group", + lambda: SimpleNamespace(world_size=2), + ) + monkeypatch.setattr( + "vllm.distributed.device_communicators.all2all.get_forward_context", + lambda: SimpleNamespace(dp_metadata=metadata), + ) + with metadata.sp_local_sizes(1, pcp_size=2, use_ep=enable_ep): + hidden_states, _, _ = manager.dispatch( + torch.tensor(local_tokens), + torch.ones(len(local_tokens)), + torch.zeros(len(local_tokens)), + ) + combined = manager.combine(hidden_states) + assert combined.tolist() == local_tokens + sizes = [1, 1, 2, 2] if enable_ep else [2, 4] + assert calls == [("gather", sizes), ("scatter", sizes)] + + +@pytest.mark.parametrize("slots", [[3, 4], [3, 4, -1, -1, -1, -1]]) +def test_decode_cache_write_ignores_dp_padding(slots): + kv = torch.arange(12).reshape(6, 2) + pe = torch.arange(6).reshape(6, 1, 1) + slots = torch.tensor(slots) + cache_kv, cache_pe, cache_slots = maybe_gather_mla_latent_cache_inputs( + kv, pe, slots, num_decode_tokens=2, use_pcp=True + ) + torch.testing.assert_close(cache_kv, kv[:2]) + torch.testing.assert_close(cache_pe, pe[:2]) + torch.testing.assert_close(cache_slots, slots[:2]) + + +def test_expanded_slot_mapping_keeps_pcp_prefill_padding(monkeypatch): + calls = [] + + def all_gather(tensor, dim): + calls.append(tensor.shape[0]) + return torch.cat((tensor, tensor), dim=dim) + + monkeypatch.setattr( + "vllm.v1.attention.ops.pcp.get_pcp_group", + lambda: SimpleNamespace(world_size=2, all_gather=all_gather), + ) + kv = torch.zeros(3, 2) # Two decodes followed by one PCP padding row. + pe = torch.zeros(3, 1, 1) + slots = torch.tensor([3, 4, 8, 3, 4, -1]) + cache_kv, _, cache_slots = maybe_gather_mla_latent_cache_inputs( + kv, pe, slots, num_decode_tokens=2, use_pcp=True + ) + assert calls == [1, 1] + assert cache_kv.shape == (4, 2) + assert cache_slots.tolist() == [3, 4, 8, -1] diff --git a/tests/test_sampling_params.py b/tests/test_sampling_params.py index 3c909f4969..98ef59a089 100644 --- a/tests/test_sampling_params.py +++ b/tests/test_sampling_params.py @@ -1,11 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass +from types import SimpleNamespace import pytest from vllm import SamplingParams +from vllm.config.diffusion import DiffusionConfig from vllm.exceptions import VLLMValidationError +from vllm.utils.diffusion import validate_diffusion_sampling_params +from vllm.v1.engine.input_processor import InputProcessor @dataclass @@ -13,6 +17,7 @@ class MockModelConfig: is_diffusion: bool = False max_logprobs: int = 20 logits_processors: list | None = None + return_sampling_mask: bool = False def get_vocab_size(self) -> int: return 1024 @@ -59,6 +64,206 @@ def test_verify_leaves_logits_processors_to_admission(): ) +def _verify_diffusion(params: SamplingParams, canvas_length: int | None = None): + model_config = MockModelConfig(is_diffusion=True) + params.verify(model_config, None, None, None) + validate_diffusion_sampling_params( + params, + canvas_length=canvas_length, + vocab_size=model_config.get_vocab_size(), + async_scheduling=True, + ) + + +def test_diffusion_extra_args_are_validated_without_a_served_canvas(): + # No --diffusion-config: the canvas is unknown, the ids are still checked + # and a read-only request is still normalised. + params = SamplingParams( + max_tokens=64, + extra_args={"diffusion_seed_canvas": [0, 1], "diffusion_read_only": True}, + ) + _verify_diffusion(params, canvas_length=None) + assert params.ignore_eos is True + + bad = SamplingParams(extra_args={"diffusion_seed_canvas": [0, 10**9]}) + with pytest.raises(VLLMValidationError, match="ids must be in"): + _verify_diffusion(bad, canvas_length=None) + + +@pytest.mark.parametrize("async_scheduling", [False, True]) +@pytest.mark.parametrize( + "extra_args", + [ + {}, + {"diffusion_canvas_length": None}, + {"diffusion_canvas_length": 4}, + {"diffusion_canvas_length": 8}, + ], +) +def test_narrow_diffusion_canvas_requires_async_scheduling( + async_scheduling, extra_args +): + processor = SimpleNamespace( + model_config=MockModelConfig(is_diffusion=True), + vllm_config=SimpleNamespace( + scheduler_config=SimpleNamespace(async_scheduling=async_scheduling) + ), + speculative_config=None, + structured_outputs_config=None, + diffusion_config=DiffusionConfig(canvas_length=8), + tokenizer=None, + validate_logits_processors_params=lambda params: None, + ) + params = SamplingParams(extra_args=extra_args) + if not async_scheduling and extra_args.get("diffusion_canvas_length") == 4: + with pytest.raises(VLLMValidationError, match="requires --async-scheduling"): + InputProcessor._validate_params(processor, params, ("generate",)) + else: + InputProcessor._validate_params(processor, params, ("generate",)) + + +@pytest.mark.parametrize( + "extra_args, match", + [ + ({"diffusion_seed_canvas": "abc"}, "list of token ids"), + ({"diffusion_seed_canvas": [1, 2.5]}, "list of token ids"), + ({"diffusion_seed_canvas": [1, True]}, "list of token ids"), + ({"diffusion_seed_canvas": [0, 1024]}, r"in \[0, 1024\)"), + ({"diffusion_seed_canvas": [0, -1]}, r"in \[0, 1024\)"), + ({"diffusion_max_steps": 0}, "positive integer"), + ({"diffusion_max_steps": "1"}, "positive integer"), + ({"diffusion_max_steps": True}, "positive integer"), + ({"diffusion_read_only": "yes"}, "boolean"), + ({"diffusion_read_only": 2}, "boolean"), + ({"diffusion_pinned": "0,1"}, "list of canvas positions"), + ({"diffusion_pinned": [0, True]}, "list of canvas positions"), + ({"diffusion_pinned": [0]}, "needs a diffusion_seed_canvas"), + ], +) +def test_diffusion_rejects_bad_extra_args(extra_args: dict, match: str): + with pytest.raises(VLLMValidationError, match=match): + _verify_diffusion(SamplingParams(extra_args=extra_args)) + + +@pytest.mark.parametrize("flag", [True, 1]) +def test_diffusion_constrained_needs_logprob_token_ids(flag): + bad = SamplingParams(extra_args={"diffusion_constrained": flag}) + with pytest.raises(VLLMValidationError, match="needs logprob_token_ids"): + _verify_diffusion(bad) + + ok = SamplingParams( + logprob_token_ids=[3, 5], extra_args={"diffusion_constrained": flag} + ) + _verify_diffusion(ok) + + # An unset or false flag needs no ids. + _verify_diffusion(SamplingParams(extra_args={"diffusion_constrained": False})) + _verify_diffusion(SamplingParams(extra_args={"diffusion_constrained": 0})) + + +@pytest.mark.parametrize("value", ["yes", 2, [1]]) +def test_diffusion_constrained_must_be_a_bool(value): + params = SamplingParams(extra_args={"diffusion_constrained": value}) + with pytest.raises(VLLMValidationError, match="must be a boolean"): + _verify_diffusion(params) + + +def test_diffusion_seed_canvas_must_fill_the_canvas(): + params = SamplingParams(extra_args={"diffusion_seed_canvas": [0] * 7}) + with pytest.raises(VLLMValidationError, match="exactly 8 ids, got 7"): + _verify_diffusion(params, canvas_length=8) + # Without the served diffusion config the canvas length is unknown. + _verify_diffusion(params) + + +@pytest.mark.parametrize("max_tokens, expected", [(100, 8), (5, 5), (None, 8)]) +@pytest.mark.parametrize("flag", [True, 1]) +def test_diffusion_read_only_ends_after_one_canvas(max_tokens, expected, flag): + params = SamplingParams( + max_tokens=max_tokens, extra_args={"diffusion_read_only": flag} + ) + _verify_diffusion(params, canvas_length=8) + assert params.max_tokens == expected + assert params.ignore_eos + + +@pytest.mark.parametrize( + "width, match", + [ + (0, "positive integer"), + ("4", "positive integer"), + (True, "positive integer"), + (9, "no larger"), + ], +) +def test_diffusion_rejects_bad_canvas_length(width, match): + with pytest.raises(VLLMValidationError, match=match): + _verify_diffusion( + SamplingParams(extra_args={"diffusion_canvas_length": width}), + canvas_length=8, + ) + + +def test_diffusion_canvas_length_sizes_the_seed_and_the_read(): + params = SamplingParams( + max_tokens=100, + extra_args={ + "diffusion_canvas_length": 4, + "diffusion_seed_canvas": [1, 2, 3, 4], + "diffusion_read_only": True, + }, + ) + _verify_diffusion(params, canvas_length=8) + assert params.max_tokens == 4 + + params = SamplingParams( + extra_args={"diffusion_canvas_length": 4, "diffusion_seed_canvas": [0] * 8} + ) + with pytest.raises(VLLMValidationError, match="exactly 4 ids, got 8"): + _verify_diffusion(params, canvas_length=8) + + +def test_diffusion_pinned_positions_stay_inside_the_canvas(): + seed = [0] * 8 + with pytest.raises(VLLMValidationError, match="inside the canvas"): + _verify_diffusion( + SamplingParams( + extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [7, 8]} + ), + canvas_length=8, + ) + with pytest.raises(VLLMValidationError, match="inside the canvas"): + _verify_diffusion( + SamplingParams( + extra_args={"diffusion_seed_canvas": seed, "diffusion_pinned": [-1]} + ) + ) + # A narrower request canvas bounds the positions. + with pytest.raises(VLLMValidationError, match="inside the canvas"): + _verify_diffusion( + SamplingParams( + extra_args={ + "diffusion_canvas_length": 4, + "diffusion_seed_canvas": seed[:4], + "diffusion_pinned": [4], + } + ), + canvas_length=8, + ) + + +def test_diffusion_accepts_extra_args(): + params = SamplingParams( + extra_args={ + "diffusion_seed_canvas": list(range(8)), + "diffusion_pinned": [0, 1, 7], + "diffusion_max_steps": 4, + "diffusion_read_only": True, + } + ) + _verify_diffusion(params, canvas_length=8) + + @pytest.mark.parametrize("value", [-(2**63) - 1, 2**64]) def test_extra_args_rejects_nested_integer_overflow(value): """Reject extension values before they reach the engine transport.""" diff --git a/tests/tokenizers_/fixtures/deepseek_v41/test_output_1.txt b/tests/tokenizers_/fixtures/deepseek_v41/test_output_1.txt index 0897086688..a41d88e924 100644 --- a/tests/tokenizers_/fixtures/deepseek_v41/test_output_1.txt +++ b/tests/tokenizers_/fixtures/deepseek_v41/test_output_1.txt @@ -1,4 +1,4 @@ -<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) You are a helpful assistant. diff --git a/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt b/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt index 4e93ce2cbb..fa481ca5cd 100644 --- a/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt +++ b/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt @@ -1,3 +1,3 @@ -<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) You are a helpful assistant.<|User|>Hello<|Assistant|>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|>The user asks about the capital of France. It is Paris.The capital of France is Paris.<|end▁of▁sentence|> \ No newline at end of file diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index 4da6381a0b..4a1a2b1560 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object): def test_tokenizer_like_protocol(): - tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True) + tokenizer = get_tokenizer("openai-community/gpt2") assert isinstance(tokenizer, TokenizersBackend) _assert_tokenizer_like(tokenizer) diff --git a/tests/tokenizers_/test_deepseek_v41.py b/tests/tokenizers_/test_deepseek_v41.py index 13c0375fbb..10c66f23b0 100644 --- a/tests/tokenizers_/test_deepseek_v41.py +++ b/tests/tokenizers_/test_deepseek_v41.py @@ -35,9 +35,9 @@ def test_reference_encoder_fixtures(case_id): @pytest.mark.parametrize( ("effort", "budget"), [ - (None, 50), - ("low", 25), - ("high", 50), + (None, 75), + ("low", 50), + ("high", 75), ("xhigh", 75), ("max", 100), (1, 1), @@ -138,7 +138,7 @@ def test_top_level_effort_overrides_template_effort(): chat_template_kwargs={"reasoning_effort": 100}, ) kwargs = request.build_chat_params(None, "auto").get_apply_chat_template_kwargs() - assert "Reasoning Effort: 25 " in render(request.messages, **kwargs) + assert "Reasoning Effort: 50 " in render(request.messages, **kwargs) def test_encode_uses_one_bos_and_forwards_truncation(): diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index b7201c1cfe..a4ee4a755e 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -69,21 +69,11 @@ def test_resolve_tokenizer_args_idempotent(runner_type): ) -@pytest.mark.parametrize( - ("tokenizer_mode", "input_kwargs"), - [ - ("hf", {}), - ("hf", {"mistral_format": False}), - ("slow", {}), - ("slow", {"mistral_format": False}), - ], -) -def test_resolve_tokenizer_args_forces_hf_mistral_format_false( - tokenizer_mode, input_kwargs -): +@pytest.mark.parametrize("input_kwargs", [{}, {"mistral_format": False}]) +def test_resolve_tokenizer_args_forces_hf_mistral_format_false(input_kwargs): resolved_mode, _, _, kwargs = resolve_tokenizer_args( "mistralai/Mistral-Nemo-Instruct-2407", - tokenizer_mode=tokenizer_mode, + tokenizer_mode="hf", **input_kwargs, ) @@ -91,15 +81,14 @@ def test_resolve_tokenizer_args_forces_hf_mistral_format_false( assert kwargs["mistral_format"] is False -@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"]) -def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode): +def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(): with pytest.raises( ValueError, match="mistral_format=True is not supported with tokenizer_mode='hf'", ): resolve_tokenizer_args( "mistralai/Mistral-Nemo-Instruct-2407", - tokenizer_mode=tokenizer_mode, + tokenizer_mode="hf", mistral_format=True, ) diff --git a/tests/tool_parsers/test_granite_tool_parser.py b/tests/tool_parsers/test_granite_tool_parser.py index af3386112f..5c51f389be 100644 --- a/tests/tool_parsers/test_granite_tool_parser.py +++ b/tests/tool_parsers/test_granite_tool_parser.py @@ -16,7 +16,7 @@ from tests.tool_parsers.utils import ( split_string_into_token_deltas, ) from vllm.tokenizers import get_tokenizer -from vllm.tool_parsers.granite_tool_parser import GraniteToolParser +from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser class TestGraniteToolParser(ToolParserTests): @@ -76,23 +76,6 @@ I'll get that information.""", single_tool_call_expected_content=None, parallel_tool_calls_count=2, parallel_tool_calls_names=["get_weather", "get_time"], - # xfail markers - xfail_streaming={ - "test_malformed_input": ( - "Streaming mode incorrectly creates tool call from malformed JSON" - ), - "test_surrounding_text": ( - "Parser doesn't handle surrounding text correctly in streaming" - ), - "test_streaming_reconstruction": ( - "Streaming mode doesn't strip <|tool_call|> marker from content" - ), - }, - xfail_nonstreaming={ - "test_surrounding_text": ( - "Parser doesn't handle surrounding text correctly in non-streaming" - ), - }, ) # Granite-Specific Tests @@ -140,7 +123,7 @@ def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size): parallel calls must not drop the first call's name. granite streams arguments before name, so the name only completes as the next call appears. """ - parser = GraniteToolParser(granite_tokenizer) + parser = GraniteEngineToolParser(granite_tokenizer) model_output = ( '<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, ' '{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]' diff --git a/tests/tool_use/test_responses_request_validations.py b/tests/tool_use/test_responses_request_validations.py index b50482960c..caeb7042d6 100644 --- a/tests/tool_use/test_responses_request_validations.py +++ b/tests/tool_use/test_responses_request_validations.py @@ -187,6 +187,27 @@ def test_responses_request_empty_tools_named_tool_choice(): ) +@pytest.mark.parametrize("detail", [None, "low"], ids=["default", "explicit"]) +def test_input_image_detail_defaults_or_preserves(detail: str | None) -> None: + image = { + "type": "input_image", + "image_url": "https://example.com/image.png", + } + if detail is not None: + image["detail"] = detail + + item = { + "role": "user", + "content": [image], + } + + request = ResponsesRequest.model_validate({"input": [item]}) + + assert request.input[0]["content"][0]["detail"] == ( + "auto" if detail is None else detail + ) + + # Regression tests for parallel_tool_calls=null crash in Responses API # (from_request() passed None to ResponsesResponse.parallel_tool_calls, # a non-optional bool field, causing a Pydantic ValidationError during diff --git a/tests/tools/test_docker_build_metadata_args.py b/tests/tools/test_docker_build_metadata_args.py index 127768afc7..d7b6579088 100644 --- a/tests/tools/test_docker_build_metadata_args.py +++ b/tests/tools/test_docker_build_metadata_args.py @@ -177,7 +177,7 @@ def test_rust_build_cache_excludes_git_metadata() -> None: assert "source=.git,target=.git" not in cached_run assert "source=.git,target=.git" in exact_version_stage assert 'SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}"' in exact_version_stage - assert "bash build_rust.sh" in exact_version_stage + assert "bash tools/build_rust.sh" in exact_version_stage def test_rocm_ci_base_bake_embeds_content_hash_label() -> None: diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 229abfba8c..19773dfda4 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -5,13 +5,14 @@ only get the `eos_token_id` from the tokenizer as defined by `BaseRenderer.get_eos_token_id`. """ +import json import math from types import SimpleNamespace from typing import cast from unittest.mock import MagicMock, patch import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.model import ModelConfig from vllm.tokenizers import get_tokenizer @@ -31,6 +32,50 @@ from vllm.transformers_utils.configs.glm5_next import ( from vllm.transformers_utils.configs.mistral import adapt_config_dict +@pytest.mark.parametrize("layout", ["mixed", "flat"]) +def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout): + """Remove redundant shared entries while preserving per-layer and flat RoPE.""" + from transformers import Gemma4TextConfig + + per_layer = { + "full_attention": { + "rope_type": "proportional", + "partial_rotary_factor": 0.25, + "rope_theta": 1000000.0, + }, + "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, + } + rope_parameters: dict[str, object] = dict(per_layer) + if layout == "mixed": + rope_parameters.update(rope_type="default", rope_theta=None) + else: + rope_parameters = {"rope_type": "default", "rope_theta": 12345.0} + (tmp_path / "config.json").write_text( + json.dumps( + { + "model_type": "gemma4_text", + "architectures": ["Gemma4DSparkModel"], + "num_hidden_layers": 1, + "layer_types": ["full_attention"], + "rope_parameters": rope_parameters, + } + ) + ) + _, config = config_module.HFConfigParser().parse( + tmp_path, trust_remote_code=False, max_position_embeddings=8192 + ) + assert isinstance(config, Gemma4TextConfig) + assert config.name_or_path == str(tmp_path) + assert config.max_position_embeddings == 8192 + if layout == "flat": + assert config.rope_parameters["rope_theta"] == 12345.0 + else: + for layer_type, expected in per_layer.items(): + for key, value in expected.items(): + assert config.rope_parameters[layer_type][key] == value + assert set(config.rope_parameters) == set(per_layer) + + def test_patch_legacy_rope_type_preserves_nope_layers(): """NoPE layers stay disabled while later RoPE layers are normalized.""" rope_parameters = { @@ -208,6 +253,7 @@ def test_model_config_generation_fallback_forwards_code_revision(): model="org/model", trust_remote_code=True, revision="model-pin", + _hf_config_revision=None, code_revision="code-pin", config_format="auto", hf_token=None, @@ -223,7 +269,7 @@ def test_model_config_generation_fallback_forwards_code_revision(): patch.object( config_module, "get_config", - return_value=PretrainedConfig(), + return_value=PreTrainedConfig(), ) as get_config, ): ModelConfig.try_get_generation_config(model_config) @@ -269,7 +315,7 @@ def test_safetensors_metadata_of_repo_without_safetensors(): ], ) def test_mrope_num_dims(section_key, mrope_section, expected_num_dims): - config = PretrainedConfig() + config = PreTrainedConfig() config.rope_parameters = {"rope_type": "default", section_key: mrope_section} assert uses_mrope(config) @@ -280,7 +326,7 @@ def test_mrope_num_dims(section_key, mrope_section, expected_num_dims): def test_mrope_num_dims_from_config_attribute(section_name): """Some configs expose the section as an attribute rather than under `rope_parameters`.""" - config = PretrainedConfig() + config = PreTrainedConfig() setattr(config, section_name, [16, 16, 16, 16]) assert uses_mrope(config) @@ -289,7 +335,7 @@ def test_mrope_num_dims_from_config_attribute(section_name): def test_mrope_num_dims_from_nested_rope_parameters(): """Sections nested by layer type must be found, not silently defaulted.""" - config = PretrainedConfig() + config = PreTrainedConfig() config.rope_parameters = { "full_attention": {"mrope_section": [16, 16, 16, 16]}, "linear_attention": {"rope_type": "default"}, @@ -300,4 +346,4 @@ def test_mrope_num_dims_from_nested_rope_parameters(): def test_mrope_num_dims_without_mrope(): - assert mrope_num_dims(PretrainedConfig()) == 0 + assert mrope_num_dims(PreTrainedConfig()) == 0 diff --git a/tests/transformers_utils/test_config_parser_registry.py b/tests/transformers_utils/test_config_parser_registry.py index 0931bd734f..41dcf40782 100644 --- a/tests/transformers_utils/test_config_parser_registry.py +++ b/tests/transformers_utils/test_config_parser_registry.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.transformers_utils.config import get_config_parser, register_config_parser from vllm.transformers_utils.config_parser_base import ConfigParserBase @@ -19,7 +19,7 @@ class CustomConfigParser(ConfigParserBase): revision: str | None = None, code_revision: str | None = None, **kwargs, - ) -> tuple[dict, PretrainedConfig]: + ) -> tuple[dict, PreTrainedConfig]: raise NotImplementedError diff --git a/tests/transformers_utils/test_hf_overrides_model_type.py b/tests/transformers_utils/test_hf_overrides_model_type.py index b739b4294a..71356edee0 100644 --- a/tests/transformers_utils/test_hf_overrides_model_type.py +++ b/tests/transformers_utils/test_hf_overrides_model_type.py @@ -5,12 +5,12 @@ import json import tempfile -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.transformers_utils.config import _CONFIG_REGISTRY, get_config -class _TestCustomConfig(PretrainedConfig): +class _TestCustomConfig(PreTrainedConfig): model_type = "test_custom_model" def __init__(self, custom_attr=42, **kw): diff --git a/tests/transformers_utils/test_speculators_override.py b/tests/transformers_utils/test_speculators_override.py index 35330f327f..07f607cb88 100644 --- a/tests/transformers_utils/test_speculators_override.py +++ b/tests/transformers_utils/test_speculators_override.py @@ -40,7 +40,7 @@ _SPECULATORS_CONFIG_DICT = { def _patched_get_config_dict(config_dict): """Return a patch context that bypasses HF Hub for get_config_dict.""" return patch( - "vllm.transformers_utils.config.PretrainedConfig.get_config_dict", + "vllm.transformers_utils.config.PreTrainedConfig.get_config_dict", return_value=(config_dict, {}), ) diff --git a/tests/utils.py b/tests/utils.py index 399a741df0..10fcffffea 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import asyncio import atexit import contextlib import copy @@ -23,7 +22,6 @@ from contextlib import ExitStack, contextmanager from multiprocessing import Process, get_context from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, cast -from unittest.mock import patch import anthropic import cloudpickle @@ -33,12 +31,9 @@ import pytest import requests import torch import torch.nn.functional as F -from huggingface_hub.constants import HF_HUB_OFFLINE -from openai.types.completion import Completion from typing_extensions import ParamSpec import vllm.envs as envs -from tests.models.utils import TextTextLogprobs from vllm.distributed import ( ensure_model_parallel_initialized, init_distributed_environment, @@ -52,7 +47,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.model_executor.model_loader import get_model_loader from vllm.platforms import current_platform from vllm.tokenizers import get_tokenizer -from vllm.transformers_utils.repo_utils import hf_api from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.mem_constants import GB_bytes from vllm.utils.network_utils import get_open_port @@ -69,24 +63,6 @@ logger = init_logger(__name__) FP8_DTYPE = current_platform.fp8_dtype() -def prewarm_hf_cache(assets: list[tuple[str, str]]) -> None: - """Pre-populate the HF cache for (repo_id, filename) pairs that upstream - trust_remote_code modules would otherwise fetch from third-party CDNs - (often unreachable from US-based CI).""" - if HF_HUB_OFFLINE: - return - for repo_id, filename in assets: - try: - hf_api().hf_hub_download(repo_id=repo_id, filename=filename) - except Exception as e: - logger.warning( - "Failed to prefetch %s/%s: %r. Tests depending on this asset may fail.", - repo_id, - filename, - e, - ) - - if current_platform.is_cuda(): from vllm.third_party.pynvml import ( nvmlDeviceGetHandleByIndex, @@ -1556,17 +1532,6 @@ def assert_rocm_custom_allreduce_backend_state_on_worker( ) -@contextmanager -def error_on_warning(category: type[Warning] = Warning): - """Within the scope of this context manager, tests will fail if any warning - of the given category is emitted. - """ - with warnings.catch_warnings(): - warnings.filterwarnings("error", category=category) - - yield - - def get_physical_device_indices(devices: list[int]): visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") if visible_devices is None: @@ -1928,7 +1893,7 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None] The child inherits the parent's stdout/stderr so its output (engine cores, NCCL, CUDA, ...) reaches the test runner live; the Python-level - traceback is serialized to ``tb_file`` for structured re-raising. A + traceback or skip reason is serialized to ``tb_file`` for propagation. A native crash leaves ``tb_file`` empty — the diagnostic is then only in the inherited subprocess output. """ @@ -1953,7 +1918,7 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None] ) child_script = ( - "import sys, importlib, cloudpickle, traceback\n" + "import sys, importlib, cloudpickle, traceback, json\n" "try:\n" " from _pytest.outcomes import Skipped\n" "except ImportError:\n" @@ -1965,7 +1930,9 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None] " target = getattr(target, name)\n" "try:\n" " target(*data['args'], **data['kwargs'])\n" - "except Skipped:\n" + "except Skipped as exc:\n" + " with open(data['tb_file'], 'w') as fp:\n" + " json.dump(str(exc), fp)\n" " sys.exit(0)\n" "except BaseException:\n" " with open(data['tb_file'], 'w') as fp:\n" @@ -1984,18 +1951,20 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None] env=env, ) + try: + with open(tb_file) as fp: + tb = fp.read() + except OSError: + tb = "" if result.returncode != 0: - try: - with open(tb_file) as fp: - tb = fp.read() - except OSError: - tb = "" if not tb: tb = "" raise RuntimeError( f"Test subprocess '{f.__name__}' failed " f"({_format_subprocess_exit(result.returncode)}):\n{tb}" ) + if tb: + pytest.skip(json.loads(tb)) finally: with contextlib.suppress(OSError): os.remove(tb_file) @@ -2153,88 +2122,6 @@ def multi_gpu_only(*, num_gpus: int = 2): return wrapper -async def completions_with_server_args( - prompts: list[str], - model_name: str, - server_cli_args: list[str], - num_logprobs: int | None, - max_wait_seconds: int = 240, - max_tokens: int | list = 5, -) -> list[Completion]: - """Construct a remote OpenAI server, obtain an async client to the - server & invoke the completions API to obtain completions. - - Args: - prompts: test prompts - model_name: model to spin up on the vLLM server - server_cli_args: CLI args for starting the server - num_logprobs: Number of logprobs to report (or `None`) - max_wait_seconds: timeout interval for bringing up server. - Default: 240sec - max_tokens: max_tokens value for each of the given input prompts. - if only one max_token value is given, the same value is used - for all the prompts. - - Returns: - OpenAI Completion instance - - """ - if isinstance(max_tokens, int): - max_tokens = [max_tokens] * len(prompts) - - assert len(max_tokens) == len(prompts) - - outputs = None - with RemoteOpenAIServer( - model_name, server_cli_args, max_wait_seconds=max_wait_seconds - ) as server: - client = server.get_async_client() - outputs = [ - client.completions.create( - model=model_name, - prompt=[p], - temperature=0, - stream=False, - max_tokens=max_tok, - logprobs=num_logprobs, - ) - for p, max_tok in zip(prompts, max_tokens) - ] - outputs = await asyncio.gather(*outputs) - - assert outputs is not None, "Completion API call failed." - - return outputs - - -def get_client_text_generations(completions: list[Completion]) -> list[str]: - """Extract generated tokens from the output of a - request made to an Open-AI-protocol completions endpoint. - """ - assert all([len(x.choices) == 1 for x in completions]) - return [x.choices[0].text for x in completions] - - -def get_client_text_logprob_generations( - completions: list[Completion], -) -> list[TextTextLogprobs]: - """Operates on the output of a request made to an Open-AI-protocol - completions endpoint; obtains top-rank logprobs for each token in - each {class}`SequenceGroup` - """ - text_generations = get_client_text_generations(completions) - text = "".join(text_generations) - return [ - ( - text_generations, - text, - (None if x.logprobs is None else x.logprobs.top_logprobs), - ) - for completion in completions - for x in completion.choices - ] - - def has_module_attribute(module_name, attribute_name): """Helper function to check if a module has a specific attribute.""" try: @@ -2263,50 +2150,6 @@ def get_attn_backend_list_based_on_platform() -> list[str]: raise ValueError("Unsupported platform") -@contextmanager -def override_cutlass_fp8_supported(value: bool): - with patch( - "vllm.model_executor.layers.quantization.utils.w8a8_utils.cutlass_fp8_supported", - return_value=value, - ): - yield - - -def disable_aiter_plain_rmsnorm(monkeypatch) -> None: - """Patch dispatch_rocm_rmsnorm_func so the plain (non-fused) rms_norm path - always uses the native float32 kernel for the duration of a test. - - The fused path (rms_norm2d_with_add, selected when with_fused_add=True) is - left on AITER -- only the plain path is redirected to native. - - AITER's plain rms_norm accumulates variance in bfloat16 (~1 ULP/call), - which drifts the KV cache over many decode steps. This drift is irrelevant - for a trained model (rank-1/rank-2 gap ~1-3 nats >> 1 ULP), but breaks - logprob comparison tests with randomly-initialised models like - TitanML/tiny-mixtral whose rank-1/rank-2 gap is only O(1/sqrt(V)) ~0.006 - nats -- smaller than the accumulated per-step error. - """ - import torch - - import vllm.model_executor.layers.layernorm as _ln_mod - from vllm.model_executor.layers.layernorm import rms_norm as _native - - _orig = _ln_mod.dispatch_rocm_rmsnorm_func - - def _native_plain( - with_fused_add: bool, dtype: torch.dtype, use_aiter: bool = False - ): - if ( - use_aiter - and not with_fused_add - and dtype in (torch.float16, torch.bfloat16) - ): - return _native - return _orig(with_fused_add, dtype, use_aiter) - - monkeypatch.setattr(_ln_mod, "dispatch_rocm_rmsnorm_func", _native_plain) - - def prep_prompts(batch_size: int, ln_range: tuple[int, int] = (800, 1100)): """Generate prompts which a bunch of assignments, then asking for the value of one of them. diff --git a/tests/utils_/test_spawn_decorator.py b/tests/utils_/test_spawn_decorator.py index 1740ea30de..d92d589ff1 100644 --- a/tests/utils_/test_spawn_decorator.py +++ b/tests/utils_/test_spawn_decorator.py @@ -2,10 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for spawn_new_process_for_each_test decorator.""" +from pathlib import Path + import pytest from tests.utils import spawn_new_process_for_each_test +pytest_plugins = ["pytester"] + @spawn_new_process_for_each_test def test_spawn_decorator_passing(): @@ -26,6 +30,15 @@ def test_spawn_decorator_skip(): pytest.skip("intentional skip") +def test_spawn_decorator_reports_skip(pytester): + """A child skip must remain a skip in the parent pytest report.""" + result = pytester.runpytest_subprocess( + f"{Path(__file__).resolve()}::test_spawn_decorator_skip", "-rs" + ) + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines(["SKIPPED *intentional skip"]) + + @spawn_new_process_for_each_test @pytest.mark.parametrize("x,y,expected", [(1, 2, 3), (0, 0, 0)]) def test_spawn_decorator_parametrized(x, y, expected): diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index b02c85f682..517184c0f1 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for v1 attention backends without GPUModelRunner dependency.""" +from dataclasses import replace from functools import partial from types import SimpleNamespace import pytest import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention +from transformers import LlamaConfig from tests.v1.attention.utils import ( BatchSpec, @@ -28,8 +30,10 @@ from vllm.utils.torch_utils import ( ) from vllm.v1.attention.backend import ( AttentionCGSupport, + AttentionMetadataBuilder, AttentionType, CommonAttentionMetadata, + max_decode_query_len, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout @@ -938,8 +942,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa impl.o_sf_scale = None impl.window_left = -1 impl.sinks = None - impl.cache_config = unittest.mock.Mock() - impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC layer = unittest.mock.Mock( _q_scale=torch.tensor(1.0), @@ -955,6 +957,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa q_len_per_req=1, ) attn_metadata = flashinfer_backend.FlashInferMetadata( + kv_cache_layout=KVCacheLayout.LBHNC, num_actual_tokens=2, slot_mapping=torch.empty(0, dtype=torch.int64), q_data_type_prefill=torch.bfloat16, @@ -1066,6 +1069,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets( assert q_cu_seq_lens.tolist() == qo_indptr_values +@pytest.mark.skipif( + AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, + reason="FlashInfer is not available.", +) +@pytest.mark.parametrize( + "adaptive,decode_kernel,dcp_size,expected_bound", + [ + (None, "TRTLLM_GEN", 1, None), + (False, "TRTLLM_GEN", 1, None), + (True, "TRTLLM_GEN", 1, 8), + (True, "XQA", 1, None), + (True, "TRTLLM_GEN", 2, None), + ], +) +def test_flashinfer_varlen_cudagraph_capability( + monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound +): + """Only the trtllm-gen path that adaptive verification enables replays + varlen decode graphs, up to the decode width; the uniform level is fixed.""" + from vllm.model_executor.layers.attention.chunked_local_attention import ( + create_chunked_local_attention_backend, + ) + from vllm.v1.attention.backends import flashinfer as fi + + config = SimpleNamespace( + attention_config=SimpleNamespace(use_non_causal=False), + speculative_config=SimpleNamespace( + enable_adaptive_verification=adaptive, + num_speculative_tokens=7, + parallel_drafting=True, + ) + if adaptive is not None + else None, + parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size), + model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32), + use_v2_model_runner=True, + ) + monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True) + monkeypatch.setattr( + fi.FlashInferMetadataBuilder, + "_get_flashinfer_trtllm_api_decode_kernel", + staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]), + ) + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.bfloat16, + ) + builder_cls = fi.FlashInferMetadataBuilder + assert builder_cls.get_cudagraph_support(config, spec) == ( + AttentionCGSupport.UNIFORM_BATCH + if dcp_size == 1 + else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + ) + bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec) + assert bound == expected_bound + if bound is not None: + # A wrapper forcing NEVER inherits the override but not the bound. + chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16) + wrapped_cls = chunked_local.get_builder_cls() + assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None + + +@pytest.mark.parametrize( + "use_v2_model_runner,parallel_drafting,expected", + [ + (True, None, 1), + (True, False, 8), + (True, True, 8), + (False, False, 8), + (False, True, 15), + ], +) +def test_spec_as_decode_width_follows_model_runner( + use_v2_model_runner, parallel_drafting, expected +): + """Only model runner V1's parallel drafter widens decode requests.""" + + class Builder(AttentionMetadataBuilder): + def __init__(self, vllm_config): + self.vllm_config = vllm_config + + def build(self, common_prefix_len, common_attn_metadata, fast_build=False): + raise NotImplementedError + + config = SimpleNamespace( + speculative_config=None + if parallel_drafting is None + else SimpleNamespace( + num_speculative_tokens=7, parallel_drafting=parallel_drafting + ), + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + use_v2_model_runner=use_v2_model_runner, + ) + assert max_decode_query_len(config) == expected + builder = Builder(config) + builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True) + assert builder.reorder_batch_threshold == expected + + +@pytest.mark.skipif( + AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST + or not current_platform.is_device_capability_family(100), + reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs", +) +@pytest.mark.parametrize( + "head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads", + [ + pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"), + pytest.param( + 256, + 1024, + 40, + 32, + 16, + 8, + id="gemma4-sliding-gqa", + ), + pytest.param( + 512, + None, + 40, + 32, + 16, + 1, + id="gemma4-global-mqa", + ), + ], +) +def test_flashinfer_varlen_decode_graph_replays_changed_layout( + tmp_path, + monkeypatch, + head_size, + sliding_window, + num_tokens, + num_reqs, + num_heads, + num_kv_heads, +): + """Replay changes device offsets, active slots, context, and KV writes. + + The graph is captured with evenly spread dummy queries and an explicit + bound of eight. CPU offsets then remain stale while device offsets change. + Compare real output rows with independent FP32 causal/windowed attention, + and verify cache writes including preservation of the null padding block. + """ + from vllm.v1.attention.backends import flashinfer as fi + from vllm.v1.attention.backends.utils import PerLayerParameters + + set_random_seed(0) + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + block_size = 16 + dtype = torch.bfloat16 + LlamaConfig( + architectures=["LlamaForCausalLM"], + hidden_size=num_heads * head_size, + head_dim=head_size, + num_attention_heads=num_heads, + num_key_value_heads=num_kv_heads, + num_hidden_layers=1, + max_position_embeddings=4096, + ).save_pretrained(tmp_path) + num_blocks = 1 + num_reqs * cdiv(2112, block_size) + config = create_vllm_config( + model_name=str(tmp_path), + max_model_len=4096, + dtype=dtype, + num_gpu_blocks=num_blocks, + block_size=block_size, + max_num_seqs=32, + max_num_batched_tokens=256, + ) + config.cache_config.kv_cache_layout = "BLHNC" + config.speculative_config = SimpleNamespace( + num_speculative_tokens=7, + enable_adaptive_verification=True, + parallel_drafting=False, + ) + spec = create_standard_kv_cache_spec(config) + scale = head_size**-0.5 + monkeypatch.setattr( + fi, + "get_per_layer_parameters", + lambda *_: { + "layer": PerLayerParameters( + window_left=-1 if sliding_window is None else sliding_window - 1, + logits_soft_cap=0.0, + sm_scale=scale, + has_sinks=False, + ), + }, + ) + device = torch.device("cuda") + layout = KVCacheLayout.BLHNC + query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device) + key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device) + value = torch.empty_like(key) + output = torch.empty_like(query) + dummy_lens = [ + num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs) + ] + common = create_common_attn_metadata( + BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens), + block_size, + device, + ) + common.max_query_len = 8 + cache = None + + def prepare(query_lens): + nonlocal cache + query_lens = query_lens + [0] * (num_reqs - len(query_lens)) + query.normal_() + key.normal_() + value.normal_() + # Alternating short prefills, nonempty decode contexts, and contexts + # beyond the sliding window; inactive requests have zero sequence length. + contexts = [ + ([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens) + ] + ks = [ + torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device) + for c in contexts + ] + vs = [torch.randn_like(k) for k in ks] + actual = create_common_attn_metadata( + BatchSpec( + seq_lens=[c + q for c, q in zip(contexts, query_lens)], + query_lens=query_lens, + ), + block_size, + device, + ) + fresh_cache = create_and_prepopulate_kv_cache( + ks, + vs, + block_size, + num_kv_heads, + head_size, + dtype, + device, + num_blocks, + actual, + layout, + randomize_blocks=True, + ) + if cache is None: + cache = fresh_cache + else: + cache.copy_(fresh_cache) + common.query_start_loc.copy_(actual.query_start_loc) + common.seq_lens.copy_(actual.seq_lens) + common.block_table_tensor.zero_() + common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_( + actual.block_table_tensor + ) + common.slot_mapping.fill_(-1) + common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping) + return contexts, ks, vs + + prepare(dummy_lens) + with set_current_vllm_config(config): + builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device) + metadata = builder.build(0, common) + assert metadata.num_prefills == 0 + assert metadata.decode.q_len_per_req == 8 + impl = fi.FlashInferImpl( + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=head_size, + scale=scale, + alibi_slopes=None, + sliding_window=sliding_window, + kv_cache_dtype="auto", + attn_type=AttentionType.DECODER, + ) + layer = MockAttentionLayer(device) + + def forward(attn_metadata=metadata): + impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping) + impl.forward(layer, query, key, value, cache, attn_metadata, output=output) + + for _ in range(3): + forward() + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + forward() + + layouts = [[8], [1] * num_reqs] + if num_tokens >= 16: + layouts.append([8, 4, 3, 1]) + if num_tokens >= 40: + layouts.append([8] * 5) + + def check(query_lens, run): + contexts, ks, vs = prepare(query_lens) + assert cache is not None + output.fill_(float("nan")) + run() + start = 0 + for i, q_len in enumerate(query_lens): + if q_len == 0: + continue + end = start + q_len + full_k = ( + torch.cat((ks[i], key[start:end])) + .float() + .repeat_interleave(num_heads // num_kv_heads, dim=1) + ) + full_v = ( + torch.cat((vs[i], value[start:end])) + .float() + .repeat_interleave(num_heads // num_kv_heads, dim=1) + ) + scores = ( + torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale + ) + q_pos = torch.arange(q_len, device=device) + contexts[i] + k_pos = torch.arange(full_k.shape[0], device=device) + allowed = k_pos[None, :] <= q_pos[:, None] + if sliding_window is not None: + allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window + probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1) + reference = torch.einsum("hqk,khd->qhd", probs, full_v) + torch.testing.assert_close( + output[start:end].float(), + reference, + atol=2e-2, + rtol=2e-2, + ) + slots = common.slot_mapping[start:end] + written = cache[slots // block_size, :, slots % block_size, :] + torch.testing.assert_close(written[..., :head_size], key[start:end]) + torch.testing.assert_close(written[..., head_size:], value[start:end]) + start = end + assert torch.count_nonzero(cache[0]) == 0 + + for query_lens in layouts: + check(query_lens, graph.replay) + + if num_tokens == 40: + # A longer prefill forces a separate launch in eager/piecewise mode. + # Its length must not become the decode kernel's query bound, and + # compacted CPU placeholders still need not match the GPU allocation. + common.num_reqs = 5 + common.max_query_len = 24 + common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40]) + common.query_start_loc_cpu[6:] = 40 + + def mixed_forward(): + with set_current_vllm_config(config): + metadata = builder.build( + 0, + replace( + common, + query_start_loc=common.query_start_loc[:6], + query_start_loc_cpu=common.query_start_loc_cpu[:6], + seq_lens=common.seq_lens[:5], + seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5], + block_table_tensor=common.block_table_tensor[:5], + ), + ) + assert metadata.num_decodes == 4 + assert metadata.num_prefills == 1 + assert metadata.decode.q_len_per_req == 8 + forward(metadata) + + check([8, 4, 3, 1, 24], mixed_forward) + + @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", diff --git a/tests/v1/attention/test_backend_per_kind.py b/tests/v1/attention/test_backend_per_kind.py index ab77d58825..53dedf3d6b 100644 --- a/tests/v1/attention/test_backend_per_kind.py +++ b/tests/v1/attention/test_backend_per_kind.py @@ -29,7 +29,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind def test_largest_kernel_block_within(supported_sizes, expected): class Backend: @staticmethod - def get_supported_kernel_block_sizes(): + def get_supported_kernel_block_sizes(kv_cache_spec=None): return supported_sizes assert _largest_kernel_block_within(Backend, 1024, 1024 * 1536, 2048) == expected diff --git a/tests/v1/attention/test_cpu_mla_backend.py b/tests/v1/attention/test_cpu_mla_backend.py index fc3867099e..938babb262 100644 --- a/tests/v1/attention/test_cpu_mla_backend.py +++ b/tests/v1/attention/test_cpu_mla_backend.py @@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import ( from vllm.v1.attention.backends.mla.prefill.selector import ( get_mla_prefill_backend, ) +from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import ( + ZenCPUSDPAMLAPrefillBackend, +) def _reference_ragged_attention( @@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None: def test_cpu_mla_prefill_backend_selected() -> None: # On CPU (no device capability) the MLA prefill backend must be the # SDPA-based CPU backend, not flash-attn which is unavailable on CPU. + # Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU + # backend) are also valid selections. backend_cls = get_mla_prefill_backend(None) - assert backend_cls is CPUSDPAMLAPrefillBackend + assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend) @pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") @@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None: assert lse.float().allclose(ref_lse.float(), atol=2e-2) assert out.shape == (5, 4, 16) assert lse.shape == (4, 5) + + +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") +@pytest.mark.skipif( + not ZenCPUSDPAMLAPrefillBackend.is_available(), + reason="requires a Zen CPU with the zentorch_sdpa op registered", +) +def test_zen_cpu_mla_prefill_new_tokens() -> None: + # Runs the zentorch kernel itself. The head dims are deliberately + # asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V + # padding and output slicing are exercised; the generic-CPU cases above + # use equal dims and never reach that code. + qk_head_dim = 24 + v_head_dim = 16 + backend = ZenCPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + + class _PrefillMeta: + query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32) + + backend.prepare_metadata(_PrefillMeta()) + q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16) + + out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False) + ref, _ = _reference_ragged_attention( + q, + k, + v, + _PrefillMeta.query_start_loc, + _PrefillMeta.query_start_loc, + scale=0.25, + causal=True, + ) + assert out.shape == (6, 4, v_head_dim) + assert out.float().allclose(ref.float(), atol=2e-2) + + # zentorch_sdpa does not return a usable LSE, so that path must fall back + # to the fp32 reference and match the generic backend exactly. + generic = CPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + generic.prepare_metadata(_PrefillMeta()) + zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + assert torch.equal(zen_out, gen_out) + assert torch.equal(zen_lse, gen_lse) diff --git a/tests/v1/attention/test_flashinfer_mla_dcp.py b/tests/v1/attention/test_flashinfer_mla_dcp.py index 039d30c9aa..a72dbc21d1 100644 --- a/tests/v1/attention/test_flashinfer_mla_dcp.py +++ b/tests/v1/attention/test_flashinfer_mla_dcp.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import MethodType from unittest.mock import MagicMock import pytest @@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch): attn_metadata.max_seq_len = 1 attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32) attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32) + attn_metadata.decode.max_query_len = 1 query = torch.ones(2, 24, 576, dtype=torch.bfloat16) kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16) @@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal): impl.dcp_world_size = 8 impl.dcp_rank = 3 impl.cp_kv_cache_interleave_size = 1 + impl._flattened_decode_metadata = MethodType( + flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl + ) num_reqs, query_len = 2, 3 num_tokens = num_reqs * query_len @@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal): attn_metadata.decode.block_table = block_table attn_metadata.decode.seq_lens = seq_lens attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens + attn_metadata.decode.max_query_len = query_len + attn_metadata.decode.query_start_loc = torch.arange( + 0, num_tokens + 1, query_len, dtype=torch.int32 + ) + attn_metadata.decode.query_len = 0 query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16) kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16) diff --git a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py index 8dc082b08b..3428625e96 100644 --- a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py +++ b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU). +"""Tests for the FlashInfer SM90 sparse MLA backend wiring and index packing. The FlashInfer wrapper and top-k conversion are replaced by CPU recorders; the tests pin the contract between the impl and the kernel API: page_size=1 @@ -20,6 +20,7 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import ( FlashInferMLASparseSM90Builder, FlashInferMLASparseSM90Impl, ) +from vllm.v1.kv_cache_interface import MLAAttentionSpec # isort: on BLOCK_SIZE = 64 @@ -65,12 +66,18 @@ class FakeState: def __init__(self, width, max_tokens=64): self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32) self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32) + self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32) self.wrapper = FakeWrapper() self.plan_calls = [] def plan(self, num_tokens, kv_lens): self.plan_calls.append((num_tokens, kv_lens)) + def pack_indices(self, slots): + for row in range(slots.shape[0]): + start, end = self.kv_indptr[row : row + 2].tolist() + self.kv_indices[start:end] = slots[row, : end - start].clamp(min=0) + def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK): impl = object.__new__(FlashInferMLASparseSM90Impl) @@ -113,6 +120,9 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype): [6, 5, 4] + [-1] * (TOPK - 3), [2] + [-1] * (TOPK - 1), ] + impl.topk_indices_buffer.copy_(torch.tensor(topk_rows, dtype=torch.int32)) + state.kv_indptr[1:5] = torch.tensor([8, 13, 16, 17], dtype=torch.int32) + state.kv_indptr[5:] = 17 meta = make_batch(rows, topk_rows, [3]) meta.state = state q_nope = torch.randn(rows, impl.num_heads, HEAD) @@ -133,11 +143,13 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype): ref_slots, ref_counts = ref_convert( meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer ) - width = TOPK - got_slots = state.kv_indices[: rows * width].view(rows, width) + offset = 0 for t in range(rows): k = int(ref_counts[t]) - assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist() + assert ( + state.kv_indices[offset : offset + k].tolist() == ref_slots[t, :k].tolist() + ) + offset += k assert state.plan_calls == [] assert state.wrapper.run_args is not None @@ -221,13 +233,51 @@ def test_plan_uses_state_params(monkeypatch): args, kwargs = wrapper.plan_args (qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries - assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)] - assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width + assert kv.tolist() == [0, 2, 7, 14, 14] + assert kv_len.tolist() == [2, 5, 7, 0] assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False) assert scale == 576**-0.5 assert kwargs["q_data_type"] == torch.bfloat16 assert kwargs["kv_data_type"] == torch.bfloat16 + # Replanning a smaller batch must clear the previous rows' lengths. + state.plan(1, torch.tensor([TOPK], dtype=torch.int32)) + assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK] + assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0] + state.plan(0, torch.empty(0, dtype=torch.int32)) + assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0] + assert state._lens_cpu.tolist() == [0, 0, 0, 0] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") +@pytest.mark.parametrize("width", [128, 2048, 2176]) +def test_pack_indices_replays_with_updated_offsets(width): + """Graph replay packs exact prefixes after row lengths and slots change.""" + state = sm90_mod._SM90State.__new__(sm90_mod._SM90State) + state.kv_indptr = torch.zeros(5, dtype=torch.int32, device="cuda") + state.kv_indices = torch.full((4 * width,), -99, dtype=torch.int32, device="cuda") + # Exercise a contiguous view with an unaligned starting address. + slots = torch.arange(4 * width + 1, dtype=torch.int32, device="cuda")[1:] + slots = slots.view(4, width) + state.kv_indptr.copy_(torch.tensor([0, 1, 4, 4, 4])) + state.pack_indices(slots) + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + state.pack_indices(slots) + + for lengths in ([1, 3, 0, 0], [width, 0, 17, width - 1], [0, 0, 0, 0]): + offsets = torch.tensor([0, *torch.tensor(lengths).cumsum(0).tolist()]) + state.kv_indptr.copy_(offsets) + state.kv_indices.fill_(-99) + slots.add_(1) + graph.replay() + expected = torch.cat( + [slots[row, :length] for row, length in enumerate(lengths)] + ) + torch.testing.assert_close(state.kv_indices[: expected.numel()], expected) + assert (state.kv_indices[expected.numel() :] == -99).all() + def test_kv_lens_host_formula(): """Per-row host lengths: context == position + 1; capped at @@ -264,6 +314,59 @@ def test_kv_lens_host_empty(): assert num_rows == 0 and lens.numel() == 0 +_NO_KPOOL = object() + + +@pytest.mark.parametrize( + "index_kpool,prefill_lens", + [ + (4, [2048, 2049, 2050, 2051]), + (None, [2048] * 4), + (_NO_KPOOL, [2048] * 4), + ], + ids=["kpool4", "kpool_none", "no_kpool_attr"], +) +def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens): + """The builder took kpool from the KV cache spec, whose tokens_per_state is + 1, so the tail pool was never read.""" + monkeypatch.setattr( + sm90_mod.FlashInferMLASparseMetadataBuilder, + "__init__", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None) + impl, _ = make_impl(64) + spec = MLAAttentionSpec( + block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 + ) + assert spec.tokens_per_state == 1 + hf_config = SimpleNamespace(index_topk=2048) + if index_kpool is not _NO_KPOOL: + hf_config.index_kpool = index_kpool + vllm_config = SimpleNamespace( + compilation_config=SimpleNamespace( + static_forward_context={"attn": SimpleNamespace(impl=impl)} + ), + scheduler_config=SimpleNamespace( + max_num_batched_tokens=64, async_scheduling=False + ), + model_config=SimpleNamespace(hf_text_config=hf_config), + ) + builder = FlashInferMLASparseSM90Builder( + spec, ["attn"], vllm_config, torch.device("cpu") + ) + # req0: prefill chunk ending at context 42295; req1: context <= topk. + cam = SimpleNamespace( + num_reqs=2, + query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32), + seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32), + positions=None, + ) + num_rows, lens = builder._kv_lens_host(cam) + assert num_rows == 6 + assert lens.tolist() == prefill_lens + [99, 100] + + def test_supports_combination_gates(monkeypatch, default_vllm_config): monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True) call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination( diff --git a/tests/v1/attention/test_group_head_counts.py b/tests/v1/attention/test_group_head_counts.py index 54fe3cfdbc..40d9e2d00f 100644 --- a/tests/v1/attention/test_group_head_counts.py +++ b/tests/v1/attention/test_group_head_counts.py @@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group(): num_kv_heads=2, head_size=64, dtype=torch.bfloat16, + dcp_sharded=True, ) with ( diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 83fceef018..9c92ae554e 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -7,6 +7,9 @@ import pytest import torch from tests.v1.attention.utils import create_vllm_config +from vllm.model_executor.layers.attention.sparse_mla_attention import ( + SparseMLACommonMetadataBuilder, +) from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend, @@ -95,8 +98,11 @@ def test_fused_indexer_decode_metadata(query_lens, padding): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -@pytest.mark.parametrize("query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0]]) -def test_device_token_request_mapping(query_lens): +@pytest.mark.parametrize( + "query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]] +) +@pytest.mark.parametrize("use_sparse_mla_builder", [False, True]) +def test_device_token_request_mapping(query_lens, use_sparse_mla_builder): """Graph replay follows device boundaries even when CPU lengths are stale.""" lengths = torch.tensor(query_lens, device="cuda", dtype=torch.int32) qsl = torch.cat( @@ -117,7 +123,16 @@ def test_device_token_request_mapping(query_lens): ), slot_mapping=torch.full((n + 7,), -1, device="cuda", dtype=torch.int64), ) - result = common.token_to_req_indices(output) + + def build_mapping(): + if use_sparse_mla_builder: + builder = SimpleNamespace(req_id_per_token_buffer=output) + return SparseMLACommonMetadataBuilder._build_req_id_per_token( + builder, common + ) + return common.token_to_req_indices(output) + + result = build_mapping() assert result.data_ptr() == output.data_ptr() expected = torch.repeat_interleave( torch.arange(len(query_lens), device="cuda", dtype=torch.int32), lengths @@ -127,7 +142,7 @@ def test_device_token_request_mapping(query_lens): graph = torch.cuda.CUDAGraph() common._token_to_req_indices_cache = None with torch.cuda.graph(graph): - common.token_to_req_indices(output) + build_mapping() reversed_lens = lengths.flip(0) qsl[1:].copy_(reversed_lens.cumsum(0)) graph.replay() diff --git a/tests/v1/attention/test_kpool_tail_slot_mapping.py b/tests/v1/attention/test_kpool_tail_slot_mapping.py index ed0e250b18..157366e49c 100644 --- a/tests/v1/attention/test_kpool_tail_slot_mapping.py +++ b/tests/v1/attention/test_kpool_tail_slot_mapping.py @@ -24,6 +24,8 @@ from types import SimpleNamespace import pytest import torch +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.models.glm5next.common.attention import Glm5NextTailCache from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.mla.indexer import ( KpoolTailBackend, @@ -56,6 +58,30 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic(): assert content_stride == 1 +@pytest.mark.parametrize( + "num_speculative_tokens, ring", [(0, 4), (1, 8), (4, 8), (7, 16), (13, 32)] +) +def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring): + """With 7 draft tokens the ring was 12 slots. 12 does not divide the + 640-token KDA attention block, so the scheduler block grew to their lcm + and prefix-cache hits were cut down to multiples of 1920.""" + with set_current_vllm_config(VllmConfig()): + cache = Glm5NextTailCache( + head_dim=128, + dtype=torch.bfloat16, + prefix="tail", + cache_config=SimpleNamespace(block_size=640), + index_kpool=KPOOL, + ) + spec = cache.get_kv_cache_spec( + SimpleNamespace(num_speculative_tokens=num_speculative_tokens) + ) + + assert spec.block_size == spec.sliding_window == ring + assert ring >= KPOOL + num_speculative_tokens + assert 640 % ring == 0 + + def test_tail_spec_opts_out_of_generic_slot_mapping(): """The tail row is one block wide (padded to the block-table alignment), so the generic kernel's ``pos // kpool`` column index runs off the end of the @@ -296,31 +322,30 @@ def test_builder_reuses_slot_mapping_storage(): class TailRingMirror: - """Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel - addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool - completing at pos reads ring slots (pool_start + s) % kpool and uses the - current token's own K/score for the last member.""" + """Mirror of the tail-ring addressing in _kpool_tail_seed_kernel and + _kpool_decode_update_batched_kernel.""" - def __init__(self, num_blocks, kpool=KPOOL): + def __init__(self, num_blocks, kpool=KPOOL, ring=None): self.kpool = kpool - self.k = torch.full((num_blocks, kpool, 3), float("nan")) - self.s = torch.full((num_blocks, kpool, 3), float("nan")) + self.ring = ring or kpool + self.k = torch.full((num_blocks, self.ring, 3), float("nan")) + self.s = torch.full((num_blocks, self.ring, 3), float("nan")) def stash(self, tail_slot, pos, k, s): - blk, off = tail_slot // self.kpool, pos % self.kpool + blk, off = tail_slot // self.ring, pos % self.ring self.k[blk, off] = k self.s[blk, off] = s seed = stash # the seed kernel writes with the same addressing def complete(self, tail_slot, pos, k, s): - blk = tail_slot // self.kpool + blk = tail_slot // self.ring start = pos - (self.kpool - 1) kk = torch.stack( - [self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)] + [self.k[blk, (start + i) % self.ring] for i in range(self.kpool)] ) ss = torch.stack( - [self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)] + [self.s[blk, (start + i) % self.ring] for i in range(self.kpool)] ) kk[-1], ss[-1] = k, s # is_current for the completing token w = torch.softmax(ss, dim=0) @@ -454,3 +479,35 @@ def test_triton_mapping_reads_strided_block_table(): for req, blk in enumerate(own_blocks): start, end = int(qsl[req]), int(qsl[req + 1]) assert (got[start:end] // KPOOL == blk).all() + + +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_rejected_completing_draft_needs_ring_slots(ring_pools): + """With a one-pool ring, the drafts behind a rejected pool-completing draft + overwrote the pool's earlier keys, so its redo compressed wrong keys.""" + ring_size = ring_pools * KPOOL + truth = TailRingMirror(num_blocks=2, ring=ring_size) + ring = TailRingMirror(num_blocks=2, ring=ring_size) + block = 1 + + def slot(pos): + return block * ring_size + pos % ring_size + + for pos in range(4, 7): + truth.stash(slot(pos), pos, *token_kv(0, pos)) + ring.stash(slot(pos), pos, *token_kv(0, pos)) + expected = truth.complete(slot(7), 7, *token_kv(0, 7)) + + # Draft 7 completes the pool and is rejected. With a one-pool ring, drafts + # 8..10 overwrite the slots of positions 4..6, which are read by the redo + # of 7. + for pos in range(7, 11): + k, s = token_kv(9, pos) # draft values + if pos % KPOOL == KPOOL - 1: + ring.complete(slot(pos), pos, k, s) + ring.stash(slot(pos), pos, k, s) + redo = ring.complete(slot(7), 7, *token_kv(0, 7)) + if ring_pools == 1: + assert not torch.allclose(redo, expected) + else: + torch.testing.assert_close(redo, expected) diff --git a/tests/v1/attention/test_mamba_update_block_table.py b/tests/v1/attention/test_mamba_update_block_table.py index f4451fe7af..49a298fe19 100644 --- a/tests/v1/attention/test_mamba_update_block_table.py +++ b/tests/v1/attention/test_mamba_update_block_table.py @@ -427,3 +427,40 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer(): # Tail values past num_decodes: zero-filled padding for cudagraph capture. assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0) + + +def test_num_state_writes_p_matches_block_idx_tensors(): + """The prefill state save sizes its batched copy from + num_state_writes_p instead of reading the block index tensors back from + the GPU, so the host count must equal what those tensors say.""" + # (block_size, seq_lens, query_lens, is_prefilling) + cases = [ + # block-aligned and unaligned computed tokens, and a chunked prefill + (16, [64, 33, 17], [48, 17, 1], [True, True, True]), + (32, [200, 100, 64], [200, 36, 64], [True, True, True]), + # decode rows ahead of the prefills + (16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]), + # prefills that stay inside their first block: nothing to write + (16, [65, 66, 67], [1, 2, 3], [True, True, True]), + ] + + for block_size, seq_lens, query_lens, is_prefilling in cases: + metadata = MockMambaBuilder.build_mamba_metadata( + _make_vllm_config(4096, len(seq_lens), block_size=block_size), + seq_lens=seq_lens, + query_lens=query_lens, + is_prefilling=is_prefilling, + ) + num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills + block_idx_last_p = metadata.block_idx_last_scheduled_token[ + num_reqs - num_prefills : num_reqs + ] + expected = int( + (block_idx_last_p - metadata.block_idx_first_scheduled_token_p) + .clamp(min=0) + .sum() + ) + + assert metadata.num_state_writes_p == expected, ( + f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}" + ) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 597ffb03fc..9ae0fc216a 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -9,7 +9,7 @@ Known Issues: """ import sys -from types import SimpleNamespace +from types import MethodType, SimpleNamespace import pytest import torch @@ -218,12 +218,14 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype( cache_dtype: str, expected_quant_mode: KVQuantMode ): layer = SimpleNamespace( + attn_backend=flashmla_module.FlashMLABackend, kv_cache_dtype=cache_dtype, head_size=576, indexer=None, non_causal_multi_token_decode=False, sliding_window=None, ) + layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer) vllm_config = SimpleNamespace( cache_config=SimpleNamespace(block_size=64), model_config=None ) @@ -1267,6 +1269,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=2, + max_query_len=1, dcp_tot_seq_lens_device=None, ) diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 098eddfbf6..87d188c6a0 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend: def test_cpu_uses_sdpa_prefill(self): vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + # A Zen CPU with zentorch selects the Zen backend, so expect whichever + # one this host offers. + expected = ( + zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class() + ) with patch("vllm.platforms.current_platform") as mock_platform: mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) + assert backend is expected + + def test_zen_cpu_prefers_zentorch_prefill(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=True), + ): + mock_platform.is_cpu.return_value = True + + backend = get_mla_prefill_backend(vllm_config) + assert backend is zen_cls + + def test_zen_cpu_falls_back_when_zentorch_unavailable(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=False), + ): + mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) assert backend is MLAPrefillBackendEnum.CPU.get_class() diff --git a/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py b/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py index a86c7e617b..e6390ce36e 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py +++ b/tests/v1/attention/test_rocm_aiter_mla_fp8_decode_routing.py @@ -151,6 +151,25 @@ def test_decode_and_verify_are_disjoint(kv_cache_dtype, num_heads): ) +@pytest.mark.parametrize("kv_cache_dtype", UNQUANTIZED_DTYPES) +def test_oversized_kv_cache_refuses_gluon(gluon_available, kv_cache_dtype): + """Past 2 GiB per layer Gluon drops its KV bounds mask and reads out of range. + + ``None`` is the profiling case, before the cache is sized: nothing to + refuse yet, so the usual route stands. + """ + bound = rocm_aiter_mla._GLUON_MAX_KV_CACHE_BYTES + for kv_cache_bytes in (None, bound): + assert AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, kv_cache_bytes) + assert AiterMLAHelper.use_gluon_verify( + 8, 8, kv_cache_dtype, kv_cache_bytes=kv_cache_bytes + ) + assert not AiterMLAHelper.use_gluon_decode(8, 1, kv_cache_dtype, bound + 1) + assert not AiterMLAHelper.use_gluon_verify( + 8, 8, kv_cache_dtype, kv_cache_bytes=bound + 1 + ) + + @pytest.mark.parametrize("num_heads", [1, 2, 3, 5, 6, 7, 8, 9, 12, 15]) def test_padded_query_is_contiguous(num_heads): """asm_mla.cu:805 requires Q.is_contiguous(). diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index b8ba46d8de..1970dbb54f 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -107,6 +107,8 @@ def _builder( _dcp_verify_buffers=None, _graph_seq_lens=None, _kv_cache_dtype_str=kv_cache_dtype, + # Unsized as during profiling, so the Gluon KV bound stays out of the way. + _kv_cache_bytes=None, paged_kv_last_page_len=torch.ones(max_decode_rows, dtype=torch.int32), paged_kv_indices=torch.empty(1024, dtype=torch.int32), paged_kv_indptr=torch.empty(max_decode_rows + 1, dtype=torch.int32), @@ -278,6 +280,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch): query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), num_decode_tokens=2 * qlen, + max_query_len=qlen, dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32), ) @@ -584,7 +587,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( get_num_attention_heads=lambda parallel_config: num_heads, ), scheduler_config=SimpleNamespace(max_num_seqs=2), - cache_config=SimpleNamespace(cache_dtype="fp8_e4m3"), + cache_config=SimpleNamespace(cache_dtype="fp8_e4m3", num_gpu_blocks=None), compilation_config=SimpleNamespace( cudagraph_mode=SimpleNamespace(has_full_cudagraphs=lambda: False), # Empty: the per-layer head-count probe finds no attention layer and @@ -633,6 +636,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch): query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32), query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32), num_decode_tokens=8, + max_query_len=4, dcp_tot_seq_lens_device=None, ) @@ -683,6 +687,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch): query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * mtp_qlen, + max_query_len=mtp_qlen, dcp_tot_seq_lens_device=None, ) @@ -733,6 +738,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr( query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32), num_decode_tokens=seq_lens.numel() * mtp_qlen, + max_query_len=mtp_qlen, dcp_tot_seq_lens_device=None, ) @@ -801,6 +807,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch): query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32), query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32), num_decode_tokens=seq_lens.numel(), + max_query_len=1, dcp_tot_seq_lens_device=None, ) @@ -892,6 +899,7 @@ def test_persistent_metadata_gate( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * qo_len, + max_query_len=qo_len, dcp_tot_seq_lens_device=None, ) @@ -961,6 +969,7 @@ def test_persistent_metadata_gate_without_gluon_build( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * qo_len, + max_query_len=qo_len, dcp_tot_seq_lens_device=None, ) @@ -993,6 +1002,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen) query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), num_decode_tokens=2 * qlen, + max_query_len=qlen, dcp_tot_seq_lens_device=None, ) return metadata, get_mla_metadata_v1 diff --git a/tests/v1/attention/test_rocm_attention_backends_selection.py b/tests/v1/attention/test_rocm_attention_backends_selection.py index 2342a279a3..b99a33f9f7 100644 --- a/tests/v1/attention/test_rocm_attention_backends_selection.py +++ b/tests/v1/attention/test_rocm_attention_backends_selection.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for attention backend selectors.""" +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest @@ -34,6 +35,15 @@ def mock_get_cdna_version(): yield +@pytest.fixture +def cleared_attention_selector_cache(): + from vllm.v1.attention.selector import _cached_get_attn_backend + + _cached_get_attn_backend.cache_clear() + yield + _cached_get_attn_backend.cache_clear() + + def test_aiter_unified_attention_uses_dedicated_metadata_builder(): from vllm.v1.attention.backends.rocm_aiter_unified_attn import ( RocmAiterUnifiedAttentionBackend, @@ -434,6 +444,139 @@ def test_aiter_fa_requires_mi3xx(mock_vllm_config): ) +@pytest.fixture +def turboquant_run_config(): + """Current config of a run whose KV cache dtype is a turboquant_* preset.""" + config = SimpleNamespace( + cache_config=SimpleNamespace(cache_dtype="turboquant_k8v4") + ) + with patch( + "vllm.config.get_current_vllm_config_or_none", + return_value=config, + ): + yield config + + +def test_turboquant_boundary_selection_is_not_cached_from_ordinary_run( + cleared_attention_selector_cache, +): + from vllm.config import CacheConfig, VllmConfig, set_current_vllm_config + from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts + from vllm.v1.attention.selector import get_attn_backend + from vllm.v1.kv_cache_layout import KVCacheLayout + + ordinary_config = VllmConfig(cache_config=CacheConfig(cache_dtype="auto")) + turboquant_config = VllmConfig( + cache_config=CacheConfig(cache_dtype="turboquant_k8v4") + ) + + backend_priorities = [ + AttentionBackendEnum.ROCM_ATTN, + AttentionBackendEnum.TRITON_ATTN, + AttentionBackendEnum.TURBOQUANT, + ] + with patch( + "vllm.platforms.rocm._get_backend_priorities", + return_value=backend_priorities, + ): + with set_current_vllm_config(ordinary_config): + ordinary_backend = get_attn_backend(128, torch.float16, "auto") + + with set_current_vllm_config(turboquant_config): + boundary_backend = get_attn_backend(128, torch.float16, "auto") + quantized_backend = get_attn_backend(128, torch.float16, "turboquant_k8v4") + + assert ordinary_backend is AttentionBackendEnum.ROCM_ATTN.get_class() + assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class() + assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend]) == [ + KVCacheLayout.LBNHC + ] + + +def test_turboquant_run_does_not_mask_unsupported_selected_backend( + turboquant_run_config, +): + from vllm.platforms.rocm import RocmPlatform + + attn_selector_config = AttentionSelectorConfig( + head_size=128, + dtype=torch.float16, + kv_cache_dtype="auto", + block_size=16, + ) + + with ( + patch("vllm.platforms.rocm.get_cdna_version", return_value=1), + pytest.raises(ValueError, match="compute capability not supported"), + ): + RocmPlatform.get_attn_backend_cls( + selected_backend=AttentionBackendEnum.ROCM_AITER_FA, + attn_selector_config=attn_selector_config, + ) + + +def test_turboquant_layout_check_respects_backend_override(): + from vllm.platforms.rocm import _shares_layout_with_turboquant + from vllm.v1.kv_cache_layout import KVCacheLayout + + boundary_backend = MagicMock() + boundary_backend.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,) + turboquant_override = MagicMock() + turboquant_override.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,) + + with patch.object( + AttentionBackendEnum.TURBOQUANT, + "get_class", + return_value=turboquant_override, + ) as get_turboquant_class: + assert _shares_layout_with_turboquant(boundary_backend) + get_turboquant_class.assert_called_once_with() + + +@pytest.mark.parametrize( + "selected_backend", + [None, "ROCM_ATTN", "ROCM_AITER_FA", "TURBOQUANT"], +) +def test_turboquant_boundary_layers_share_a_layout( + selected_backend, + turboquant_run_config, + mock_get_cdna_version, +): + """A turboquant_* run must resolve to backends with a common KV layout. + + The boundary layers keep the native dtype and pick their own backend while + every other layer picks TURBOQUANT. Engine startup hard-errors when the two + share no layout, so the boundary layers must not land on a native ROCm or + AITER backend, whichever backend the run asked for. + """ + from vllm.platforms.rocm import RocmPlatform + from vllm.utils.import_utils import resolve_obj_by_qualname + from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts + + backend_enum = ( + getattr(AttentionBackendEnum, selected_backend) if selected_backend else None + ) + + def resolve(kv_cache_dtype): + path = RocmPlatform.get_attn_backend_cls( + selected_backend=backend_enum, + attn_selector_config=AttentionSelectorConfig( + head_size=128, + dtype=torch.float16, + kv_cache_dtype=kv_cache_dtype, + block_size=16, + ), + ) + return resolve_obj_by_qualname(path) + + boundary_backend = resolve("auto") + quantized_backend = resolve("turboquant_k8v4") + + assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class() + # Raises when the intersection is empty, which is the startup failure. + assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend]) + + def test_sparse_not_supported(mock_vllm_config): """Test that sparse MLA without use_mla flag raises an error.""" from vllm.platforms.rocm import RocmPlatform diff --git a/tests/v1/attention/test_sparse_mla_kv_cache_layout.py b/tests/v1/attention/test_sparse_mla_kv_cache_layout.py new file mode 100644 index 0000000000..cd3c9b2181 --- /dev/null +++ b/tests/v1/attention/test_sparse_mla_kv_cache_layout.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sparse MLA allocation and warmup must agree on physical addressing.""" + +from types import MethodType, SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from vllm.config import CacheConfig +from vllm.model_executor.layers.attention.mla_attention import MLAAttention +from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache +from vllm.platforms import current_platform +from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( + FlashInferMLASparseTRTLLMBackend, +) +from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend +from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend +from vllm.v1.attention.backends.mla.sparse_utils import ( + _CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL, + flat_kv_row_view, +) +from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts +from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups +from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs +from vllm.v1.kv_cache_layout import KVCacheLayout +from vllm.v1.worker.utils import allocate_kv_cache + +pytestmark = pytest.mark.skip_global_cleanup + + +@pytest.mark.parametrize( + "device", + [ + "cpu", + pytest.param( + "cuda", + marks=pytest.mark.skipif( + not current_platform.is_cuda(), reason="CUDA only" + ), + ), + ], +) +@pytest.mark.parametrize( + "layout", [KVCacheLayout.BLHNC, KVCacheLayout.BLNHC, KVCacheLayout.LBNHC] +) +@pytest.mark.parametrize( + "backend,cache_dtype,sm100,stride_alignment", + [ + (FlashInferMLASparseTRTLLMBackend, "auto", True, 1152), + (FlashInferMLASparseTRTLLMBackend, "fp8", True, 576), + (FlashMLASparseBackend, "auto", True, 1152), + (FlashMLASparseBackend, "fp8_ds_mla", False, None), + (FlashMLASparseBackend, "fp8_ds_mla", True, 656), + (FlashMLASparseBackend, "nvfp4_ds_mla", True, 352), + ], +) +def test_allocation_and_warmup_follow_addressing_mode( + monkeypatch, layout, backend, cache_dtype, sm100, stride_alignment, device +): + monkeypatch.setattr( + current_platform, "is_device_capability_family", lambda family: sm100 + ) + config = SimpleNamespace( + cache_config=CacheConfig(block_size=64), + model_config=SimpleNamespace(dtype=torch.bfloat16), + attention_config=SimpleNamespace(hisparse_config=None), + kernel_config=SimpleNamespace(enable_jit_warmup=True), + ) + config.cache_config.kv_cache_layout = layout.name + layer = SimpleNamespace( + _vllm_config=config, + attn_backend=backend, + kv_cache_dtype=cache_dtype, + head_size=576, + sliding_window=None, + indexer=None, + non_causal_multi_token_decode=False, + ) + layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer) + spec = MLAAttention.get_kv_cache_spec(layer, config) + indexer = SimpleNamespace( + cache_config=config.cache_config, head_dim=132, dtype=torch.uint8 + ) + index_spec = DeepseekV32IndexerCache.get_kv_cache_spec(indexer, config) + assert spec.block_stride_alignment == stride_alignment + assert index_spec.block_stride_alignment is None + assert layout in get_supported_kv_cache_layouts( + [backend, DeepseekV32IndexerBackend] + ) + specs = {"mla": spec, "indexer": index_spec} + group = KVCacheGroupSpec( + list(specs), UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs) + ) + cache = get_kv_cache_config_from_groups(config, [group], 2**20) + views = allocate_kv_cache(cache, torch.device(device), layout) + register = MagicMock() + monkeypatch.setattr( + _CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL, "register_warmup", register + ) + MLAAttention.bind_kv_cache(layer, views["mla"]) + DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"]) + assert indexer.kv_cache.stride(0) == views["indexer"].stride(0) + if stride_alignment is not None: + assert ( + views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0 + ) + if not layer._uses_flat_kv_cache(): + register.assert_not_called() + else: + rows, stride = flat_kv_row_view(layer.kv_cache, 64) + register.assert_called_once_with(config, block_stride_rows=stride) + layer.kv_cache[1, 0].fill_(3) + torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0]) + if stride_alignment is None: + assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum( + item.page_size_bytes for item in specs.values() + ) diff --git a/tests/v1/attention/test_trtllm_attention_integration.py b/tests/v1/attention/test_trtllm_attention_integration.py index 1a49a3895c..42dd8f9f48 100644 --- a/tests/v1/attention/test_trtllm_attention_integration.py +++ b/tests/v1/attention/test_trtllm_attention_integration.py @@ -9,6 +9,7 @@ import pytest import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention +from tests.kernels.attention.test_trtllm_kvfp8_dequant import make_random_kv_cache from tests.v1.attention.test_attention_backends import create_and_prepopulate_kv_cache from tests.v1.attention.utils import ( BatchSpec, @@ -17,7 +18,11 @@ from tests.v1.attention.utils import ( ) from vllm.config import set_current_vllm_config from vllm.platforms import current_platform -from vllm.utils.torch_utils import nvfp4_kv_cache_full_dim, set_random_seed +from vllm.utils.torch_utils import ( + is_strictly_contiguous, + nvfp4_kv_cache_full_dim, + set_random_seed, +) from vllm.v1.attention.backends.utils import PerLayerParameters from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode @@ -33,6 +38,7 @@ from vllm.v1.attention.backends.flashinfer import ( # noqa: E402 FlashInferMetadataBuilder, FlashInferTrtllmAPIDecode, TRTLLMPrefill, + trtllm_prefill_attn_kvfp8_dequant, ) @@ -83,6 +89,70 @@ def _mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): } +@pytest.mark.parametrize( + "block_size,num_prefills,has_host_lengths,quantize_query", + [ + (16, 1, True, False), + (64, 2, True, False), + (128, 2, True, False), + (16, 2, False, False), + (16, 2, True, True), + ], +) +@torch.inference_mode() +def test_prefill_dequant_scratch_ignores_unused_capacity( + block_size, num_prefills, has_host_lengths, quantize_query, monkeypatch +): + """Bound scratch by attended prefill KV, including cached prefixes.""" + device = torch.device("cuda") + config = create_vllm_config(model_name=MODEL, max_model_len=32768) + config.cache_config.cache_dtype = "fp8" + config.cache_config.kv_cache_layout = "LBHNC" + config.attention_config.use_trtllm_attention = True + config.attention_config.disable_flashinfer_q_quantization = not quantize_query + monkeypatch.setattr( + "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", + _mock_get_per_layer_parameters, + ) + spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=64, + dtype=torch.uint8, + kv_quant_mode=KVQuantMode.FP8_PER_TENSOR, + ) + kv_cache, scale = make_random_kv_cache(32, 2, block_size, 64) + batch = BatchSpec( + seq_lens=[1024] + [17, 65][-num_prefills:], + query_lens=[1] + [2, 8][-num_prefills:], + ) + common = create_common_attn_metadata(batch, block_size, device) + if not has_host_lengths: + common.seq_lens_cpu_upper_bound = None + live_pages = ((65 if has_host_lengths else 1024) + block_size - 1) // block_size + with set_current_vllm_config(config): + builder = FlashInferMetadataBuilder(spec, ["layer.0"], config, device) + for capacity in (64, 256): + # Unused columns can contain nonzero page IDs. + common.block_table_tensor = torch.randint( + 1, 32, (batch.batch_size, capacity), dtype=torch.int32, device=device + ) + metadata = builder.build(0, common) + assert isinstance(metadata.prefill, TRTLLMPrefill) + table = metadata.prefill.block_tables + assert is_strictly_contiguous(table) + assert (metadata.q_data_type_prefill == current_platform.fp8_dtype()) == ( + quantize_query + ) + if not quantize_query: + converted, _ = trtllm_prefill_attn_kvfp8_dequant( + kv_cache, table, scale, scale, metadata.q_data_type_prefill + ) + assert converted.shape[0] == num_prefills * live_pages + 1 + width = capacity if quantize_query else live_pages + torch.testing.assert_close(table, common.block_table_tensor[1:, :width]) + + def _create_nvfp4_hnd_kv_cache( k_contexts, v_contexts, diff --git a/tests/v1/attention/utils.py b/tests/v1/attention/utils.py index cd062cb486..abb29a03c4 100644 --- a/tests/v1/attention/utils.py +++ b/tests/v1/attention/utils.py @@ -262,27 +262,6 @@ def create_vllm_config( ) -def create_dummy_kv_cache( - block_size: int, - num_kv_heads: int, - head_size: int, - dtype: torch.dtype, - device: torch.device, - num_blocks: int = 100, -) -> torch.Tensor: - """Create a dummy KV cache tensor for testing.""" - kv_cache = torch.randn( - num_blocks, - 2, # K and V - block_size, - num_kv_heads, - head_size, - dtype=dtype, - device=device, - ) - return kv_cache - - @dataclass class BackendConfig: name: str diff --git a/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py b/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py index 4cb9afe871..324aef425b 100644 --- a/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py +++ b/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py @@ -196,6 +196,28 @@ def test_sibling_resumes_below_the_block_grid_when_the_prefix_ends_early(): assert hit == resume, f"expected the resume point at {resume}, got {hit}" +@pytest.mark.parametrize("mtp_draft_group_identified", [False, True]) +def test_mamba_prefix_cache_drops_hash_block(mtp_draft_group_identified): + block_size, hash_block_size = 1024, 64 + manager = _manager( + block_size, + hash_block_size, + eagle_group=0 if mtp_draft_group_identified else None, + ) + prompt_a = PREFIX[:1600] + hit_after_mtp_block_drop = len(prompt_a) - hash_block_size + + _prefill( + manager, + _stub(manager, block_size, hash_block_size), + make_request("A", prompt_a, hash_block_size, sha256), + ) + + tokens_after_a = [-1] * 500 + hit = _sibling_hit(manager, len(prompt_a), tokens_after_a, hash_block_size) + assert hit == hit_after_mtp_block_drop + + # -------------------------------------------------------------------------- # Invariants the two clauses have to keep diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py index 702372bbc2..5b43e2c561 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py @@ -7,6 +7,11 @@ import pytest import vllm.v1.core.kv_cache_utils as kv_cache_utils from vllm.distributed.kv_events import BlockRemoved, BlockStored +from vllm.multimodal.inputs import ( + MultiModalFeatureSpec, + MultiModalKwargsItem, + PlaceholderRange, +) from vllm.sampling_params import SamplingParams from vllm.utils.hashing import sha256 from vllm.v1.core.block_pool import BlockPool @@ -34,12 +39,28 @@ def make_request( hash_block_size: int, hash_fn: Callable, session_id: str | None = None, + mm_positions: list[PlaceholderRange] | None = None, + mm_hashes: list[str] | None = None, ) -> Request: + mm_features = [] + if mm_positions is not None: + for j, position in enumerate(mm_positions): + identifier = mm_hashes[j] if mm_hashes else f"hash_{j}" + mm_features.append( + MultiModalFeatureSpec( + data=MultiModalKwargsItem.dummy(), + mm_position=position, + identifier=identifier, + modality="image", + ) + ) + sampling_params = SamplingParams(max_tokens=17) sampling_params.update_from_generation_config({}, eos_token_id=100) return Request( request_id=request_id, prompt_token_ids=prompt_token_ids, + mm_features=mm_features if mm_features else None, sampling_params=sampling_params, pooling_params=None, block_hasher=get_request_block_hasher(hash_block_size, hash_fn), @@ -173,6 +194,54 @@ def test_cache_partial_block_kv_cache_events(): assert removed_event.group_idx == kv_cache_group_id +def test_cache_partial_block_event_keeps_every_mm_feature(): + """The event window can reach into more than one multimodal feature. + + ``cache_partial_block`` builds the ``BlockStored`` extra keys over + ``[(num_hash_blocks - 1) * hash_block_size, num_tokens)``. Starting that + scan at the last feature drops any earlier one that also reaches into the + window, so two requests differing only in that feature would be published + under the same key. + """ + hash_block_size = 4 + block_size = 8 + num_tokens = 12 + # The event window is tokens [8, 12): "A" ends inside it and "B" sits + # entirely within it. + mm_positions = [ + PlaceholderRange(offset=6, length=3), + PlaceholderRange(offset=10, length=2), + ] + + pool = BlockPool( + num_gpu_blocks=2, + enable_caching=True, + hash_block_size=hash_block_size, + enable_kv_cache_events=True, + ) + req = make_request( + "req_partial_mm", + prompt_token_ids=list(range(num_tokens)), + hash_block_size=hash_block_size, + hash_fn=sha256, + mm_positions=mm_positions, + mm_hashes=["A", "B"], + ) + + assert pool.cache_partial_block( + request=req, + block=pool.get_new_blocks(1)[0], + num_tokens=num_tokens, + kv_cache_group_id=0, + block_size=block_size, + ) + + events = pool.take_events() + assert len(events) == 1 + assert isinstance(events[0], BlockStored) + assert events[0].extra_keys == [(("A", -2), ("B", 2))] + + def test_partial_block_replacement_emits_remove_then_store_events(): hash_block_size = 2 block_size = 6 diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index 27107aa6b0..52167a5073 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -171,42 +171,6 @@ def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch): assert observed_placeholders == [0, 1] -def test_preempt(): - scheduler = create_scheduler(async_scheduling=True) - requests = create_requests(num_requests=10, max_tokens=20) - - for req in requests: - scheduler.add_request(req) - - sched_outputs: deque[SchedulerOutput] = deque() - sched_outputs.append(scheduler.schedule()) - sched_outputs.append(scheduler.schedule()) - - abort_order = [0, 8, 3, 1, 6, 4, 2, 5, 7, 9] - abort_order_copy = abort_order.copy() - - def abort_request(): - if not abort_order: - return - req = requests[abort_order.pop(0)] - scheduler.finish_requests(req.request_id, RequestStatus.FINISHED_ABORTED) - - while sched_outputs: - # Abort a scheduled request. - abort_request() - sched_output = sched_outputs.popleft() - model_runner_output = _make_model_runner_output(sched_output) - scheduler.update_from_output(sched_output, model_runner_output) - - sched_output = scheduler.schedule() - if sched_output.num_scheduled_tokens: - sched_outputs.append(sched_output) - - for i, req in enumerate(requests): - assert req.status == RequestStatus.FINISHED_ABORTED - assert req.num_output_tokens == abort_order_copy.index(i) - - def test_prefix_caching_for_prefill_dedup(): CHUNK_SIZE = 1000 BLOCK_SIZE = 16 @@ -353,6 +317,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.finished_req_ids = set() scheduler.finished_req_ids_dict = None scheduler.grammar_compile_error_reqs = set() + scheduler.encoder_cache_mismatch_reqs = set() scheduler.vllm_config = Mock() scheduler.aux_output_connector = None scheduler.return_sampling_mask = False diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 58d4e20bf7..638deeedc5 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -10,6 +10,7 @@ block); the allocation is the same either way. """ from dataclasses import replace +from math import lcm from unittest.mock import MagicMock import pytest @@ -26,9 +27,11 @@ from vllm.v1.core.kv_cache_utils import ( get_kv_cache_groups, resolve_kv_cache_block_sizes, ) +from vllm.v1.hisparse.layout import _build_hisparse_kv_cache_tensors from vllm.v1.kv_cache_interface import ( CircularBufferSpec, FullAttentionSpec, + HiSparseHotSpec, KVCacheGroupSpec, KVCacheLayout, KVCacheSpec, @@ -79,6 +82,30 @@ def _mock_vllm_config(layout: str | None): return config +def test_packed_alignment_preserves_hot_pages_and_generic_stride(): + """One shared stride must satisfy both constraints, regardless of rounding order.""" + spec = replace(_mla(128), block_stride_alignment=1152) + hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2) + groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)] + alignment = lcm(1152, hot.page_size_bytes) + stride = _pool_bytes_per_block(groups) + assert stride == alignment + layout = KVCacheLayout.BLHNC + tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride) + assert {tensor.block_stride for tensor in tensors} == {alignment} + assert {tensor.size for tensor in tensors} == {3 * alignment} + + +@pytest.mark.parametrize("spec", [_full(), _mla(128)]) +def test_block_alignment_survives_merge_and_rejects_incompatible_specs(spec): + aligned = replace(spec, block_stride_alignment=512) + assert type(spec).merge([aligned, aligned]).block_stride_alignment == 512 + with pytest.raises(AssertionError): + type(spec).merge([aligned, spec]) + with pytest.raises(ValueError, match="must be positive"): + replace(spec, block_stride_alignment=0) + + def _pages(groups) -> dict[str, int]: return { name: group.kv_cache_spec.kv_cache_specs[name].page_size_bytes diff --git a/tests/v1/core/test_encoder_cache_manager.py b/tests/v1/core/test_encoder_cache_manager.py index 3948f7c6e3..3c61f3367f 100644 --- a/tests/v1/core/test_encoder_cache_manager.py +++ b/tests/v1/core/test_encoder_cache_manager.py @@ -114,6 +114,7 @@ def test_eviction_when_cache_is_full(): # 'x' should have been evicted. assert "x" not in manager.cached + assert "x" not in manager.num_embeds_by_hash assert "req1" not in manager.request_cached_ids assert "x" in manager.get_freed_mm_hashes() @@ -273,6 +274,31 @@ def test_encoder_cache_with_is_embed_mask(): assert savings_ratio == 12.5 +def test_encoder_cache_tracks_embedding_count_by_identifier(): + class MockRequestWithMask(MockRequest): + def get_num_encoder_embeds(self, input_id: int) -> int: + return self.mm_features[input_id].mm_position.get_num_embeds() + + is_embed = torch.tensor([False, True, True, False, True, False]) + request = MockRequestWithMask("owner", ["img1"], [len(is_embed)]) + request.mm_features[0] = MultiModalFeatureSpec( + data=None, + modality="image", + identifier="img1", + mm_position=PlaceholderRange(offset=0, length=6, is_embed=is_embed), + ) + + manager = EncoderCacheManager(cache_size=10) + manager.allocate(request, 0) + assert manager.get_cached_num_encoder_embeds(request, 0) == 3 + + manager.free(request) + assert manager.get_cached_num_encoder_embeds(request, 0) == 3 + + manager.reset() + assert manager.get_cached_num_encoder_embeds(request, 0) is None + + def test_encoder_cache_mask_based_retrieval(): class MockRequestWithMask(MockRequest): def get_num_encoder_embeds(self, input_id: int) -> int: @@ -336,6 +362,7 @@ def test_reset_clears_all_state(): manager.reset() assert len(manager.cached) == 0 + assert len(manager.num_embeds_by_hash) == 0 assert len(manager.request_cached_ids) == 0 assert len(manager.freeable) == 0 assert len(manager.freed) == 0 diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index bc2a761d86..c2c86053f0 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1018,6 +1018,37 @@ def test_request_block_hasher(hash_fn): assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),))) +@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) +def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_fn): + mm_positions = [ + PlaceholderRange(offset=4, length=2), + PlaceholderRange(offset=6, length=1), + ] + incremental = make_request( + request_id="incremental", + prompt_token_ids=list(range(7)), + block_size=4, + hash_fn=hash_fn, + mm_positions=mm_positions, + mm_hashes=["A", "B"], + ) + incremental.append_output_token_ids(7) + fresh = make_request( + request_id="fresh", + prompt_token_ids=list(range(8)), + block_size=4, + hash_fn=hash_fn, + mm_positions=mm_positions, + mm_hashes=["A", "B"], + ) + + expected_second_hash = hash_fn( + (incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2))) + ) + assert incremental.block_hashes[1] == expected_second_hash + assert incremental.block_hashes == fresh.block_hashes + + @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) def test_hash_tokens_different_mm_input(hash_fn): request1 = make_request( @@ -1568,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker(): assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"} -def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only(): - dcp = 8 - full = FullAttentionSpec( - block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 +@pytest.mark.parametrize("sliding_window", [None, 256]) +@pytest.mark.parametrize("disable_hybrid", [False, True]) +@pytest.mark.parametrize("pcp_size", [1, 4]) +def test_dcp_target_allocates_replicated_draft_independently( + monkeypatch, sliding_window, disable_hybrid, pcp_size +): + """A draft must retain all positions even when the target shards them.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + config.parallel_config.prefill_context_parallel_size = pcp_size + config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid + config.cache_config.block_size = 16 + config.cache_config.kv_cache_layout = None + draft_args = dict( + block_size=16, + num_kv_heads=8, + head_size=64, + dtype=torch.bfloat16, + dcp_sharded=False, ) - mla = new_mla_spec() - mamba = new_mamba_spec() - uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla}) - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1 - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1 + draft = ( + FullAttentionSpec(**draft_args) + if sliding_window is None + else SlidingWindowSpec(**draft_args, sliding_window=sliding_window) + ) + specs = {"target": new_mla_spec(), "draft": draft} + layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values()) + assert layout == KVCacheLayout.BLHNC + groups = get_kv_cache_groups(config, specs) + assert [group.layer_names for group in groups] == [["target"], ["draft"]] + widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups] + assert widths == [16, 64] + + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=16 * 1024 * 1024 + ) + scheduler_config = generate_scheduler_kv_cache_config([cache_config]) + assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0 + assert get_max_concurrency_for_kv_cache_config(config, cache_config) == ( + get_max_concurrency_for_kv_cache_config(config, scheduler_config) + ) + manager = KVCacheManager( + scheduler_config, + max_model_len=1024, + hash_block_size=16, + scheduler_block_size=64, + dcp_world_size=4, + pcp_world_size=pcp_size, + enable_caching=True, + ) + assert manager.coordinator.group_block_sizes == (64, 16) + request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256) + blocks = manager.allocate_slots(request, 65) + assert blocks is not None + assert [len(group) for group in blocks.blocks] == [2, 5] + manager.cache_blocks(request, 64) + cached_request = make_request( + "cached-draft", [1] * 65, block_size=16, hash_fn=sha256 + ) + cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request) + assert num_cached == 64 + assert [len(group) for group in cached_blocks.blocks] == [1, 4] + + +@pytest.mark.parametrize("use_mla", [False, True]) +def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla): + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + draft = replace( + new_mla_spec() if use_mla else new_kv_cache_spec(), + block_size=16, + dcp_sharded=False, + ) + merged = type(draft).merge([draft, draft]) + wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged}) + assert wrapped is not None and not wrapped.dcp_sharded + assert merged.max_num_blocks_per_req(config, 1024) == 64 + assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes + assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16 + with pytest.raises(AssertionError): + type(draft).merge([draft, replace(draft, dcp_sharded=True)]) + + +@pytest.mark.parametrize("with_draft", [False, True]) +@pytest.mark.parametrize("indexer_alignment", [None, 512]) +def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment): + from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view + from vllm.v1.worker.utils import allocate_kv_cache + + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.cache_config.kv_cache_layout = "BLHNC" + config.parallel_config.decode_context_parallel_size = 4 + common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8) + specs = { + "target": MLAAttentionSpec( + **common, + head_size=576, + state_content_bytes=656, + block_stride_alignment=656, + cache_dtype_str="fp8_ds_mla", + is_index_group_leader=True, + ), + "indexer": MLAAttentionSpec( + **common, + head_size=132, + cache_role=SparseCacheRole.INDEXER, + block_stride_alignment=indexer_alignment, + ), + "draft": SlidingWindowSpec( + block_size=64, + num_kv_heads=64, + head_size=64, + dtype=torch.bfloat16, + sliding_window=2048, + dcp_sharded=False, + ), + } + if not with_draft: + del specs["draft"] + groups = get_kv_cache_groups(config, specs) + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=8 * 1024 * 1024 + ) + caches = allocate_kv_cache( + cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups) + ) + cache = caches["target"].squeeze(1) + if indexer_alignment is not None: + assert caches["indexer"].stride(0) % indexer_alignment == 0 + rows, stride_rows = flat_kv_row_view(cache, 64) + cache[1, 0].fill_(7) + torch.testing.assert_close(rows[stride_rows], cache[1, 0]) + assert torch.all(rows[stride_rows] == 7) @pytest.mark.parametrize( @@ -3784,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs(): "layer_1": before_spec_1, "layer_2": before_spec_2, } + kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False) + kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False) kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) expected_spec_1 = new_kv_cache_spec(block_size=64) expected_spec_2 = new_kv_cache_spec( @@ -3792,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs(): assert kv_cache_spec["layer_1"] == expected_spec_1 assert kv_cache_spec["layer_2"] == expected_spec_2 assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024 + assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False) # 2. has_full_attention and has_chunked_local_attention before_spec_1 = new_kv_cache_spec() @@ -3843,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs(): with pytest.raises(ValueError): kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + # Replicated Mamba state still requires the hybrid cache manager. + kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()} + with pytest.raises(ValueError): + kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + def test_unify_kv_cache_spec_page_size_mamba(): """Regression test for https://github.com/vllm-project/vllm/issues/43626. diff --git a/tests/v1/core/test_kv_sharing.py b/tests/v1/core/test_kv_sharing.py index ca8787e8d8..4a030c674b 100644 --- a/tests/v1/core/test_kv_sharing.py +++ b/tests/v1/core/test_kv_sharing.py @@ -16,38 +16,6 @@ def new_kv_cache_spec(): ) -def test_initialize_kv_cache_for_kv_sharing_different_attn_groups(): - """Test initializing KV cache sharing with different attention groups. - Layers in the same KV cache group might be placed in different attn groups - if they have different attention backends. - """ - shared_kv_cache_layers = { - "model.layers.2": "model.layers.0", - "model.layers.3": "model.layers.1", - } - - # Layers 0 and 1 both belong in KV cache group 0 - # However, if they have different attention backends, they will be - # placed in different attention groups for KV cache group 0 - kv_cache_groups = [ - KVCacheGroupSpec(["model.layers.0", "model.layers.1"], new_kv_cache_spec()), - ] - - add_kv_sharing_layers_to_kv_cache_groups( - shared_kv_cache_layers=shared_kv_cache_layers, - kv_cache_groups=kv_cache_groups, - ) - - # Check that the layers were added to the correct KV cache group - assert len(kv_cache_groups) == 1 - assert kv_cache_groups[0].layer_names == [ - "model.layers.0", - "model.layers.1", - "model.layers.2", - "model.layers.3", - ] - - def test_initialize_kv_cache_for_kv_sharing_same_attn_groups(): """Test case assuming that all layers in the same KV cache group have the same attention backends. This is true for most models. diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 0a34a6131d..99d3235006 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types( ) -def test_prefix_cache_hit_uses_per_group_dcp_geometry(): +@pytest.mark.parametrize("draft_sharded", [True, False]) +def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded): """Prefix lookup must use each group's DCP size, not the process-wide one. - Target and draft MLA are both sharded (DCP=8); Mamba stays replicated - (DCP=1). Hits then align to the sharded full-attention block, not the - unsharded page size. + Target MLA is sharded; draft MLA can opt out and Mamba stays replicated. + Hits must align to the target block in either case. """ block_size = 16 dcp = 8 @@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): KVCacheGroupSpec( ["draft_mla"], MLAAttentionSpec( + dcp_sharded=draft_sharded, block_size=block_size, num_kv_heads=1, head_size=1, @@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): ) target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers assert target_mgr.dcp_world_size == dcp - assert draft_mgr.dcp_world_size == dcp + assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1) assert mamba_mgr.dcp_world_size == 1 assert target_mgr.block_size == sharded_block - assert draft_mgr.block_size == sharded_block + assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size) assert mamba_mgr.block_size == block_size hash_fn = sha256 @@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn) computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 2 * sharded_block - assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16] + assert [len(group) for group in computed_blocks.blocks] == [ + 2, + 2 if draft_sharded else 16, + 16, + ] manager.free(req0) manager.free(req1) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index e31163923b..505eef5919 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -40,6 +40,10 @@ from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash +from vllm.v1.core.sched.diffusion_scheduler import ( + DiffusionAsyncScheduler, + diffusion_canvas_width, +) from vllm.v1.core.sched.interface import PauseState from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler @@ -792,6 +796,26 @@ def test_long_prefill_threshold_applies_with_other_requests(): assert output.num_scheduled_tokens[short_req.request_id] == 10 +def test_long_prefill_threshold_floored_by_fair_share(): + """With the adaptive flag, the effective threshold never falls below the + fair share of the token budget: max_num_batched_tokens / num queued + + running requests.""" + scheduler = create_scheduler( + max_num_batched_tokens=1024, + long_prefill_token_threshold=100, + long_prefill_token_threshold_adaptive=True, + ) + long_req = create_requests(num_requests=1, num_tokens=2000)[0] + short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0] + for request in [long_req, short_req]: + scheduler.add_request(request) + + output = scheduler.schedule() + # 100 is below the fair share (1024 // 2 = 512), so the floor binds. + assert output.num_scheduled_tokens[long_req.request_id] == 512 + assert output.num_scheduled_tokens[short_req.request_id] == 10 + + def test_update_from_output_routes_sampling_masks_by_request(): """Each request receives the sampler row at its own batch index.""" scheduler = create_scheduler() @@ -3872,6 +3896,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.finished_req_ids_dict = None scheduler.aux_output_connector = None scheduler.grammar_compile_error_reqs = set() + scheduler.encoder_cache_mismatch_reqs = set() scheduler.vllm_config = Mock() scheduler.return_sampling_mask = False scheduler.recompute_kv_load_failures = False @@ -6065,6 +6090,201 @@ def test_unavailable_encoder_input_fails_the_request_as_retryable(): assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR] +def _create_mm_request_with_embedding_positions( + request_id: str, identifier: str, is_embed: list[bool] +) -> Request: + return create_requests( + num_requests=1, + num_tokens=len(is_embed), + mm_hashes_list=[[identifier]], + mm_positions=[ + [ + PlaceholderRange( + offset=0, + length=len(is_embed), + is_embed=torch.tensor(is_embed), + ) + ] + ], + req_ids=[request_id], + )[0] + + +def test_encoder_cache_accepts_matching_embed_count(): + """An actively referenced entry is reused when embedding counts match.""" + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + identifier = "reused-client-uuid" + owner = _create_mm_request_with_embedding_positions( + "owner", identifier, [True, True, True, False, False, False] + ) + compatible = _create_mm_request_with_embedding_positions( + "compatible", identifier, [True, False, True, False, True, False, False] + ) + scheduler.encoder_cache_manager.allocate(owner, 0) + scheduler.add_request(compatible) + + scheduler_output = scheduler.schedule() + + assert compatible.request_id in scheduler_output.num_scheduled_tokens + assert compatible.request_id not in scheduler.encoder_cache_mismatch_reqs + assert compatible.request_id not in scheduler_output.scheduled_encoder_inputs + + +def test_encoder_cache_rejects_mismatched_embed_count(caplog_vllm): + """A completed request's freeable entry rejects an incompatible reuse.""" + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + identifier = "reused-client-uuid" + owner = _create_mm_request_with_embedding_positions( + "owner", identifier, [True, True, True, False, False, False] + ) + mismatched = _create_mm_request_with_embedding_positions( + "mismatched", identifier, [True, True, True, True, False, False] + ) + scheduler.encoder_cache_manager.allocate(owner, 0) + scheduler.encoder_cache_manager.free(owner) + freeable_slots_before = scheduler.encoder_cache_manager.num_freeable_slots + scheduler.add_request(mismatched) + + scheduler_output = scheduler.schedule() + assert not scheduler_output.num_scheduled_tokens + assert mismatched.request_id in scheduler.encoder_cache_mismatch_reqs + assert ( + "multimodal identifier 'reused-client-uuid' holds 3 embeddings" + in caplog_vllm.text + and "expects 4" in caplog_vllm.text + ) + + outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput(req_ids=[], req_id_to_index={}), + ) + + assert mismatched.status == RequestStatus.FINISHED_ERROR + assert not scheduler.encoder_cache_mismatch_reqs + assert mismatched.request_id not in scheduler.requests + assert scheduler.encoder_cache_manager.cached[identifier] == set() + assert ( + mismatched.request_id not in scheduler.encoder_cache_manager.request_cached_ids + ) + assert scheduler.encoder_cache_manager.freeable[identifier] == 3 + assert scheduler.encoder_cache_manager.num_freeable_slots == freeable_slots_before + output = outputs[mismatched.client_index].outputs[0] + assert output.request_id == mismatched.request_id + assert output.finish_reason == FinishReason.ERROR + + healthy = create_requests(num_requests=1, req_ids=["healthy"], num_tokens=6)[0] + scheduler.add_request(healthy) + next_output = scheduler.schedule() + assert healthy.request_id in next_output.num_scheduled_tokens + + +def test_encoder_cache_embed_count_mismatch_restores_prior_hit_state(): + """A prior hit is released when a later mismatch fails the request.""" + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + manager = scheduler.encoder_cache_manager + valid_identifier = "valid-reuse" + mismatch_identifier = "mismatched-reuse" + valid_owner = _create_mm_request_with_embedding_positions( + "valid-owner", valid_identifier, [True, True, False] + ) + mismatch_owner = _create_mm_request_with_embedding_positions( + "mismatch-owner", mismatch_identifier, [True, True, True, False] + ) + manager.allocate(valid_owner, 0) + manager.free(valid_owner) + manager.allocate(mismatch_owner, 0) + manager.free(mismatch_owner) + freeable_slots_before = manager.num_freeable_slots + + request = create_requests( + num_requests=1, + num_tokens=7, + mm_hashes_list=[[valid_identifier, mismatch_identifier]], + mm_positions=[ + [ + PlaceholderRange( + offset=0, + length=3, + is_embed=torch.tensor([True, True, False]), + ), + PlaceholderRange( + offset=3, + length=4, + is_embed=torch.tensor([True, True, True, True]), + ), + ] + ], + req_ids=["partially-mutated"], + )[0] + scheduler.add_request(request) + + scheduler_output = scheduler.schedule() + assert not scheduler_output.num_scheduled_tokens + assert request.request_id in scheduler.encoder_cache_mismatch_reqs + assert request.request_id in manager.cached[valid_identifier] + assert valid_identifier not in manager.freeable + assert manager.num_freeable_slots == freeable_slots_before - 2 + + outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput(req_ids=[], req_id_to_index={}), + ) + + assert request.status == RequestStatus.FINISHED_ERROR + assert request.request_id not in scheduler.requests + assert request.request_id not in manager.request_cached_ids + assert manager.cached[valid_identifier] == set() + assert manager.freeable[valid_identifier] == 2 + assert manager.freeable[mismatch_identifier] == 3 + assert manager.num_freeable_slots == freeable_slots_before + output = outputs[request.client_index].outputs[0] + assert output.finish_reason == FinishReason.ERROR + + +def test_encoder_cache_rejects_mismatched_embed_count_within_request(): + """One request cannot reuse an identifier with a different embed count.""" + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + identifier = "reused-within-request" + request = create_requests( + num_requests=1, + num_tokens=8, + mm_hashes_list=[[identifier, identifier]], + mm_positions=[ + [ + PlaceholderRange( + offset=0, + length=4, + is_embed=torch.tensor([True, True, False, False]), + ), + PlaceholderRange( + offset=4, + length=4, + is_embed=torch.tensor([True, True, True, False]), + ), + ] + ], + req_ids=["mismatched-within-request"], + )[0] + scheduler.add_request(request) + + scheduler_output = scheduler.schedule() + assert not scheduler_output.num_scheduled_tokens + assert request.request_id in scheduler.encoder_cache_mismatch_reqs + assert identifier not in scheduler.encoder_cache_manager.cached + + outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput(req_ids=[], req_id_to_index={}), + ) + + assert request.status == RequestStatus.FINISHED_ERROR + assert not scheduler.encoder_cache_mismatch_reqs + assert request.request_id not in scheduler.requests + output = outputs[request.client_index].outputs[0] + assert output.request_id == request.request_id + assert output.finish_reason == FinishReason.ERROR + + def test_free_encoder_inputs_defers_for_eagle_lookahead(): """With EAGLE speculative decoding, the encoder input is retained one extra position so the drafter's +1 look-ahead mm-embedding gather (which reads one @@ -6872,3 +7092,141 @@ def test_update_draft_token_ids_in_output_strips_padding(): -1, ] assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2} + + +def test_diffusion_canvas_width_defaults_to_the_served_canvas(): + def req(extra): + return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra)) + + assert diffusion_canvas_width(req(None), 64) == 64 + assert diffusion_canvas_width(req({}), 64) == 64 + assert diffusion_canvas_width(req({"diffusion_canvas_length": 16}), 64) == 16 + assert diffusion_canvas_width(SimpleNamespace(sampling_params=None), 64) == 64 + + +def _diffusion_request(req_id: str, extra_args: dict) -> Request: + (request,) = create_requests( + num_requests=1, num_tokens=8, max_tokens=64, req_ids=[req_id] + ) + request.sampling_params.extra_args = extra_args + return request + + +@pytest.fixture +def diffusion_model_runner(monkeypatch): + # These CPU tests only exercise scheduling, not Triton kernels. + monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True) + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + +def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler: + scheduler = create_scheduler( + async_scheduling=True, + diffusion_canvas_length=8, + scheduler_cls=DiffusionAsyncScheduler, + **kwargs, + ) + assert isinstance(scheduler, DiffusionAsyncScheduler) + return scheduler + + +@pytest.mark.parametrize("async_scheduling", [False, True]) +@pytest.mark.usefixtures("diffusion_model_runner") +def test_diffusion_scheduler_is_selected_by_default(async_scheduling): + config = create_scheduler( + async_scheduling=async_scheduling, diffusion_canvas_length=8 + ).vllm_config.scheduler_config + assert config.get_scheduler_cls() is ( + DiffusionAsyncScheduler if config.async_scheduling else Scheduler + ) + + +@pytest.mark.usefixtures("diffusion_model_runner") +def test_diffusion_scheduler_narrows_the_canvas_per_request(): + scheduler = _diffusion_scheduler() + wide = _diffusion_request("wide", {}) + narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4}) + scheduler.add_request(wide) + scheduler.add_request(narrow) + + # The prefill step lays down the served canvas as placeholders. + scheduler.schedule() + output = scheduler.schedule() + assert output.scheduled_spec_decode_tokens["wide"] == [-1] * 8 + assert output.scheduled_spec_decode_tokens["narrow"] == [-1] * 4 + assert output.num_scheduled_tokens["narrow"] == 4 + assert narrow.num_output_placeholders == 4 + + +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.usefixtures("diffusion_model_runner") +def test_diffusion_scheduler_trims_full_width_worker_drafts(structured): + """Padded worker drafts must be narrowed before scheduling or grammar validation.""" + scheduler = _diffusion_scheduler() + wide = _diffusion_request("wide", {}) + narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4}) + for request in (wide, narrow): + scheduler.add_request(request) + prefill = scheduler.schedule() + _model_output(scheduler, prefill, [[], []]) + + if structured: + for request in (wide, narrow): + request.structured_output_request = SimpleNamespace( + grammar=_RecordingGrammar(), reasoning_ended=True + ) + tokens = list(range(8)) if structured else [-1] * 8 + drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()]) + output = scheduler.schedule() + scheduler.update_draft_token_ids_in_output(drafts, output) + + assert output.scheduled_spec_decode_tokens == { + "wide": tokens, + "narrow": tokens[:4], + } + assert output.num_scheduled_tokens == {"wide": 8, "narrow": 4} + if structured: + assert wide.structured_output_request.grammar.seen == [tokens] + assert narrow.structured_output_request.grammar.seen == [tokens[:4]] + + +@pytest.mark.usefixtures("diffusion_model_runner") +def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight(): + scheduler = _diffusion_scheduler() + one = _diffusion_request( + "one", {"diffusion_read_only": True, "diffusion_max_steps": 1} + ) + two = _diffusion_request( + "two", {"diffusion_read_only": True, "diffusion_max_steps": 2} + ) + # Only reads are deferred. A capped generation keeps its extra async step. + gen = _diffusion_request("gen", {"diffusion_max_steps": 1}) + for request in (one, two, gen): + scheduler.add_request(request) + + scheduler.schedule() + assert set(scheduler.schedule().num_scheduled_tokens) == {"one", "two", "gen"} + assert one.num_output_placeholders == 8 + assert set(scheduler.schedule().num_scheduled_tokens) == {"two", "gen"} + assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"} + # The deferral is renewed every call. + assert set(scheduler.schedule().num_scheduled_tokens) == {"gen"} + + +@pytest.mark.usefixtures("diffusion_model_runner") +def test_diffusion_read_deferral_keeps_a_longer_pp_wait(): + scheduler = _diffusion_scheduler(pipeline_parallel_size=3, use_v2_model_runner=True) + read = _diffusion_request( + "read", {"diffusion_read_only": True, "diffusion_max_steps": 1} + ) + scheduler.add_request(read) + + scheduler.schedule() + assert read.next_decode_eligible_step == 4 + for _ in range(2): + assert "read" not in scheduler.schedule().num_scheduled_tokens + assert "read" in scheduler.schedule().num_scheduled_tokens + assert read.next_decode_eligible_step == 7 + # Deferring this step alone would ask for 6. The PP wait to 7 stands. + assert "read" not in scheduler.schedule().num_scheduled_tokens + assert read.next_decode_eligible_step == 7 diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index f69b10ced3..46ca29f67b 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -7,6 +7,7 @@ import vllm.envs as envs from tests.v1.kv_connector.unit.utils import MockKVConfig from vllm.config import ( CacheConfig, + DiffusionConfig, ECTransferConfig, KVTransferConfig, ModelConfig, @@ -57,6 +58,7 @@ def create_scheduler( enable_chunked_prefill: bool = True, enable_prefix_caching: bool = False, long_prefill_token_threshold: int = 0, + long_prefill_token_threshold_adaptive: bool = False, disable_chunked_mm_input: bool = False, use_kv_connector: None | bool | str | MockKVConfig = None, kv_role: str = "kv_both", @@ -77,6 +79,8 @@ def create_scheduler( kv_cache_spec: KVCacheSpec | None = None, per_request_spec_decode_metrics: str = "none", scheduling_policy: SchedulerPolicy = "fcfs", + diffusion_canvas_length: int | None = None, + scheduler_cls: type[Scheduler] | None = None, ) -> Scheduler | AsyncScheduler: """Create scheduler under test. @@ -87,6 +91,9 @@ def create_scheduler( enable_prefix_caching: optionally force APC config (True/False) or use default (False) + long_prefill_token_threshold: cap on prefill chunk size + long_prefill_token_threshold_adaptive: floor the cap at a + fair share of the token budget Returns: {class}`Scheduler` instance @@ -112,6 +119,7 @@ def create_scheduler( max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len, long_prefill_token_threshold=long_prefill_token_threshold, + long_prefill_token_threshold_adaptive=(long_prefill_token_threshold_adaptive), disable_chunked_mm_input=disable_chunked_mm_input, enable_chunked_prefill=enable_chunked_prefill, async_scheduling=async_scheduling, @@ -178,6 +186,12 @@ def create_scheduler( else None ) + diffusion_config: DiffusionConfig | None = None + if diffusion_canvas_length is not None: + # A diffusion checkpoint declares its canvas in the HF config. + model_config.hf_config.canvas_length = diffusion_canvas_length + diffusion_config = DiffusionConfig(canvas_length=diffusion_canvas_length) + vllm_config = VllmConfig( scheduler_config=scheduler_config, model_config=model_config, @@ -188,6 +202,7 @@ def create_scheduler( ), kv_transfer_config=kv_transfer_config, speculative_config=speculative_config, + diffusion_config=diffusion_config, ec_transfer_config=ec_transfer_config, observability_config=ObservabilityConfig( per_request_spec_decode_metrics=per_request_spec_decode_metrics, @@ -207,7 +222,8 @@ def create_scheduler( ) cache_config.num_gpu_blocks = num_blocks register_all_kvcache_specs(vllm_config) - scheduler_cls = AsyncScheduler if async_scheduling else Scheduler + if scheduler_cls is None: + scheduler_cls = AsyncScheduler if async_scheduling else Scheduler scheduler = scheduler_cls( vllm_config=vllm_config, kv_cache_config=kv_cache_config, diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index c907be36b8..d46476f8d0 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -22,6 +22,9 @@ from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.input_batch import InputBuffers from vllm.v1.worker.gpu.pcp_manager import PCPManager +from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( + SpeculatorCudaGraphManager, +) pytestmark = pytest.mark.cpu_test @@ -165,6 +168,89 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): block_tables.get_dummy_slot_mappings.assert_not_called() +@pytest.mark.parametrize( + "full_cudagraph,max_query_len,expected_max_query_len", + [(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)], +) +def test_capture_query_length_covers_replay_batches( + full_cudagraph, max_query_len, expected_max_query_len +): + """Mixed FULL graphs must capture prefill kernels as well as decode kernels.""" + num_tokens = num_reqs = 8 + buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) + block_tables = MagicMock() + block_tables.cp_size = 1 + block_tables.get_dummy_block_tables.return_value = () + block_tables.get_dummy_slot_mappings.return_value = torch.empty( + 0, num_tokens, dtype=torch.int64 + ) + model_state = MagicMock() + kv_cache_config = KVCacheConfig( + num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[] + ) + + gpu_cudagraph_utils.prepare_inputs_to_capture( + num_reqs, + num_tokens, + model_state, + buffers, + block_tables, + [], + kv_cache_config, + full_cudagraph=full_cudagraph, + max_query_len=max_query_len, + ) + + batch = model_state.prepare_attn.call_args.args[0] + assert batch.num_scheduled_tokens.max() == 1 + assert batch.max_query_len == expected_max_query_len + + +@pytest.mark.parametrize( + "uniform_token_count,max_query_len,expected_max_query_len", + [(1, None, 1), (None, 3, 3), (None, None, 8)], +) +def test_speculator_capture_preserves_decode_query_bounds( + monkeypatch, uniform_token_count, max_query_len, expected_max_query_len +): + """Draft decode graphs keep narrow queries while mixed graphs cover prefill.""" + num_tokens = num_reqs = 8 + desc = BatchExecutionDescriptor( + CUDAGraphMode.FULL, + num_tokens, + num_reqs, + uniform_token_count=uniform_token_count, + max_query_len=max_query_len, + ) + manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager) + manager.max_num_reqs = num_reqs + manager.dp_size = 1 + buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) + block_tables = MagicMock() + block_tables.cp_size = 1 + block_tables.get_dummy_block_tables.return_value = () + block_tables.get_dummy_slot_mappings.return_value = torch.empty( + 0, num_tokens, dtype=torch.int64 + ) + model_state = MagicMock() + kv_cache_config = KVCacheConfig( + num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[] + ) + + def capture(self, create_forward_fn, progress_bar_desc): + for warmup in (True, False): + create_forward_fn(desc, warmup)(desc.cg_mode) + + monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture) + manager.capture( + MagicMock(), model_state, buffers, block_tables, [], kv_cache_config + ) + + assert model_state.prepare_attn.call_count == 2 + for call in model_state.prepare_attn.call_args_list: + assert call.args[0].max_query_len == expected_max_query_len + + _DECODE_QUERY_LEN = 3 @@ -208,6 +294,7 @@ def _make_spec_decode_manager( dynamic_spec_schedule: list[tuple[int, int, int]] | None = None, max_num_seqs: int = 8, use_kda_recoverssm: bool = False, + varlen_decode: bool = False, ) -> gpu_cudagraph_utils.CudaGraphManager: monkeypatch.setattr( gpu_cudagraph_utils, @@ -230,6 +317,7 @@ def _make_spec_decode_manager( device=torch.device("cpu"), cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, decode_query_len=decode_query_len, + varlen_decode=varlen_decode, ) manager._graphs_captured = True return manager @@ -287,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch): assert desc.num_tokens == 16 +def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch): + """A varlen decode graph replays any decode batch within its query-length + bound. The runner passes no bound for a batch with a prefill, which must not + replay one however short the prefill is.""" + manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True) + + def dispatch(max_query_len): + return manager.dispatch( + num_reqs=3, + num_tokens=6, + uniform_token_count=None, + num_active_loras=0, + max_query_len=max_query_len, + ) + + desc = dispatch(_DECODE_QUERY_LEN) + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.max_query_len == _DECODE_QUERY_LEN + assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE + assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE + + def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch): """A mixed batch must not fall to eager where only decode graphs are staged. diff --git a/tests/v1/cudagraph/test_cudagraph_mode.py b/tests/v1/cudagraph/test_cudagraph_mode.py index e413fd91e3..0147f90df5 100644 --- a/tests/v1/cudagraph/test_cudagraph_mode.py +++ b/tests/v1/cudagraph/test_cudagraph_mode.py @@ -54,13 +54,16 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte llm = LLM( model="Qwen/Qwen2-1.5B-Instruct", - max_num_seqs=256, + max_num_seqs=16, + max_num_batched_tokens=128, trust_remote_code=True, gpu_memory_utilization=0.45, max_model_len=1024, attention_config=attention_config, compilation_config=CompilationConfig( - mode=CompilationMode.VLLM_COMPILE, cudagraph_mode=cudagraph_mode + mode=CompilationMode.VLLM_COMPILE, + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=[1, 2, 4, 8, 16, 64, 128], ), ) llm.generate(["Hello, my name is"] * 10) @@ -72,11 +75,9 @@ attn_backend = "RocmAttn" if current_platform.is_rocm() else "FA2" combo_cases_2 = [ (attn_backend, "FULL", CompilationMode.NONE, True), - (attn_backend, "FULL", CompilationMode.VLLM_COMPILE, True), (attn_backend, "PIECEWISE", CompilationMode.NONE, True), (attn_backend, "PIECEWISE", CompilationMode.VLLM_COMPILE, True), (attn_backend, "FULL_AND_PIECEWISE", CompilationMode.NONE, True), - (attn_backend, "FULL_AND_PIECEWISE", CompilationMode.VLLM_COMPILE, True), (attn_backend, "FULL_DECODE_ONLY", CompilationMode.NONE, True), (attn_backend, "FULL_DECODE_ONLY", CompilationMode.VLLM_COMPILE, True), (attn_backend, "NONE", CompilationMode.NONE, True), @@ -100,13 +101,16 @@ def test_cudagraph_compilation_combo( llm = LLM( model="Qwen/Qwen2-1.5B-Instruct", - max_num_seqs=256, + max_num_seqs=16, + max_num_batched_tokens=128, trust_remote_code=True, gpu_memory_utilization=0.45, max_model_len=1024, attention_config=attention_config, compilation_config=CompilationConfig( - mode=compilation_mode, cudagraph_mode=cudagraph_mode + mode=compilation_mode, + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=[1, 2, 4, 8, 16, 64, 128], ), ) llm.generate(["Hello, my name is"] * 10) diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index e370f87833..9af55b9fbb 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -6,6 +6,7 @@ import random import pytest import torch +from transformers import AutoTokenizer from utils import ( BACKENDS, TEST_MODEL, @@ -203,13 +204,8 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN( # Use more realistic prompts for better token generation prompts = [_random_prompt(10, 50) for _ in range(32)] - # TODO: Update prompts to have ragged lengths in order to test chunked prefill - # The above tests are not currently long enough to exercise chunking. - # prompts = ( - # [_random_prompt(10, 50) for _ in range(28)] - # + [_random_prompt(256, 512) for _ in range(50)] - # + [_random_prompt(2048, 4096) for _ in range(50)] - # ) + # Chunked prefill is covered by + # test_logprobs_bitwise_batch_invariance_ragged_chunked_prefill below. sp = SamplingParams( temperature=0.6, @@ -389,6 +385,80 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN( pytest.fail(msg) +@skip_unsupported +@pytest.mark.parametrize( + "backend", + BACKENDS, +) +def test_logprobs_bitwise_batch_invariance_ragged_chunked_prefill(backend): + """Batch invariance must hold when a prefill is split across chunks.""" + random.seed(int(os.getenv("VLLM_TEST_SEED", "12345"))) + + prompts = ( + [_random_prompt(10, 50) for _ in range(8)] + + [_random_prompt(300, 450) for _ in range(4)] + + [_random_prompt(800, 1200) for _ in range(2)] + ) + random.shuffle(prompts) + + # _random_prompt takes a word target, not a token count, and the ratio + # depends on the tokenizer, so derive the budget rather than hardcode it: + # half the longest prompt splits that prefill whatever TEST_MODEL is, while + # the shorter prompts are still co-scheduled whole. Floor is max_num_seqs. + tokenizer = AutoTokenizer.from_pretrained(TEST_MODEL) + prompt_lens = [len(tokenizer(p).input_ids) for p in prompts] + max_num_batched_tokens = max(len(prompts), max(prompt_lens) // 2) + assert max_num_batched_tokens < max(prompt_lens), ( + f"a {max_num_batched_tokens}-token budget does not split the longest " + f"prompt ({max(prompt_lens)} tokens), so nothing would be chunked" + ) + + sp = SamplingParams( + temperature=0.6, top_p=1.0, max_tokens=16, seed=1234, logprobs=5 + ) + llm = LLM_with_max_seqs( + model=TEST_MODEL, + max_num_seqs=len(prompts), + gpu_memory_utilization=float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.5")), + max_model_len=4096, + attention_config={"backend": backend}, + max_num_batched_tokens=max_num_batched_tokens, + ) + + try: + bs1 = [] + for p in prompts: + out = llm.generate([p], sp, use_tqdm=False)[0] + bs1.append(_extract_step_logprobs(out)) + if any(logprobs is None for logprobs, _ in bs1): + pytest.skip("Logprobs are not available on RequestOutput.") + + outs_batched = llm.generate(prompts, sp, use_tqdm=False) + assert len(outs_batched) == len(prompts) + + mismatches = [] + for i, out in enumerate(outs_batched): + logprobs, tokens = _extract_step_logprobs(out) + bs1_logprobs, bs1_tokens = bs1[i] + n = prompt_lens[i] + if tokens != bs1_tokens: + mismatches.append(f"prompt {i} ({n} tokens): tokens differ") + elif not torch.equal(bs1_logprobs, logprobs): + delta = torch.max(torch.abs(bs1_logprobs - logprobs)).item() + mismatches.append( + f"prompt {i} ({n} tokens): logprobs differ, max|delta|={delta:.3e}" + ) + + assert not mismatches, ( + "Batch invariance violated under chunked prefill " + f"(max_num_batched_tokens={max_num_batched_tokens}): " + + "; ".join(mismatches) + ) + finally: + with contextlib.suppress(Exception): + llm.shutdown() + + @skip_unsupported @pytest.mark.parametrize( "backend", @@ -923,6 +993,7 @@ def LLM_with_max_seqs( max_model_len: int, attention_config: dict | None = None, kernel_config: dict | None = None, + max_num_batched_tokens: int | None = None, ) -> LLM: """Helper to construct an LLM with a specific max_num_seqs (batch-size limit) using the high-level v1 LLM API, while constraining memory usage. @@ -930,6 +1001,8 @@ def LLM_with_max_seqs( extra_kwargs: dict = {} if kernel_config is not None: extra_kwargs["kernel_config"] = kernel_config + if max_num_batched_tokens is not None: + extra_kwargs["max_num_batched_tokens"] = max_num_batched_tokens return LLM( model=model, max_num_seqs=max_num_seqs, diff --git a/tests/v1/e2e/general/test_sharded_sampling.py b/tests/v1/e2e/general/test_sharded_sampling.py index a580605fb0..8686186776 100644 --- a/tests/v1/e2e/general/test_sharded_sampling.py +++ b/tests/v1/e2e/general/test_sharded_sampling.py @@ -2,26 +2,30 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Batch-sharded sampling must be a drop-in for replicated sampling. -The comparison is tolerance-based, not bitwise: engine boots land in -slightly different kernel/collective states (measured: same-state boots are -bitwise identical across the two modes; cross-state boots differ by up to -~0.25 logprob and can flip a near-tie token). A real sharding bug — rows -routed to the wrong request, misaligned gathers — produces many-nats -divergence and scrambled top-k sets, far outside these bounds.""" +The two boots are compared token for token, which is only meaningful because +both schedule the same batches. With the engine in the caller's process every +request is queued before the first step, so the first prefill has the same +shape in both runs. Give the engine its own process and the comparison turns +into a coin toss: prefill starts before the last requests arrive over IPC, so +one boot batches two requests where the other batches one. The batch shape +sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties +flip — measured at up to 5 of 16 completions, with sharding disabled on both +sides. A real sharding bug — rows routed to the wrong request, misaligned +gathers — produces many-nats divergence and scrambled top-k sets.""" import pytest from tests.utils import multi_gpu_test from vllm import LLM, SamplingParams from vllm.distributed import cleanup_dist_env_and_memory +from vllm.logprobs import Logprob, LogprobsOnePosition MODEL = "meta-llama/Llama-3.2-1B-Instruct" -# Boot-state numeric noise bound (measured <= 0.25 on identical contexts). +# Misrouted rows move logprobs by nats; anything under this is float noise. +# With matched batches the two boots agreed bitwise over 320 positions, so the +# bound is not expected to be reached. LOGPROB_TOL = 0.5 -# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per -# engine boot across ~16 completions, plus one of headroom. -MAX_DIVERGENT_PROMPTS = 3 PROMPTS = [ "The capital of France is", @@ -54,6 +58,10 @@ def _is_sharded_sampling_active(worker) -> bool: return worker.model_runner.batch_sharder is not None +def _sampling_params() -> list[SamplingParams]: + return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))] + + def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): llm = LLM( model=MODEL, @@ -75,9 +83,7 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): # intended sampling mode. modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active) assert all(mode == (not disable_sharding) for mode in modes), modes - params = [ - SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS)) - ] + params = _sampling_params() # Two waves: the second reuses the request slots freed by the first # (in whatever order requests finished), covering the cross-rank # slot-recycling path that request ownership derives from. @@ -89,13 +95,36 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): cleanup_dist_env_and_memory() -def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None: - # Allow one boundary entry of the top-k set to swap at a near-tie. - common = set(ref_lps) & set(out_lps) - assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, ( - f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}" +def _requested_top_k(lps: LogprobsOnePosition, num_logprobs: int) -> set[int]: + """Exclude the extra sampled/prompt token, including at rank k+1.""" + top = { + token_id + for token_id, logprob in lps.items() + if logprob.rank is not None and 1 <= logprob.rank <= num_logprobs + } + assert len(top) == num_logprobs, ( + f"expected {num_logprobs} ranked top-k entries, got {len(top)}" ) - for token_id in common: + return top + + +def _assert_logprob_dicts_close( + ref_lps: LogprobsOnePosition, + out_lps: LogprobsOnePosition, + what: str, + num_logprobs: int, +) -> None: + # Allow one boundary entry of the top-k set to swap at a near-tie. Compare + # the top-k alone: counting the sampled token would spend that allowance + # on the extra entry itself whenever only one side carries one. + ref_top = _requested_top_k(ref_lps, num_logprobs) + out_top = _requested_top_k(out_lps, num_logprobs) + shared_top = ref_top & out_top + assert len(shared_top) >= num_logprobs - 1, ( + f"{what}: top-k token sets diverge: {sorted(ref_top)} vs {sorted(out_top)}" + ) + # Values are comparable for every shared token, sampled one included. + for token_id in set(ref_lps) & set(out_lps): diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob) assert diff <= LOGPROB_TOL, ( f"{what}[{token_id}]: logprob diff {diff} " @@ -103,20 +132,72 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None: ) +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("num_logprobs", "ref_ranks", "out_ranks"), + [ + (2, {0: 1, 1: 2, 2: 3}, {0: 1, 3: 2, 2: 4}), + (1, {0: 1, 1: 2}, {2: 1, 1: 3}), + (2, {0: 1, 1: 2}, {0: 1, 2: 2, 1: 3}), + (2, {0: 1, 1: 2, 2: 4}, {0: 1, 1: 2, 2: 5}), + (0, {0: 2}, {0: 3}), + ], + ids=["decode-boundary", "prompt-boundary", "sampled-in-top-k", "gap", "zero"], +) +def test_logprob_comparison_allows_sampled_token_rank_changes( + num_logprobs: int, ref_ranks: dict[int, int], out_ranks: dict[int, int] +): + ref_lps, out_lps = ( + {token: Logprob(-2.0 - rank / 100, rank=rank) for token, rank in ranks.items()} + for ranks in (ref_ranks, out_ranks) + ) + _assert_logprob_dicts_close(ref_lps, out_lps, "boundary swap", num_logprobs) + + +@pytest.mark.cpu_test +def test_logprob_comparison_rejects_multiple_top_k_swaps(): + ref = {0: Logprob(-2.0, rank=1), 1: Logprob(-2.01, rank=2)} + out = {2: Logprob(-2.0, rank=1), 3: Logprob(-2.01, rank=2)} + with pytest.raises(AssertionError, match="top-k token sets diverge"): + _assert_logprob_dicts_close(ref, out, "misrouted top-k", num_logprobs=2) + + +@pytest.mark.cpu_test +def test_logprob_comparison_checks_extra_sampled_token_value(): + ref = {0: Logprob(-1.0, rank=1), 1: Logprob(-3.0, rank=3)} + out = {0: Logprob(-1.0, rank=1), 1: Logprob(-4.0, rank=3)} + with pytest.raises(AssertionError, match="logprob diff"): + _assert_logprob_dicts_close(ref, out, "sampled token", num_logprobs=1) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("rank", [None, 2]) +def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None): + lps = {0: Logprob(-1.0, rank=rank)} + with pytest.raises(AssertionError, match="expected 1 ranked top-k entries"): + _assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1) + + @multi_gpu_test(num_gpus=2) def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): """Generation, logprobs, and prompt logprobs match between batch-sharded - sampling and the replicated fallback, up to measured boot-state noise.""" + sampling and the replicated fallback.""" monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") # Required for the collective_rpc mode check below. monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + # Queue every request before the first step, so both boots schedule the + # same batches -- see the module docstring. + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") ref_outputs = _generate(monkeypatch, disable_sharding=True) shard_outputs = _generate(monkeypatch, disable_sharding=False) assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS) - num_divergent = 0 - for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): + params = _sampling_params() + divergent = [] + for i, (ref, out, sampling_params) in enumerate( + zip(ref_outputs, shard_outputs, params + params[::-1]) + ): ref_completion = ref.outputs[0] out_completion = out.outputs[0] ref_ids = list(ref_completion.token_ids) @@ -131,21 +212,24 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): break prefix += 1 if ref_ids != out_ids: - num_divergent += 1 + divergent.append((i, prefix)) if ref_completion.logprobs is not None: assert out_completion.logprobs is not None + assert sampling_params.logprobs is not None comparable = min(prefix + 1, len(ref_ids), len(out_ids)) for pos in range(comparable): _assert_logprob_dicts_close( ref_completion.logprobs[pos], out_completion.logprobs[pos], f"prompt {i} logprobs[{pos}]", + sampling_params.logprobs, ) # The prompt is fixed, so every prompt-logprob position is comparable. if ref.prompt_logprobs is not None: assert out.prompt_logprobs is not None + assert sampling_params.prompt_logprobs is not None for pos, (ref_lps, out_lps) in enumerate( zip(ref.prompt_logprobs, out.prompt_logprobs) ): @@ -153,10 +237,14 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): assert ref_lps is None and out_lps is None continue _assert_logprob_dicts_close( - ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]" + ref_lps, + out_lps, + f"prompt {i} prompt_logprobs[{pos}]", + sampling_params.prompt_logprobs, ) - assert num_divergent <= MAX_DIVERGENT_PROMPTS, ( - f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie " - "boot-state noise; sharded sampling is likely misrouting requests" + assert not divergent, ( + f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at " + f"(prompt, position) {divergent}; the sampling modes did not agree " + "token for token with in-process scheduling" ) diff --git a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py index 3a474ac203..d2c2ac87b7 100644 --- a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py +++ b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py @@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace from vllm.config.kernel import MoEBackend from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform +from vllm.v1.spec_decode.draft_model import DraftModelProposer from ..utils import ( Messages, @@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism(): assert draft_config.quant_config is None +@pytest.mark.cpu_test +@pytest.mark.parametrize("dcp_size", [1, 2]) +def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size): + monkeypatch.setattr(current_platform, "device_count", lambda: 16) + target_config = EngineArgs( + model="Qwen/Qwen3-1.7B", + tensor_parallel_size=16, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + dcp_comm_backend="ag_rs", + distributed_executor_backend="mp", + speculative_config={ + "model": "Qwen/Qwen3-0.6B", + "method": "draft_model", + "num_speculative_tokens": 3, + }, + ).create_engine_config() + proposer = object.__new__(DraftModelProposer) + proposer.vllm_config = target_config + proposer.speculative_config = target_config.speculative_config + draft_config = proposer._create_draft_vllm_config() + draft_parallel = draft_config.parallel_config + assert draft_parallel.decode_context_parallel_size == dcp_size + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert draft_parallel.dcp_comm_backend == "ag_rs" + assert ( + draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate + ) + assert draft_parallel is not target_config.parallel_config + assert target_config.parallel_config.decode_context_parallel_size == dcp_size + + def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig: """Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic so we can test it without instantiating a full proposer.""" diff --git a/tests/v1/e2e/spec_decode/test_sharded_sampling.py b/tests/v1/e2e/spec_decode/test_sharded_sampling.py index cc9781ca9f..8f58b21a1a 100644 --- a/tests/v1/e2e/spec_decode/test_sharded_sampling.py +++ b/tests/v1/e2e/spec_decode/test_sharded_sampling.py @@ -39,6 +39,9 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") # Required for the collective_rpc mode check below. m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + # Queue every request before the first step, so both boots schedule + # the same batches -- see tests/v1/e2e/general/test_sharded_sampling.py. + m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") def run(disable_sharding: bool): llm = LLM( @@ -79,24 +82,22 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): ref_greedy, ref_seeded = run(disable_sharding=True) shard_greedy, shard_seeded = run(disable_sharding=False) - # Engine boots are not bitwise deterministic (kernel/collective - # selection shifts logits by ~1 ulp, flipping near-tie tokens), so - # allow a small number of divergent prompts. A sharding bug produces - # wholesale divergence, not isolated near-tie flips. + # A sharding bug shows up as wholesale divergence, not isolated flips. for name, ref_outputs, shard_outputs in ( ("greedy", ref_greedy, shard_greedy), ("seeded", ref_seeded, shard_seeded), ): - num_divergent = 0 + divergent = [] for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids): - num_divergent += 1 + divergent.append(i) print( f"{name} prompt {i} diverged:\n" f" replicated: {ref.outputs[0].text!r}\n" f" sharded: {out.outputs[0].text!r}" ) - assert num_divergent <= 2, ( - f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, " - "beyond near-tie boot noise" + assert not divergent, ( + f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged " + f"at {divergent}; the sampling modes did not agree token for " + "token with in-process scheduling" ) diff --git a/tests/v1/ec_connector/unit/test_ec_example_connector.py b/tests/v1/ec_connector/unit/test_ec_example_connector.py index a6645fccf4..725bb99eb0 100644 --- a/tests/v1/ec_connector/unit/test_ec_example_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_example_connector.py @@ -56,6 +56,22 @@ def test_metadata_preserves_repeated_item_positions(): } +def test_audio_metadata_survives_processor_cache_hit(): + resolver = Mock() + resolver.fields_for.return_value = {"audio_num_tokens"} + feature = MultiModalFeatureSpec( + data=None, + modality="audio", + identifier="audio", + mm_position=PlaceholderRange( + offset=0, length=4, is_embed=torch.tensor([False, True, True, False]) + ), + ) + assert collect_ec_item_metadata([feature], resolver) == { + "audio": {"metadata": {"audio_num_tokens": [2]}, "item_indices": [0]} + } + + @pytest.fixture def temp_storage(tmp_path): """Fixture providing temporary storage path.""" diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py index 8db22fe1b9..08bca0e40f 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -87,6 +87,82 @@ class _EncoderSession: return _EncoderResponse(self._replies.pop(0)) +@pytest.mark.parametrize("second_type", ["image_url", "audio_url", "input_audio"]) +@pytest.mark.parametrize("transfer", ["example", "nixl", "mooncake"]) +def test_partial_audio_metadata_falls_back_with_all_transfers( + proxy, monkeypatch, second_type, transfer +): + """Never mix metadata-only audio with raw media, or lose EC identities.""" + import copy + + seen = [] + reported = {} + + class Session: + async def post(self, url, data=None, headers=None): + body = msgspec.json.decode(data) + seen.append(body) + item = body["messages"][0]["content"][0] + mm_hash = item["uuid"] + "-processed" + entry: dict[str, Any] = { + "metadata": {"audio_feature_lengths": [100]} if len(seen) == 1 else {}, + "item_indices": [0], + } + if transfer == "nixl": + entry.update(peer_host="encoder", peer_port=1234, size_bytes=4096) + reported[mm_hash] = entry + return _EncoderResponse({mm_hash: entry}) + + second = ( + {"type": second_type, "input_audio": {"data": "large", "format": "wav"}} + if second_type == "input_audio" + else {"type": second_type, second_type: {"url": "large"}} + ) + body = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": "small"}, + "uuid": "user-audio", + }, + second, + ], + } + ], + "mm_processor_kwargs": {"sampling_rate": 16000}, + } + original = copy.deepcopy(body) + monkeypatch.setattr(proxy, "encode_session", Session()) + monkeypatch.setattr(proxy, "NO_REWRITE", False) + consumer = "tcp://consumer:1" if transfer == "mooncake" else None + prepared, _, _ = asyncio.run( + proxy.prepare_for_decode(body, "r", ["http://encoder"], "", consumer) + ) + assert body == original + assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"] + content = prepared["messages"][0]["content"] + assert [item["type"] for item in content] == ["audio_url", second_type] + assert content[0]["uuid"] == "user-audio" + for item, encoded in zip(content, seen): + assert item == encoded["messages"][0]["content"][0] + params = prepared["ec_transfer_params"] + for mm_hash, entry in reported.items(): + assert params[mm_hash] == entry + if transfer == "mooncake": + assert params["ec_items"] == [ + { + "mm_hash": mm_hash, + "transfer_id": encoded["ec_transfer_params"]["ec_items"][0][ + "transfer_id" + ], + } + for mm_hash, encoded in zip(reported, seen) + ] + + @pytest.mark.parametrize("no_rewrite", [False, True]) @pytest.mark.parametrize( "batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])] @@ -509,7 +585,11 @@ def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer): original = copy.deepcopy(body) # Collector indices describe two processed features from one video item. video_metadata = collect_ec_item_metadata( - [SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None + [ + SimpleNamespace(identifier=key, modality=key, data=None) + for key in ("video", "audio") + ], + SimpleNamespace(fields_for=lambda _: set()), ) if transfer == "handle": video_metadata["audio"]["transfer_id"] = "reservation" diff --git a/tests/v1/engine/test_async_llm.py b/tests/v1/engine/test_async_llm.py index cac43d8204..dc3cf1aa00 100644 --- a/tests/v1/engine/test_async_llm.py +++ b/tests/v1/engine/test_async_llm.py @@ -11,7 +11,7 @@ import pytest import vllm.v1.engine.async_llm as async_llm_module from vllm import SamplingParams from vllm.assets.image import ImageAsset -from vllm.config import VllmConfig +from vllm.config import LoggingConfig, VllmConfig from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, @@ -89,6 +89,7 @@ def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch): def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch): vllm_config = MagicMock() + vllm_config.logging_config = LoggingConfig() vllm_config.observability_config.otlp_traces_endpoint = None vllm_config.scheduler_config.stream_interval = 1 vllm_config.profiler_config.profiler = "cuda" @@ -111,6 +112,7 @@ def test_cuda_only_torch_profiler_skips_frontend_cpu_trace( monkeypatch: pytest.MonkeyPatch, ): vllm_config = MagicMock() + vllm_config.logging_config = LoggingConfig() vllm_config.observability_config.otlp_traces_endpoint = None vllm_config.scheduler_config.stream_interval = 1 vllm_config.profiler_config.profiler = "torch" diff --git a/tests/v1/engine/test_input_processor_trace_replay.py b/tests/v1/engine/test_input_processor_trace_replay.py index 24302f6de6..957f70b718 100644 --- a/tests/v1/engine/test_input_processor_trace_replay.py +++ b/tests/v1/engine/test_input_processor_trace_replay.py @@ -80,10 +80,12 @@ def _validate(enable_trace_replay: bool) -> None: model_config=SimpleNamespace( return_sampling_mask=False, enable_trace_replay=enable_trace_replay, + is_diffusion=False, ), vllm_config=SimpleNamespace(reasoning_config=None), speculative_config=None, structured_outputs_config=None, + diffusion_config=None, tokenizer=None, validate_logits_processors_params=lambda params: None, ) diff --git a/tests/v1/engine/utils.py b/tests/v1/engine/utils.py index 476359510f..48fc00264f 100644 --- a/tests/v1/engine/utils.py +++ b/tests/v1/engine/utils.py @@ -196,23 +196,6 @@ def _create_random_top_token_test_matrix( return matrix, prompt_token_ranks -def decode_token( - tok_id: int, - tokenizer: PythonBackend, -) -> str: - """Reproduce the process of detokenizing a token for testing purposes. - - Args: - tok_id: token id to detokenize - tokenizer: tokenizer to use for detokenization - - Returns: - string representation of token - - """ - return tokenizer.convert_ids_to_tokens(tok_id) - - def generate_dummy_sample_logprobs( sampled_tokens_list: list, num_logprobs: int, diff --git a/tests/v1/executor/test_multiproc_executor.py b/tests/v1/executor/test_multiproc_executor.py index 4ebaba5f11..706bfae381 100644 --- a/tests/v1/executor/test_multiproc_executor.py +++ b/tests/v1/executor/test_multiproc_executor.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import weakref +from collections import deque from types import SimpleNamespace from typing import Any import pytest -from vllm.v1.executor.multiproc_executor import WorkerProc +from vllm.v1.executor.multiproc_executor import MultiprocExecutor, WorkerProc +from vllm.v1.outputs import DraftTokenIds class _ExitWorkerLoop(RuntimeError): @@ -65,3 +67,29 @@ def test_execute_worker_rpc_returns_worker_exception(): assert len(outputs) == 1 assert isinstance(outputs[0], RuntimeError) assert str(outputs[0]) == "test error" + + +@pytest.mark.parametrize("stalled", [False, True]) +def test_take_draft_token_ids_uses_execute_model_timeout(monkeypatch, stalled): + """A wedged draft-token readback must not block EngineCore forever.""" + monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "7") + draft = DraftTokenIds(["req"], [[1, 2, 3]]) + + def dequeue(*, timeout=None): + assert timeout is not None and 0 < timeout <= 7 + if stalled: + raise TimeoutError + return WorkerProc.ResponseStatus.SUCCESS, draft + + executor: Any = MultiprocExecutor.__new__(MultiprocExecutor) + executor.is_failed = False + executor.output_rank = 1 + executor.futures_queue = deque() + executor.rpc_broadcast_mq = SimpleNamespace(enqueue=lambda payload: None) + executor.response_mqs = [None, SimpleNamespace(dequeue=dequeue)] + + if stalled: + with pytest.raises(TimeoutError, match="take_draft_token_ids timed out"): + executor.take_draft_token_ids() + else: + assert executor.take_draft_token_ids() is draft diff --git a/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py b/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py new file mode 100644 index 0000000000..766c4adabe --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Mooncake P/D accuracy with same-host HIP transport on ROCm.""" + +import pytest + +from tests.utils import spawn_new_process_for_each_test +from tests.v1.kv_connector.rocm_pd_accuracy_utils import ( + QWEN_MODEL, + run_rocm_pd_accuracy, +) +from vllm.platforms import current_platform + +pytestmark = [ + pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"), + pytest.mark.distributed(num_gpus=4), +] + + +@pytest.mark.parametrize( + "model,prefill_tp,decode_tp", + [ + pytest.param(QWEN_MODEL, 1, 1, id="mooncake-hip-qwen-p1-d1"), + pytest.param(QWEN_MODEL, 2, 2, id="mooncake-hip-qwen-p2-d2"), + ], +) +@spawn_new_process_for_each_test +def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch): + run_rocm_pd_accuracy("mooncake", model, prefill_tp, decode_tp, monkeypatch) diff --git a/tests/v1/kv_connector/nixl_integration/__init__.py b/tests/v1/kv_connector/nixl_integration/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py b/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py new file mode 100644 index 0000000000..5f83330ef9 --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NIXL P/D accuracy with AITER unified attention on ROCm.""" + +import pytest + +from tests.utils import spawn_new_process_for_each_test +from tests.v1.kv_connector.rocm_pd_accuracy_utils import ( + DEEPSEEK_MODEL, + QWEN_MODEL, + run_rocm_pd_accuracy, +) +from vllm.platforms import current_platform + +pytestmark = [ + pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"), + pytest.mark.distributed(num_gpus=4), +] + + +@pytest.mark.parametrize( + "model,prefill_tp,decode_tp", + [ + pytest.param(QWEN_MODEL, 2, 2, id="nixl-aiter-qwen-p2-d2"), + pytest.param(QWEN_MODEL, 1, 2, id="nixl-aiter-qwen-p1-d2"), + pytest.param(QWEN_MODEL, 2, 1, id="nixl-aiter-qwen-p2-d1"), + pytest.param(DEEPSEEK_MODEL, 1, 1, id="nixl-aiter-deepseek-p1-d1"), + pytest.param(DEEPSEEK_MODEL, 1, 2, id="nixl-aiter-deepseek-p1-d2"), + pytest.param(DEEPSEEK_MODEL, 2, 1, id="nixl-aiter-deepseek-p2-d1"), + ], +) +@spawn_new_process_for_each_test +def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch): + run_rocm_pd_accuracy("nixl", model, prefill_tp, decode_tp, monkeypatch) diff --git a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py new file mode 100644 index 0000000000..e515c0a0bd --- /dev/null +++ b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py @@ -0,0 +1,254 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared ROCm P/D accuracy harness for Mooncake and NIXL integration tests. + +Keep the models, TP pairs, full GSM8K workload, and acceptance checks of the +Mooncake and FlashInfer/NIXL sweeps. Mooncake uses same-host HIP IPC; NIXL uses +AITER unified attention. These cases do not exercise RDMA or FlashInfer. +""" + +import contextlib +import importlib +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import requests +from prometheus_client.parser import text_string_to_metric_families + +from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port + +_ROOT = Path(__file__).resolve().parents[3] +QWEN_MODEL = "Qwen/Qwen3-0.6B" +DEEPSEEK_MODEL = "deepseek-ai/deepseek-vl2-tiny" +_REVISIONS = { + QWEN_MODEL: "c1899de289a04d12100db370d81485cdf75e47ca", + DEEPSEEK_MODEL: "66c54660eae7e90c9ba259bfdf92d07d6e3ce8aa", +} + + +def _metric_total(server, name: str, **labels: str) -> float: + response = requests.get(server.url_for("metrics"), timeout=10) + response.raise_for_status() + return sum( + sample.value + for family in text_string_to_metric_families(response.text) + for sample in family.samples + if sample.name == name + and all(sample.labels.get(key) == value for key, value in labels.items()) + ) + + +@contextlib.contextmanager +def _proxy(connector: str, prefill, decode, bootstrap_port: int): + port = get_open_port() + if connector == "mooncake": + script = ( + _ROOT / "examples/disaggregated/mooncake_connector/" + "mooncake_connector_proxy.py" + ) + args = [ + "--prefill", + prefill.url_root, + str(bootstrap_port), + "--decode", + decode.url_root, + ] + else: + script = Path(__file__).parent / "nixl_integration/toy_proxy_server.py" + args = [ + "--prefiller-hosts", + "127.0.0.1", + "--prefiller-ports", + str(prefill.port), + "--decoder-hosts", + "127.0.0.1", + "--decoder-ports", + str(decode.port), + ] + process = subprocess.Popen( + [sys.executable, str(script), "--port", str(port), *args], + start_new_session=True, + ) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + assert process.poll() is None, "P/D proxy exited during startup" + try: + requests.get(f"http://127.0.0.1:{port}/", timeout=1) + break + except requests.ConnectionError: + time.sleep(0.2) + else: + raise TimeoutError("P/D proxy did not start") + yield f"http://127.0.0.1:{port}/v1" + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): + """Transferred KV must feed decoding while full upstream accuracy holds.""" + # Preserve the caller's allocation instead of querying host-wide SMI indices. + selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES") + selector = next((key for key in selectors if key in os.environ), selectors[0]) + visible = os.getenv(selector) + devices = ( + visible.split(",") + if visible is not None + else [str(i) for i in range(current_platform.device_count())] + ) + assert len(devices) >= prefill_tp + decode_tp, "Insufficient allocated GPUs" + for key in selectors: + if key != selector: + monkeypatch.delenv(key, raising=False) + + if connector == "mooncake": + import mooncake.engine + + assert mooncake.engine.SUPPORT_HIP, "Mooncake requires its ROCm/HIP build" + connector_name = "MooncakeConnector" + extra = {"mooncake_protocol": "hip", "device_name": ""} + else: + from vllm.distributed.nixl_utils import NixlWrapper + + assert NixlWrapper is not None, "NIXL ROCm bindings must import" + connector_name = "NixlConnector" + extra = {} + + common_args = [ + "--revision", + _REVISIONS[model], + "--dtype", + "bfloat16", + "--enforce-eager", + "--distributed-executor-backend", + "mp", + "--block-size", + "128", + "--max-num-seqs", + "128", + "--max-num-batched-tokens", + "4096", + "--gpu-memory-utilization", + "0.2", + "--kv-cache-memory-bytes", + str(8 * 1024**3), + "--moe-backend", + "triton", + "--attention-backend", + "ROCM_AITER_UNIFIED_ATTN", + ] + common_env = { + "PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"], + "VLLM_HOST_IP": "127.0.0.1", + "VLLM_WORKER_MULTIPROC_METHOD": "spawn", + "VLLM_USE_V2_MODEL_RUNNER": "1", + "VLLM_ROCM_USE_AITER": "1", + "VLLM_KV_CACHE_LAYOUT": "HND", + } + bootstrap_port = get_open_port() + with contextlib.ExitStack() as stack: + servers: list[RemoteOpenAIServer] = [] + stack.callback(RemoteOpenAIServer.shutdown_many, servers) + for role, tp, assigned in ( + ("kv_producer", prefill_tp, devices[:prefill_tp]), + ("kv_consumer", decode_tp, devices[prefill_tp : prefill_tp + decode_tp]), + ): + config = { + "kv_connector": connector_name, + "kv_role": role, + "kv_connector_extra_config": extra, + } + env = { + **common_env, + selector: ",".join(assigned), + "VLLM_PORT": str(get_open_port()), + "VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()), + "VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port), + } + servers.append( + RemoteOpenAIServer( + model, + [ + *common_args, + "--tensor-parallel-size", + str(tp), + "--kv-transfer-config", + json.dumps(config), + ], + env_dict=env, + max_wait_seconds=1200, + ) + ) + prefill, decode = servers + proxy_url = stack.enter_context( + _proxy(connector, prefill, decode, bootstrap_port) + ) + + # Reuse the full upstream evaluation and its model-specific assertions. + accuracy = importlib.import_module( + f"tests.v1.kv_connector.{connector}_integration.test_accuracy" + ) + monkeypatch.setattr(accuracy, "MODEL_NAME", model) + monkeypatch.setattr(accuracy, "BASE_URL", proxy_url) + assert model in accuracy.EXPECTED_VALUES + + def transfer_totals(): + totals = { + "external_tokens": _metric_total( + decode, + "vllm:prompt_tokens_by_source_total", + source="external_kv_transfer", + ) + } + if connector == "nixl": + totals["nixl_bytes"] = _metric_total( + decode, "vllm:nixl_bytes_transferred_sum" + ) + return totals + + simple_evaluate = accuracy.lm_eval.simple_evaluate + + def evaluate_with_transfer_checks(*args, **kwargs): + # The upstream helper runs a smoke prompt before GSM8K. Wait until + # its counters appear, then require new transfers during evaluation. + deadline = time.monotonic() + 15 + before = transfer_totals() + while not all(value > 0 for value in before.values()): + assert time.monotonic() < deadline, ( + f"Smoke prompt did not transfer KV: {before}" + ) + time.sleep(1) + before = transfer_totals() + + # The API adapter loads a local tokenizer because these proxies do + # not expose /tokenize. Match it to the server's pinned checkpoint. + kwargs["model_args"] += f",revision={_REVISIONS[model]}" + results = simple_evaluate(*args, **kwargs) + deadline = time.monotonic() + 15 + after = transfer_totals() + while not all(after[key] > value for key, value in before.items()): + assert time.monotonic() < deadline, ( + f"GSM8K did not transfer KV: before={before}, after={after}" + ) + time.sleep(1) + after = transfer_totals() + print(f"GSM8K transfer counters: before={before}, after={after}") + return results + + monkeypatch.setattr( + accuracy.lm_eval, "simple_evaluate", evaluate_with_transfer_checks + ) + accuracy.test_accuracy() diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index c409fd125d..0121ed95bf 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -506,6 +506,44 @@ def test_event_metadata_skips_non_full_attention_group( assert events[0].block_size == 0 +@pytest.mark.parametrize("secondary_removal_in_same_batch", [False, True]) +def test_primary_removal_preserves_metadata_for_same_batch_kvcr_store( + secondary_removal_in_same_batch, +): + """A removal must not drop the payload a later store in the same batch + still needs, and must still drop it once no residency is left.""" + tracker = _tracker() + group_config = _group_config(block_size=64, blocks_per_chunk=2) + req = _request(block_hashes=[_hash(i) for i in range(4)], token_count=256) + key = _record_chunks(tracker, req, group_config, num_chunks=2)[1] + secondary_removal = _removed_event([key], ownership="kvcr") + + batch = [ + _removed_event([key]), + _stored_event([key], ownership="kvcr", removal_expected=True), + ] + if secondary_removal_in_same_batch: + batch.append(secondary_removal) + events = list(tracker.take_events(batch)) + if not secondary_removal_in_same_batch: + events += tracker.take_events([secondary_removal]) + removed, stored, final_removal = events + + hashes = [_wire_hash(_hash(2)), _wire_hash(_hash(3))] + assert isinstance(removed, BlockRemoved) + assert removed.block_hashes == hashes + assert stored.block_size == 64 + assert stored.token_ids == list(range(129, 257)) + assert stored.parent_block_hash == _wire_hash(_hash(1)) + assert stored.block_hashes == hashes + assert stored.ownership == "kvcr" + assert final_removal.block_hashes == hashes + + # Nothing holds a residency now, so the payload is gone. + [after_eviction] = tracker.take_events([_stored_event([key], ownership="kvcr")]) + assert after_eviction.block_size == 0 + + def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit(): tracker = _tracker() block_hashes = [_hash(0), _hash(1)] diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index a691295616..5b3bf1c8be 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -3440,59 +3440,6 @@ class TestEagle: # whole request. assert sched._lookup(req_status) == 4 - def test_eagle_verified_survives_eagle_tighten(self, request_runner): - """Eagle group tightening does NOT clear eagle_verified. - - Groups: 0=non-eagle full-attn, 1=eagle full-attn. - Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2 - (max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared. - Result: 8 tokens (eagle only pops once). - """ - block_size = 4 - groups = [ - KVCacheGroupSpec( - ["layer0"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - ), - is_eagle_group=False, - ), - KVCacheGroupSpec( - ["layer1"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=2, - head_size=1, - dtype=torch.float32, - ), - is_eagle_group=True, - ), - ] - runner = request_runner( - block_size=block_size, - num_gpu_blocks=100, - async_scheduling=False, - kv_cache_groups=groups, - ) - runner.manager.lookup.side_effect = lambda key, req_context: ( - LookupResult.HIT - if int(get_offload_block_hash(key).decode()) in {1, 2, 3} - else LookupResult.MISS - ) - sched = runner.connector_scheduler - req_status = self._make_req_status( - sched, - num_tokens=12, - offload_keys_per_group=[[1, 2, 3], [1, 2, 3]], - ) - # Group 0: 3 hits → max_hit=12, num_hit=12 - # Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8 - # Tightened but IS eagle → no clear. No re-iteration. - assert sched._lookup(req_status) == 8 - # ------------------------------------------------------------------- # Integration tests: store and load via request_runner # ------------------------------------------------------------------- diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index ad8d4fc362..cb766c6676 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -517,6 +517,7 @@ class RequestRunner: if complete_transfers: self.offloading_spec.complete_transfers() + self.worker_connector.wait_for_save() finished_sending, finished_recving = self.worker_connector.get_finished( scheduler_output.finished_req_ids ) diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index e1054d9ef8..6f518ab53a 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector import DecodeBenchConnectorMetadata, ) from vllm.forward_context import ForwardContext +from vllm.platforms import current_platform from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.kv_cache_utils import ( @@ -34,6 +35,8 @@ from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, + KVQuantMode, + MambaSpec, SlidingWindowSpec, ) from vllm.v1.request import Request @@ -50,7 +53,12 @@ from .utils import ( class DecodeBenchTestRunner: """Test runner for DecodeBenchConnector.""" - def __init__(self, block_size: int, num_gpu_blocks: int): + def __init__( + self, + block_size: int, + num_gpu_blocks: int, + kv_connector_extra_config: dict | None = None, + ): self.block_size = block_size self.num_gpu_blocks = num_gpu_blocks @@ -61,6 +69,7 @@ class DecodeBenchTestRunner: block_size=block_size, max_num_batched_tokens=1000, kv_connector="DecodeBenchConnector", + kv_connector_extra_config=kv_connector_extra_config, ) self.vllm_config = vllm_config @@ -491,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding(): def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): - """DCP scales full-attention blocks but not sliding-window blocks.""" + """DCP scales target blocks but not the replicated draft's window.""" block_size = 16 dcp_world_size = 2 vllm_config = create_vllm_config( @@ -506,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): dtype=torch.float32, ) sliding_window = SlidingWindowSpec( + dcp_sharded=False, block_size=block_size, num_kv_heads=1, head_size=1, @@ -780,5 +790,320 @@ def test_decode_bench_connector_concurrent_requests(): assert len(metadata2.reqs_to_fill) == 0 +def _fill_single_block(cache_dtype, spec_dtype, kv_quant_mode, cache, fill_std=0.0): + """Fill block 1 of a one-layer cache through the worker connector.""" + vllm_config = create_vllm_config( + block_size=16, + kv_connector="DecodeBenchConnector", + kv_connector_extra_config={"fill_std": fill_std}, + cache_dtype=cache_dtype, + ) + kv_cache_spec = FullAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=8, + dtype=spec_dtype, + kv_quant_mode=kv_quant_mode, + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.WORKER, + KVCacheConfig( + num_blocks=cache.shape[0], + kv_cache_tensors=[], + kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], + ), + ) + connector.register_kv_caches({"layer": cache}) + connector.bind_connector_metadata( + DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([1],), 16)}) + ) + connector.start_load_kv( + ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) + ) + + +@pytest.mark.parametrize( + "cache_dtype,spec_dtype,kv_quant_mode,fp8_dtype", + [ + ( + "fp8", + torch.uint8, + KVQuantMode.FP8_PER_TENSOR, + current_platform.fp8_dtype(), + ), + ("fp8_e5m2", torch.uint8, KVQuantMode.FP8_PER_TENSOR, torch.float8_e5m2), + ("auto", torch.float8_e4m3fn, KVQuantMode.NONE, torch.float8_e4m3fn), + ], +) +@pytest.mark.parametrize("fill_std", [0.0, 0.1]) +def test_decode_bench_connector_fills_fp8_caches( + cache_dtype, spec_dtype, kv_quant_mode, fp8_dtype, fill_std +): + """fp8 caches, including ones stored as uint8, get encoded fill values.""" + cache = torch.zeros(4, 16, 8, dtype=spec_dtype) + _fill_single_block(cache_dtype, spec_dtype, kv_quant_mode, cache, fill_std) + + values = cache.view(fp8_dtype).float() + assert torch.count_nonzero(values[[0, 2, 3]]) == 0 + filled = values[1] + if fill_std == 0: + expected = torch.tensor(0.015).to(fp8_dtype).float() + torch.testing.assert_close(filled, expected.expand_as(filled)) + else: + assert torch.isfinite(filled).all() + assert filled.std() > 0 + + +def test_decode_bench_connector_zero_fills_packed_uint8_caches(): + """uint8 layouts that do not hold plain fp8 values are zeroed.""" + cache = torch.full((4, 16, 8), 7, dtype=torch.uint8) + _fill_single_block("nvfp4", torch.uint8, KVQuantMode.NVFP4, cache) + + assert torch.count_nonzero(cache[1]) == 0 + assert torch.all(cache[[0, 2, 3]] == 7) + + +@pytest.mark.parametrize("fill_std", [0.0, 0.1]) +def test_decode_bench_connector_startup_fill_fills_registered_caches(fill_std): + """Startup fill writes every cache view in place, including strided views.""" + vllm_config = create_vllm_config( + block_size=16, + kv_connector="DecodeBenchConnector", + kv_connector_extra_config={"startup_fill": True, "fill_std": fill_std}, + ) + kv_cache_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["attention", "transposed"], + FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 + ), + ), + ], + ) + connector = DecodeBenchConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + attention = torch.full((4, 2, 16), -1.0) + transposed = torch.full((2, 4, 16), -1.0).transpose(0, 1) + connector.register_kv_caches({"attention": attention, "transposed": transposed}) + + for filled in (attention, transposed): + if fill_std == 0: + torch.testing.assert_close(filled, torch.full_like(filled, 0.015)) + else: + assert torch.isfinite(filled).all() + assert filled.std() > 0 + + +@pytest.mark.parametrize("fill_std", [0.0, 0.1]) +def test_decode_bench_connector_startup_fill_encodes_fp8_caches(fill_std): + """Startup fill encodes values in the fp8 dtype of a uint8 fp8 cache.""" + vllm_config = create_vllm_config( + block_size=16, + kv_connector="DecodeBenchConnector", + kv_connector_extra_config={"startup_fill": True, "fill_std": fill_std}, + cache_dtype="fp8", + ) + kv_cache_spec = FullAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=8, + dtype=torch.uint8, + kv_quant_mode=KVQuantMode.FP8_PER_TENSOR, + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.WORKER, + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], + ), + ) + cache = torch.zeros(4, 16, 8, dtype=torch.uint8) + connector.register_kv_caches({"layer": cache}) + + values = cache.view(current_platform.fp8_dtype()).float() + if fill_std == 0: + expected = torch.tensor(0.015).to(current_platform.fp8_dtype()).float() + torch.testing.assert_close(values, expected.expand_as(values)) + else: + assert torch.isfinite(values).all() + assert values.std() > 0 + + +def test_decode_bench_connector_startup_fill_keeps_zeroed_groups_per_request(): + """Attention blocks the engine zeroes on allocation are filled per request.""" + vllm_config = create_vllm_config( + block_size=16, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + kv_connector_extra_config={"startup_fill": True}, + ) + # Mamba layers make the engine zero newly allocated attention blocks. + kv_cache_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["attention"], + FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=16, + shapes=((4,), (4,)), + dtypes=(torch.float32, torch.float32), + ), + ), + ], + ) + assert kv_cache_config.needs_kv_cache_zeroing + scheduler_connector = DecodeBenchConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) + num_external_tokens = 17 + request = create_request( + request_id=1, num_tokens=num_external_tokens + 1, block_size=16 + ) + blocks = KVCacheBlocks( + ( + [KVCacheBlock(block_id=1), KVCacheBlock(block_id=2)], + [KVCacheBlock(block_id=3)], + ) + ) + scheduler_connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = scheduler_connector.build_connector_meta(SchedulerOutput.make_empty()) + assert isinstance(metadata, DecodeBenchConnectorMetadata) + assert metadata.reqs_to_fill == { + request.request_id: (([1, 2], []), num_external_tokens) + } + + worker_connector = DecodeBenchConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + attention = torch.full((4, 2), -1.0) + mamba_states = (torch.zeros(4, 4), torch.zeros(4, 4)) + worker_connector.register_kv_caches({"attention": attention, "mamba": mamba_states}) + assert torch.all(attention == -1.0) + + worker_connector.bind_connector_metadata(metadata) + worker_connector.start_load_kv( + ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) + ) + torch.testing.assert_close(attention[1:3], torch.full_like(attention[1:3], 0.015)) + assert torch.all(attention[[0, 3]] == -1.0) + + +def test_decode_bench_connector_startup_fill_skips_per_step_fill(): + """Admitted requests need no per-step fill and leave existing KV intact.""" + block_size = 16 + runner = DecodeBenchTestRunner( + block_size=block_size, + num_gpu_blocks=100, + kv_connector_extra_config={"startup_fill": True}, + ) + for kv_cache in runner.kv_caches.values(): + torch.testing.assert_close(kv_cache, torch.full_like(kv_cache, 0.015)) + kv_cache.fill_(7.0) + + req1 = runner.new_request([1] * (block_size * 2)) + scheduler_output, metadata = runner.run_single_step() + assert scheduler_output.num_scheduled_tokens == {req1.request_id: 1} + assert scheduler_output.scheduled_new_reqs[0].num_computed_tokens == ( + block_size * 2 - 1 + ) + assert metadata.reqs_to_fill == {} + + req2 = runner.new_request([2] * (block_size * 3)) + scheduler_output, metadata = runner.run_single_step() + assert scheduler_output.num_scheduled_tokens == { + req1.request_id: 1, + req2.request_id: 1, + } + assert metadata.reqs_to_fill == {} + + # A re-admitted (e.g. preempted) request is not reported again. + assert runner.scheduler_connector.get_num_new_matched_tokens(req1, 0) == ( + 0, + False, + ) + for kv_cache in runner.kv_caches.values(): + assert torch.all(kv_cache == 7.0) + + +def test_decode_bench_connector_startup_fill_keeps_circular_buffer_zero_fill(): + """Circular buffers are still zeroed per request with startup fill.""" + vllm_config = create_vllm_config( + block_size=16, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + kv_connector_extra_config={"startup_fill": True}, + ) + kv_cache_config = KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full_attention"], + FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 + ), + ), + KVCacheGroupSpec( + ["compressor_ring"], + CircularBufferSpec( + block_size=32, num_kv_heads=1, head_size=1, dtype=torch.float32 + ), + ), + ], + ) + scheduler_connector = DecodeBenchConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) + num_external_tokens = 17 + request = create_request( + request_id=1, num_tokens=num_external_tokens + 1, block_size=16 + ) + blocks = KVCacheBlocks( + ( + [KVCacheBlock(block_id=1), KVCacheBlock(block_id=2)], + [KVCacheBlock(block_id=4)], + ) + ) + scheduler_connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = scheduler_connector.build_connector_meta(SchedulerOutput.make_empty()) + assert isinstance(metadata, DecodeBenchConnectorMetadata) + assert metadata.reqs_to_fill == { + request.request_id: (([], [4]), num_external_tokens) + } + + worker_connector = DecodeBenchConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + attention_cache = torch.zeros(8, 2) + ring_cache = torch.ones(8, 2) + worker_connector.register_kv_caches( + {"full_attention": attention_cache, "compressor_ring": ring_cache} + ) + assert torch.all(ring_cache == 1.0) + worker_connector.bind_connector_metadata(metadata) + worker_connector.start_load_kv( + ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) + ) + + torch.testing.assert_close(attention_cache, torch.full_like(attention_cache, 0.015)) + expected_ring = torch.ones_like(ring_cache) + expected_ring[4] = 0 + assert torch.equal(ring_cache, expected_ring) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/v1/kv_connector/unit/test_kv_connector_lifecycle.py b/tests/v1/kv_connector/unit/test_kv_connector_lifecycle.py index b6b18b092b..a1df2a964b 100644 --- a/tests/v1/kv_connector/unit/test_kv_connector_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_kv_connector_lifecycle.py @@ -71,6 +71,7 @@ def test_kv_connector_mixin_clears_metadata(): assert connector._connector_metadata is None # Test connector wrapper records method calls assert connector.call_record.get("bind_connector_metadata", 0) == 1 + assert connector.call_record.get("wait_for_save", 0) == 1 assert connector.call_record.get("clear_connector_metadata", 0) == 1 finally: # Ensure we clean up the global connector between tests diff --git a/tests/v1/kv_connector/unit/test_lmcache_connector.py b/tests/v1/kv_connector/unit/test_lmcache_connector.py index a64f7f32d5..13f08a610c 100644 --- a/tests/v1/kv_connector/unit/test_lmcache_connector.py +++ b/tests/v1/kv_connector/unit/test_lmcache_connector.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Iterator from unittest.mock import MagicMock import pytest from vllm.distributed.kv_events import BlockStored +from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector import ( LMCacheConnectorV1, LMCacheKVEvents, @@ -783,3 +785,32 @@ class TestIntegrationScenarios: assert aggregated_events[0].block_hashes == ["hash_common"] assert aggregated_events[0].parent_block_hash == "parent_common" assert aggregated_events[0].token_ids == [1, 2, 3] + + +def test_legacy_layerwise_engine_does_not_reuse_storers_on_empty_steps(): + """Empty steps must not advance generators exhausted by the last forward.""" + + class LegacyLayerwiseEngine: + layerwise_storers: list[Iterator[None]] + + def save_kv_layer(self, *args, **kwargs): + self.layerwise_storers.append(iter([None])) + + def wait_for_save(self): + for storer in self.layerwise_storers: + next(storer) + + connector = LMCacheConnectorV1.__new__(LMCacheConnectorV1) + connector._lmcache_engine = LegacyLayerwiseEngine() + metadata = KVConnectorMetadata() + connector.bind_connector_metadata(metadata) + connector.save_kv_layer("layer", MagicMock(), MagicMock()) + connector.wait_for_save() + connector.clear_connector_metadata() + + for _ in range(2): + connector.bind_connector_metadata(metadata) + connector.wait_for_save() + connector.clear_connector_metadata() + + assert connector._lmcache_engine.layerwise_storers == [] diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py index 37f33ce353..a7fb2be023 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py @@ -10,6 +10,7 @@ import asyncio from unittest.mock import patch import pytest +import torch from vllm.config import set_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( @@ -22,6 +23,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im SendBlockMeta, TransferRegion, ) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, +) from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies from .utils import create_request, create_vllm_config, make_kv_cache_config @@ -440,17 +446,19 @@ def test_request_finished_with_hma_groups(): # --------------------------------------------------------------------------- # Worker-side load-failure reporting (HMA vs non-HMA) # --------------------------------------------------------------------------- -def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False): +def _make_kv_consumer_worker( + swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None +): block_size = 16 vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", block_size=block_size, ) - vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma - kv_cache_config = make_kv_cache_config( - block_size=block_size, swa_enabled=swa_enabled - ) + if kv_cache_config is None: + kv_cache_config = make_kv_cache_config( + block_size=block_size, swa_enabled=swa_enabled + ) with set_current_vllm_config(vllm_config), patch_worker_dependencies(): connector = MooncakeConnector( vllm_config, KVConnectorRole.WORKER, kv_cache_config @@ -472,19 +480,43 @@ def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqM # requires an accelerator device, so (like the worker tests in # test_mooncake_connector.py) they run on the GPU KV-connectors job only. @pytest.mark.parametrize( - "swa_enabled,disable_hma,expected_is_hma", + "swa_enabled,expected_is_hma", [ - (True, False, True), # SWA group present, HMA enabled - (True, True, False), # SWA group present, but HMA disabled - (False, False, False), # FA only, HMA not needed + (True, True), # multiple groups (FA + SWA) + (False, False), # single group, block-level reporting is unambiguous ], ) -def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma): - """Worker-side _is_hma_required mirrors the scheduler's derivation.""" - worker = _make_kv_consumer_worker(swa_enabled, disable_hma) +def test_worker_is_hma_required(swa_enabled, expected_is_hma): + """Worker-side _is_hma_required mirrors the scheduler's condition.""" + worker = _make_kv_consumer_worker(swa_enabled=swa_enabled) assert worker._is_hma_required is expected_is_hma +def test_worker_is_hma_required_multiple_full_attention_groups(): + """Two full-attention groups with different block sizes also select + request-level failure reporting, matching the scheduler.""" + kv_cache_config = KVCacheConfig( + num_blocks=100, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16 + ), + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16 + ), + ), + ], + ) + worker = _make_kv_consumer_worker(kv_cache_config=kv_cache_config) + assert worker._is_hma_required + + def test_worker_failed_recv_reports_request_level_failure_with_hma(): """With HMA, load failures report the request, not ambiguous block IDs.""" worker = _make_kv_consumer_worker(swa_enabled=True) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 6092523073..f08213444e 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -290,6 +290,8 @@ class _FakeKVTransferConfig: class _FakeModelConfig: model = "test-model" use_mla = False + enable_sleep_mode = False + enable_cumem_allocator = False def get_num_layers(self, parallel_config) -> int: return 1 @@ -432,6 +434,64 @@ def _patch_worker_runtime( monkeypatch.setattr(worker, "LookupKeyServer", MagicMock()) +def test_create_mem_pool_returns_none_when_unconfigured(): + assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None + + +def test_create_mem_pool_rejects_unknown_pool(): + with pytest.raises( + ValueError, + match="Unsupported custom_mem_pool='UNKNOWN'", + ): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "unknown"}, _FakeModelConfig() + ) + + +@pytest.mark.parametrize( + "conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"] +) +def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option): + model_config = _FakeModelConfig() + setattr(model_config, conflicting_option, True) + + with pytest.raises(ValueError, match="custom_mem_pool is incompatible"): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "NVLINK"}, model_config + ) + + +@pytest.mark.parametrize( + ("pool_type", "allocator_name"), + [("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")], +) +def test_create_mem_pool_uses_mooncake_allocator( + monkeypatch, pool_type, allocator_name +): + allocator_cls = MagicMock() + allocator = allocator_cls.get_allocator.return_value + allocator_module = types.ModuleType("mooncake.allocator") + setattr(allocator_module, allocator_name, allocator_cls) + mooncake_module = types.ModuleType("mooncake") + mooncake_module.allocator = allocator_module # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "mooncake", mooncake_module) + monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module) + + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2) + mem_pool = MagicMock() + mem_pool_ctor = MagicMock(return_value=mem_pool) + monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor) + + result = worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": pool_type}, _FakeModelConfig() + ) + + assert result is mem_pool + allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2)) + allocator.allocator.assert_called_once_with() + mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value) + + def test_pool_key_to_string_without_prefix_is_unchanged(): """Default (empty) cache_prefix keeps keys byte-identical to the historical unprefixed format so existing deployments keep their hits.""" @@ -2222,15 +2282,40 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch): assert [db.block_size for db in store_worker.token_dbs] == [800, 800] +def _make_two_full_attention_groups_kv_cache_config(): + """Two full-attention groups with different block sizes: no hybrid + layers, but block-level failure reporting is still unsupported.""" + return KVCacheConfig( + num_blocks=10, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full0"], + FullAttentionSpec( + block_size=800, num_kv_heads=8, head_size=64, dtype=None + ), + ), + KVCacheGroupSpec( + ["full1"], + FullAttentionSpec( + block_size=1600, num_kv_heads=8, head_size=64, dtype=None + ), + ), + ], + ) + + @pytest.mark.parametrize( - ("disable_hybrid_kv_cache_manager", "expected_is_hma_required"), + ("make_config", "expected_is_hma_required"), [ - (True, False), - (False, True), + (lambda: _make_qsa_hybrid_kv_cache_config(), True), + (lambda: _make_two_full_attention_groups_kv_cache_config(), True), + (lambda: _make_kv_cache_config(block_size=800), False), ], + ids=["hybrid", "two-full-attention-groups", "single-group"], ) def test_worker_is_hma_required_from_kv_cache_groups( - monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required + monkeypatch, make_config, expected_is_hma_required ): store = MagicMock() store.setup.return_value = 0 @@ -2241,16 +2326,12 @@ def test_worker_is_hma_required_from_kv_cache_groups( "load_from_config", staticmethod(lambda: _make_config()), ) - vllm_config = _make_vllm_config( - disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager - ) + vllm_config = _make_vllm_config() vllm_config.cache_config.block_size = 800 vllm_config.cache_config.enable_prefix_caching = True vllm_config.cache_config.prefix_match_unit = None - store_worker = worker.MooncakeStoreWorker( - vllm_config, _make_qsa_hybrid_kv_cache_config() - ) + store_worker = worker.MooncakeStoreWorker(vllm_config, make_config()) assert store_worker._is_hma_required is expected_is_hma_required diff --git a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py index 1cb5e271bf..9860a7b7e1 100644 --- a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py +++ b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py @@ -4,8 +4,7 @@ import importlib.util import math import threading -from collections import OrderedDict, defaultdict -from queue import Queue +from collections import OrderedDict from types import SimpleNamespace from typing import Any @@ -19,6 +18,8 @@ from vllm.v1.kv_cache_interface import ( MLAAttentionSpec, ) +from .utils import make_moriio_writer + aiter_available = importlib.util.find_spec("aiter") is not None mori_available = importlib.util.find_spec("mori") is not None @@ -90,18 +91,6 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace: return SimpleNamespace(num_blocks=num_blocks) -def _writer_with_fake_worker(fake_worker: Any) -> Any: - writer = MoRIIOWriter.__new__(MoRIIOWriter) - writer._worker_ref = lambda: fake_worker - writer._write_task_q = Queue() - writer._write_state_lock = threading.Lock() - writer._scheduled_writes = defaultdict(int) - writer._scheduled_layers = defaultdict(set) - writer._sealed_writes = {} - writer.ensure_worker_started = lambda: None - return writer - - def _wrapper_for_messages() -> Any: wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper) wrapper.lock = threading.Lock() @@ -409,7 +398,7 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count(): request_info = RemoteAllocInfo(block_ids=[4, 5]) wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = request_info - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) assert writer.schedule_write(_write_task("dense0")) assert not writer.schedule_write(_write_task("dense0")) @@ -459,12 +448,12 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish(): request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2) request_info.transfer_statuses.extend(["status-a", "status-b"]) request_info.completion_request_id = "req" - request_info.completion_remote_notify_port = 7000 + # Final notify port is resolved in _execute_write_task; finalize reads it + # as-is, so set the already-offset value (base 7000 + tp_rank-2 offset). + request_info.completion_notify_port = 7002 request_info.completion_remote_ip = "127.0.0.1" wrapper.done_remote_allocate_req_dict["xfer"] = request_info - writer = _writer_with_fake_worker( - SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2) - ) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)) writer._scheduled_writes["xfer"] = 2 writer._scheduled_layers["xfer"] = {"dense0", "indexer"} writer._sealed_writes["xfer"] = 2 @@ -512,7 +501,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain(): def test_write_failure_marks_terminal_and_clears_scheduled_state(): wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5]) - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) writer._scheduled_writes["xfer"] = 2 writer._scheduled_layers["xfer"] = {"dense0", "indexer"} writer._sealed_writes["xfer"] = 2 @@ -530,7 +519,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state(): def test_schedule_write_rejects_terminal_transfer_without_recreating_state(): wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5]) - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) writer._scheduled_writes["xfer"] = 1 writer._scheduled_layers["xfer"] = {"dense0"} writer._sealed_writes["xfer"] = 1 diff --git a/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py b/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py new file mode 100644 index 0000000000..adac007d19 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for mori-io write_done routing (#51681).""" + +import threading +from types import SimpleNamespace +from typing import Any + +import pytest + +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( + RemoteAllocInfo, + ReqMeta, + WriteTask, + get_port_offset, +) +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnectorWorker, +) + +from .utils import make_moriio_writer + +pytestmark = pytest.mark.cpu_test + + +def _fake_worker( + alloc: dict[str, Any], + *, + tp_rank: int = 0, + set_remote_dp_size_local: int | None = None, +) -> Any: + wrapper = SimpleNamespace( + done_remote_allocate_req_dict=alloc, + lock=threading.Lock(), + done_req_ids=[], + sent_notifies=[], + ) + wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: ( + wrapper.sent_notifies.append( + (transfer_id, remote_ip, remote_port, message_type) + ) + ) + wrapper.waiting_for_transfer_complete = lambda statuses: None + wrapper._mark_transfer_terminal_locked = lambda transfer_id: None + wrapper._is_transfer_terminal_locked = lambda transfer_id: False + worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank) + if set_remote_dp_size_local is not None: + worker.remote_dp_size_local = set_remote_dp_size_local + return worker + + +def _req_meta( + *, + transfer_id: str = "tA", + remote_host: str = "10.0.0.1", + multi_pod_hosts: list[str] | None = None, + remote_dp_size: int = 16, + remote_dp_size_local: int = 8, +) -> ReqMeta: + return ReqMeta( + transfer_id=transfer_id, + local_block_ids=[[1]], + remote_block_ids=[[2]], + remote_host=remote_host, + remote_port=1, + remote_handshake_port=2, + remote_notify_port=20000, + remote_engine_id="remote-engine", + tp_size=1, + remote_dp_size=remote_dp_size, + multi_pod_hosts=list(multi_pod_hosts or []), + remote_dp_size_local=remote_dp_size_local, + ) + + +def _write_task( + transfer_id: str, + remote_ip: str, + *, + multi_pod_hosts: list[str] | None = None, + remote_dp_size_local: int = 0, + request_id: str = "req", +) -> WriteTask: + return WriteTask( + request_id=request_id, + transfer_id=transfer_id, + dst_engine_id="remote-engine", + local_block_ids=[1, 3], + remote_block_ids_hint=None, + layer_name="layer0", + event=None, + remote_notify_port=20000, + remote_ip=remote_ip, + multi_pod_hosts=multi_pod_hosts or [], + remote_dp_size_local=remote_dp_size_local, + ) + + +def test_execute_write_task_ip_is_per_task_under_overwrite(): + alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} + worker = _fake_worker(alloc) + worker.multi_pod_hosts = ["10.0.0.2"] + worker.remote_dp_size_local = 1 + + writer = make_moriio_writer(worker) + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + writer._execute_write_task(task_a) + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert alloc["tA"].completion_remote_ip != "10.0.0.2" + + +def test_deferred_task_ip_is_per_task_under_overwrite(): + alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} + worker = _fake_worker(alloc) + worker.multi_pod_hosts = ["10.0.0.2"] + worker.remote_dp_size_local = 1 + + writer = make_moriio_writer(worker) + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + writer._deferred_tasks.append(task_a) + writer._process_deferred_tasks() + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert writer._deferred_tasks == [] + + +def test_two_requests_resolve_distinct_hosts(): + alloc = { + "tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0), + "tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3), + } + worker = _fake_worker(alloc) + writer = make_moriio_writer(worker) + + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + task_b = _write_task( + "tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1 + ) + worker.multi_pod_hosts = ["10.0.0.2"] + writer._execute_write_task(task_a) + writer._execute_write_task(task_b) + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert alloc["tB"].completion_remote_ip == "10.0.0.2" + + +def test_finalize_notify_port_uses_pod_local_rank(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=0) + writer = make_moriio_writer(worker) + + task = _write_task( + "tA", + "10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + writer._execute_write_task(task) + assert info.completion_remote_ip == "10.0.0.2" + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + assert worker.moriio_wrapper.sent_notifies + _tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] + assert mtype == "write_done" + assert ip == "10.0.0.2" + assert port == base + get_port_offset(9 % 8, 0) + + +def test_finalize_port_ignores_stale_worker_dp_local(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1) + writer = make_moriio_writer(worker) + + task = _write_task( + "tA", + "10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + writer._execute_write_task(task) + assert info.completion_notify_port == base + get_port_offset(9 % 8, 0) + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + _tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] + assert mtype == "write_done" + assert port == base + get_port_offset(9 % 8, 0) + assert port != base + get_port_offset(9 % 1, 0) + + +def test_finalize_single_pod_port_uses_global_rank(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=2) + writer = make_moriio_writer(worker) + + task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0) + writer._execute_write_task(task) + assert info.completion_remote_ip == "10.0.0.1" + assert info.completion_notify_port == base + get_port_offset(3, 2) + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + _tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1] + assert port == base + get_port_offset(3, 2) + + +def test_write_blocks_for_req_forwards_routing_to_schedule_write(): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + captured: dict[str, Any] = {} + + def _capture_schedule_write_blocks(**kwargs: Any) -> None: + captured.update(kwargs) + + worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] + + meta = _req_meta( + remote_host="10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None) + + assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"] + assert captured["remote_dp_size_local"] == 8 + assert captured["remote_ip"] == "10.0.0.1" + assert captured["local_block_ids"] == [1] + assert captured["remote_block_ids"] == [2] + assert not hasattr(worker, "multi_pod_hosts") + + +def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty(): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + captured: dict[str, Any] = {} + + def _capture_schedule_write_blocks(**kwargs: Any) -> None: + captured.update(kwargs) + + worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] + + meta = _req_meta( + remote_host="10.0.0.9", + multi_pod_hosts=[], + remote_dp_size=4, + remote_dp_size_local=0, + ) + worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None) + + assert captured["multi_pod_hosts"] == ["10.0.0.9"] + assert captured["remote_dp_size_local"] == 4 + assert captured["local_block_ids"] == [1] + assert captured["remote_block_ids"] == [2] diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 1a85865560..9d04a2ea53 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1): # Override just build_kv_connector_stats mock.build_kv_connector_stats = cls.build_kv_connector_stats mock.get_kv_connector_stats.return_value = None + mock.get_mem_pool_context.return_value = None return mock @classmethod @@ -201,6 +202,36 @@ def mc() -> MultiConnector: return mc +def test_multi_connector_mem_pool_context_none(mc: MultiConnector): + assert mc.get_mem_pool_context() is None + + +def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector): + context = MagicMock() + provider = MagicMock(spec_set=KVConnectorBase_V1) + provider.get_mem_pool_context.return_value = context + mc._connectors = [mc._connectors[0], provider] + + assert mc.get_mem_pool_context() is context + provider.get_mem_pool_context.assert_called_once_with() + + +def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector): + providers = [ + MagicMock(spec_set=KVConnectorBase_V1), + MagicMock(spec_set=KVConnectorBase_V1), + ] + for provider in providers: + provider.get_mem_pool_context.return_value = MagicMock() + mc._connectors = providers + + with pytest.raises( + ValueError, + match="Multiple connectors provide a KV cache memory pool", + ): + mc.get_mem_pool_context() + + # Helper function to compare directories recursively def _compare_directories(dir1: Path, dir2: Path) -> bool: """Compares two directories recursively for identical content.""" @@ -320,6 +351,7 @@ def test_multi_example_connector_consistency(): # First three events are from initialization. Layer hooks run before the # deferred load starts after the forward pass. expected_worker_prefix = [ + "get_mem_pool_context", "register_kv_caches", "set_host_xfer_buffer_ops", "get_handshake_metadata", diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 39f8f88d5c..d8b6fcb5a0 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -7,6 +7,7 @@ import os import queue import tempfile import textwrap +import threading import time import uuid from collections import defaultdict @@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlKVConnectorStats, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + RemoteMeta, + ReqMeta, compute_nixl_compatibility_hash, ) from vllm.distributed.kv_transfer.kv_transfer_state import ( @@ -2087,6 +2090,10 @@ def recv_worker(): worker._recving_transfers = defaultdict(list) worker._failed_recv_reqs = queue.Queue() worker._recv_failures = set() + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._remote_agents = {} + worker._engine_by_address = {} worker._replicated_pcp_done_sending = set() worker._invalid_block_ids = queue.Queue() worker._pending_recv_notifs = {} @@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init): assert engine_id in worker.dst_xfer_side_handles +@pytest.mark.cpu_test +class TestPeerReplacement: + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw + from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend + + config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0}) + config.kv_transfer_config.kv_buffer_device = "cpu" + platform = SimpleNamespace( + device_type="cpu", + discover_numa_topology=lambda: [], + get_nixl_memory_type=lambda: "DRAM", + is_rocm=lambda: False, + ) + with ( + patch.object(bw, "NixlWrapper", FakeNixlWrapper), + patch.object(bw, "current_platform", platform), + patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), + patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), + patch.object( + bw, "get_current_attn_backends", return_value=[FlashAttentionBackend] + ), + ): + self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0) + self.nixl = self.worker.nixl_wrapper + self.transport = MagicMock(wraps=self.nixl) + self.worker.nixl_wrapper = self.transport + monkeypatch.setattr( + self.worker._handshake_initiation_executor, + "submit", + lambda *args: Future(), + ) + self._connect("old") + yield + self.worker.shutdown() + + def _connect(self, engine_id="new", host="localhost", port=1234): + future = self.worker._ensure_handshake(engine_id, host, port, 1) + self.worker.REMOTE_ENGINE_ID = engine_id + self.nixl.REMOTE_AGENT_NAME = engine_id + future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id)) + + def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True): + metadata = NixlConnectorMetadata() + metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta( + local_block_ids=(blocks,), + local_physical_block_ids=(blocks,), + tp_size=1, + remote=RemoteMeta( + ([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill" + ), + awaiting_kvs=awaiting_kvs, + ) + return metadata + + def test_cleans_old_peer_after_confirmed_handshake(self): + self._connect("healthy", host="other-host") + old_handle = self.worker.dst_xfer_side_handles["old"][0] + self.worker.start_load_kv(self._request()) + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect() + self.worker.get_transfer_results() + self.transport.release_dlist_handle.assert_called_once_with(old_handle) + self.transport.remove_remote_agent.assert_called_once_with("old") + assert set(self.worker._remote_agents) == {"new", "healthy"} + assert self.worker._engine_by_address == { + ("localhost", 1234): "new", + ("other-host", 1234): "healthy", + } + with pytest.raises(KeyError): + self.worker.transfer_topo.get_engine_info("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert self.worker.get_transfer_results().finished_recving == {"new-req"} + self.transport.remove_remote_agent.assert_called_once_with("old") + + @pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)]) + def test_preserves_other_addresses(self, host, port): + self._connect(host=host, port=port) + self.worker.get_transfer_results() + assert "old" in self.worker._remote_agents + self.transport.remove_remote_agent.assert_not_called() + + def test_preserves_old_peer_when_handshake_fails(self): + self.worker.start_load_kv(self._request()) + self.worker._handshake_futures["new"].set_exception( + RuntimeError("handshake failed") + ) + assert self.worker.get_transfer_results().failed_recving == {"new-req"} + assert "old" in self.worker._remote_agents + assert self.worker._engine_by_address == {("localhost", 1234): "old"} + self.transport.remove_remote_agent.assert_not_called() + + @pytest.mark.parametrize( + "states,failed", + [ + (["PROC", "DONE"], False), + (["ERR", "PROC", "DONE"], True), + (["ERR", "PROC", "ERR"], True), + ], + ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"], + ) + def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed): + self.worker._recving_metadata.update(self._request("old").reqs_to_recv) + self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1] + self.transport.check_xfer_state.side_effect = states + self._connect() + + result = self.worker.get_transfer_results() + assert result.finished_recving == result.failed_recving == set() + assert self.worker.get_block_ids_with_load_errors() == set() + self.transport.remove_remote_agent.assert_not_called() + + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == ({"old-req"} if failed else set()) + assert self.worker.get_block_ids_with_load_errors() == ( + {1} if failed else set() + ) + assert self.transport.release_xfer_handle.call_count == (2 if failed else 1) + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_queued_notification(self): + metadata = self._request("old", blocks=()) + self.worker._recving_metadata.update(metadata.reqs_to_recv) + self.worker._background_nixl_handshake( + "old-req", "old", metadata.reqs_to_recv["old-req"] + ) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self.worker.start_load_kv(NixlConnectorMetadata()) + self.transport.send_notif.assert_called_once_with( + "old", notif_msg=b"old-prefill:1" + ) + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == set() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_other_handshakes(self): + self.worker._ensure_handshake("other", "other-host", 1234, 1) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect("other", host="other-host") + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_rehandshakes_queued_request_for_released_engine(self): + # A request queued after its engine's handshake, but drained only after + # that engine was replaced and released, must not read from it. + metadata = self._request("old", blocks=(), awaiting_kvs=False) + self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"])) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert "old" in self.worker._handshake_futures + self.transport.send_notif.assert_not_called() + + def test_transfer_topology_unregister(): """TransferTopology.unregister_remote_engine removes the engine.""" from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 3be1730ec3..125dfa6143 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -4,6 +4,7 @@ import gc import queue +import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -54,6 +55,9 @@ def region_pull_worker(): worker.enable_permute_local_kv = False worker.enable_heterogeneous_attn_post_process = False worker._engine_last_active = {} + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._transfer_layer_group_ids = () worker._bidirectional_kv_xfer_enabled = False worker._recving_transfers = {} worker.use_mla, worker._has_mamba = True, False @@ -94,6 +98,7 @@ def region_pull_worker(): worker.dst_num_blocks = {"P": 100, "D": 100} worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]} worker._remote_agents = {"P": {(0, 0): "P-rank0"}} + worker._engine_by_address = {("localhost", 1): "P"} worker._read_blocks_mixed = MagicMock() worker.nixl_wrapper = MagicMock() return worker diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index a4bfa9ab83..f2999bdb13 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -13,6 +13,7 @@ mid-decode (silent corruption of an unrelated request). from collections import defaultdict from threading import Event, Lock +from types import SimpleNamespace from unittest.mock import MagicMock, patch import numpy as np @@ -120,16 +121,16 @@ def test_local_descriptors_follow_each_region_pool_capacity(): assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032] -@pytest.mark.cpu_test -@pytest.mark.parametrize("push_pp", [False, True]) -def test_overlaid_transfer_groups_share_region_geometry(push_pp): - """Groups overlaid on one allocation share its transfer region.""" - import msgspec +def _register_overlaid_mla_worker( + *, push_pp: bool = False, tail_bytes: int = 0, page_covers_view: bool = True +): + """Register two MLA layers overlaid on one allocation. + ``tail_bytes`` leaves spare bytes past the last block, the way an allocation + rounded up to a page boundary does. Clearing ``page_covers_view`` narrows + each layer's view below its page, leaving the block interior non-contiguous. + """ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlAgentMetadata, - ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( NixlPushConnectorWorker, ) @@ -152,10 +153,12 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): ) page_size = spec.page_size_bytes block_stride = 2 * page_size - backing = torch.zeros(num_blocks, block_stride, dtype=torch.uint8) + view_size = page_size if page_covers_view else page_size - 8 + allocation = torch.zeros(num_blocks * block_stride + tail_bytes, dtype=torch.uint8) + backing = allocation[: num_blocks * block_stride].view(num_blocks, block_stride) caches = { - "layer.0": backing[:, :page_size], - "layer.1": backing[:, :page_size], + "layer.0": backing[:, :view_size], + "layer.1": backing[:, :view_size], } groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches] @@ -203,14 +206,9 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): worker._logical_num_blocks = num_blocks worker.region_mem_types = [] worker.region_group_ids = [] - worker.region_mem_types = [] worker._mixed_mem_types = False worker.region_names = [] worker.region_num_blocks = [] - worker._mixed_mem_types = False - worker._desc_is_dram_by_block_size = {} - worker._desc_pos_by_block_size = {} - worker._dram_src_handles_by_block_size = {} worker._transfer_layer_names = () worker._transfer_layer_region_indices = () worker._transfer_layer_group_ids = () @@ -231,7 +229,7 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): num_blocks=num_blocks, kv_cache_tensors=[ KVCacheTensor( - size=backing.nbytes, + size=allocation.nbytes, layers=[name], layer_stride=page_size, block_stride=block_stride, @@ -241,21 +239,43 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): kv_cache_groups=groups, ) - transfer_topology = MagicMock() - with ( - patch.object(bw, "TransferTopology", return_value=transfer_topology), + patch.object(bw, "TransferTopology", return_value=MagicMock()), patch.object(bw, "compute_nixl_compatibility_hash", return_value="hash"), ): worker.register_kv_caches(caches) + return SimpleNamespace( + worker=worker, + allocation=allocation, + num_blocks=num_blocks, + block_stride=block_stride, + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("push_pp", [False, True]) +def test_overlaid_transfer_groups_share_region_geometry(push_pp): + """Groups overlaid on one allocation share its transfer region.""" + import msgspec + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlAgentMetadata, + ) + + registered = _register_overlaid_mla_worker(push_pp=push_pp) + worker = registered.worker + allocation = registered.allocation + num_blocks = registered.num_blocks + block_stride = registered.block_stride + assert worker.region_group_ids == [-1] assert worker.block_stride_per_layer == [block_stride] assert worker.nixl_wrapper.registered[0][0] == [ - (backing.data_ptr(), backing.nbytes, 0, "") + (allocation.data_ptr(), allocation.nbytes, 0, "") ] expected_addrs = [ - backing.data_ptr() + block * block_stride for block in range(num_blocks) + allocation.data_ptr() + block * block_stride for block in range(num_blocks) ] num_desc_regions = 2 if push_pp else 1 assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions @@ -275,6 +295,39 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]] +def _descriptor_geometry(registered) -> dict: + """The geometry handed to NIXL, addressed relative to the allocation.""" + worker = registered.worker + base = registered.allocation.data_ptr() + return { + "block_len_per_layer": list(worker.block_len_per_layer), + "block_stride_per_layer": list(worker.block_stride_per_layer), + "desc_offsets": [addr - base for addr in worker.src_blocks_data[:, 0].tolist()], + "desc_lens": worker.src_blocks_data[:, 1].tolist(), + } + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "page_covers_view", [True, False], ids=["dense-page", "narrow-view"] +) +def test_registration_ignores_a_sub_block_padding_tail(page_covers_view): + """A tail shorter than one block must not move a single descriptor. + + An allocation rounded up to a page boundary keeps a few spare bytes past the + last block, which cannot hold another block. The narrow view is the branch + DeepSeek-V4-Flash's sliding-window cache lands in. + """ + unpadded = _register_overlaid_mla_worker(page_covers_view=page_covers_view) + # More spare bytes than blocks, so dividing the padded length by the block + # count overshoots the stride and would shift every descriptor. + padded = _register_overlaid_mla_worker( + tail_bytes=2 * unpadded.num_blocks, page_covers_view=page_covers_view + ) + + assert _descriptor_geometry(padded) == _descriptor_geometry(unpadded) + + def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blocks): """Build a real pull worker with a hybrid MLA + 2xKDA HMA layout.""" from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( diff --git a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py b/tests/v1/kv_connector/unit/test_nixl_heartbeat.py index 345e10621f..f2d95daa48 100644 --- a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py +++ b/tests/v1/kv_connector/unit/test_nixl_heartbeat.py @@ -30,6 +30,10 @@ def _worker_stub(): w = object.__new__(NixlConnectorWorker) w._reqs_to_send = {} w._lease_extension = 20 + w._reqs_to_process = set() + w.consumer_notification_counts_by_req = {} + w.expected_consumer_notifications_by_req = {} + w.xfer_stats = MagicMock() return w @@ -163,3 +167,87 @@ def test_handle_heartbeat(): assert w._reqs_to_send["req-b"] >= far_future # req-unknown: not added. assert "req-unknown" not in w._reqs_to_send + + +def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch): + """A heartbeated head entry must not strand expired leases behind it. + + Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222: + A is inserted first and kept alive via heartbeats; B and C expire and + must still be reaped. The old early-exit scan returned nothing here. + """ + w = _worker_stub() + w._reqs_to_process.update(("A", "B", "C")) + # Insertion order is A, B, C. Heartbeats will keep A at the head. + w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + now = {"t": 0.0} + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"]) + for t in (20.0, 40.0, 60.0, 80.0): + now["t"] = t + w._handle_heartbeat("A") + + # A was renewed in place: still the dict head, expiry 100. + assert next(iter(w._reqs_to_send)) == "A" + assert w._reqs_to_send["A"] == pytest.approx(100.0) + + done: set[str] = set() + now["t"] = 50.0 + w._reap_expired_send_leases(done) + + assert done == {"B", "C"} + assert list(w._reqs_to_send) == ["A"] + assert w._reqs_to_process == {"A"} + assert w.xfer_stats.record_kv_expired_req.call_count == 2 + + # now == expires is expired (now >= expires): A is due at 100. + done_eq: set[str] = set() + now["t"] = 100.0 + w._reap_expired_send_leases(done_eq) + assert done_eq == {"A"} + assert not w._reqs_to_send + assert not w._reqs_to_process + assert w.xfer_stats.record_kv_expired_req.call_count == 3 + + +def test_reaper_reclaims_when_now_equals_expiry(monkeypatch): + """The reaper must reclaim on the now == expires boundary.""" + w = _worker_stub() + w._reqs_to_process.add("eq") + w._reqs_to_send = {"eq": 50.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) + + done: set[str] = set() + w._reap_expired_send_leases(done) + + assert done == {"eq"} + assert not w._reqs_to_send + assert not w._reqs_to_process + w.xfer_stats.record_kv_expired_req.assert_called_once() + + +def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch): + """A later-inserted longer TTL must not block an earlier-expiring entry. + + Delete+reinsert on heartbeat cannot restore expiry order in this case + (issue #58222), so the reaper must scan the whole map. + """ + w = _worker_stub() + w._reqs_to_process.update(("X", "Y")) + w._reqs_to_send = {"X": 55.0, "Y": 46.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) + + done: set[str] = set() + w._reap_expired_send_leases(done) + + assert done == {"Y"} + assert list(w._reqs_to_send) == ["X"] + assert w._reqs_to_process == {"X"} diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index e35ead0015..4d8ead28f3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -50,7 +50,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( NixlKVConnectorStats, ) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + ReadSpec, + TPMapping, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( get_base_request_id, ) @@ -344,6 +347,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): ) w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + w._send_failures = set() w._sending_transfers_lock = threading.Lock() w._push_finished_blocks = {} w._pending_d_registrations = {} @@ -391,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._group_spec_types = (FullAttentionSpec,) w._engine_ttl = 0.0 w._engine_last_active = {} + w._engine_by_address = {} # Track _do_start_push_kv invocations. calls: list[tuple[str, Any, dict[str, Any]]] = [] @@ -852,6 +857,61 @@ def test_cleanup_remote_engine_pops_under_handshake_lock(): w.nixl_wrapper.remove_remote_agent.assert_called_once_with("agent-D-old") +def _recv_metadata(remote_engine_id: str, req_id: str = "req") -> NixlConnectorMetadata: + """Build metadata with a single ``reqs_to_recv`` entry pointing at + ``remote_engine_id`` (the prefill remote D receives a push from).""" + params = { + "remote_block_ids": ([0],), + "remote_engine_id": remote_engine_id, + "remote_request_id": "p-req", + "remote_host": "localhost", + "remote_port": 1234, + "tp_size": 1, + } + meta = NixlConnectorMetadata() + meta.add_new_req_to_recv(req_id, ([0],), params) + return meta + + +def test_start_load_kv_refreshes_engine_last_active_on_d_side(): + """D receiving a push refreshes ``_engine_last_active`` for the prefill + remote, but only once that engine is connected (handshaked into + ``_remote_agents`` via heartbeats), mirroring the P-side refresh which runs + after ``_ensure_handshake``. Without it the prefill engine is reaped + mid-stream once it passes its TTL.""" + w = _StubWriterWorker.fresh() + w._send_heartbeats = lambda metadata: None + w._logical_to_kernel_block_ids = lambda x, ratio: x + # Connected prefill engine (already handshaked in). + w._remote_agents["prefill-engine"] = {(0, 0): "agent-p"} + + stale = time.perf_counter() - 5.0 + w._engine_last_active["prefill-engine"] = stale + + w.start_load_kv(_recv_metadata("prefill-engine")) + + assert w._engine_last_active["prefill-engine"] > stale + assert "req" in w._recving_metadata + + +def test_start_load_kv_skips_liveness_for_unconnected_engine(): + """If the prefill engine was never handshaked into ``_remote_agents`` (e.g. + its heartbeat handshake never succeeded), the D-side refresh must NOT record + an ``_engine_last_active`` entry -- otherwise a later eviction pass hits the + ``_remote_agents`` invariant in ``_cleanup_remote_engine`` and crashes on + the D node.""" + w = _eviction_worker(engine_ttl=30.0) + w._send_heartbeats = lambda metadata: None + + w.start_load_kv(_recv_metadata("prefill-engine")) + + # Unconnected -> no orphaned liveness entry, so eviction has nothing to trip on. + assert "prefill-engine" not in w._engine_last_active + assert "prefill-engine" not in w._remote_agents + + w._evict_stale_engines() # nothing to reap, must not raise + + class TestPushWriterNotifs: def test_get_new_notifs_processes_forwarded_completion_notif(self): """Non-PUSH_REG notifs forwarded by the writer thread are drained @@ -941,9 +1001,10 @@ class TestPushWriterNotifs: assert request_id not in w._sending_transfers @pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)]) - def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma): - """P-side sends are completed by consumer notifs / lease expiry, - not by local WRITE-handle completion.""" + def test_completed_push_send_releases_blocks_without_consumer_notif( + self, pp_size, is_hma + ): + """All WRITEs completing lets P reclaim its blocks without a D ACK.""" w = self._pollable_worker() w.pp_size = pp_size w._is_hma_required = is_hma @@ -951,20 +1012,14 @@ class TestPushWriterNotifs: w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"] results = w.get_transfer_results() + assert request_id not in results.finished_sending + assert request_id in w._reqs_to_send assert w._sending_transfers[request_id] == [102] - assert request_id in w._reqs_to_send + assert w._evict_finished_inbox.empty() results = w.get_transfer_results() - - assert request_id not in results.finished_sending - assert request_id in w._reqs_to_send - assert request_id not in w._sending_transfers - - # The consumer notif is what completes the send. - w._pending_completion_notifs.put(f"{request_id}:1".encode()) - results = w.get_transfer_results() - assert request_id in results.finished_sending + assert results.finished_sending == {request_id} assert request_id not in w._reqs_to_send assert request_id not in w._reqs_to_process assert w._evict_finished_inbox.get_nowait() == request_id @@ -973,6 +1028,91 @@ class TestPushWriterNotifs: assert w.nixl_wrapper.release_xfer_handle.call_count == 2 w.xfer_stats.record_kv_expired_req.assert_not_called() + @pytest.mark.parametrize("sibling_state", ["ERR", "DONE"]) + def test_push_failure_survives_until_lease_expiry(self, sibling_state): + """A later sibling completion must not turn a failed send into success.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state] + + assert w.get_transfer_results().finished_sending == set() + assert w._sending_transfers[request_id] == [102] + assert w.get_transfer_results().finished_sending == set() + assert request_id not in w._sending_transfers + assert request_id in w._reqs_to_send + assert w._evict_finished_inbox.empty() + + w.expected_consumer_notifications_by_req = {} + w._reqs_to_send[request_id] = time.perf_counter() - 1 + assert w.get_transfer_results().finished_sending == {request_id} + assert request_id not in w._send_failures + assert w._evict_finished_inbox.get_nowait() == request_id + + @pytest.mark.parametrize("release_fails", [False, True]) + def test_push_submission_failure_is_remembered_after_sibling_success( + self, release_fails + ): + """A submission error remains a failure even if later polls succeed.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.transfer_topo.block_size_ratio.return_value = 1 + w._apply_prefix_caching = MagicMock(return_value=([[1]], [[1]])) + w._compute_desc_ids = MagicMock(return_value=[0]) + w.dst_num_blocks = {w.engine_id: 8} + w.dst_region_num_blocks[w.engine_id] = [8] + w.dst_region_group_ids[w.engine_id] = [0] + w.dst_uses_region_group_mapping[w.engine_id] = False + w.nixl_wrapper.make_prepped_xfer.return_value = 101 + w.nixl_wrapper.transfer.side_effect = RuntimeError("submit") + if release_fails: + w.nixl_wrapper.release_xfer_handle.side_effect = [ + RuntimeError("release"), + None, + None, + ] + + handle = w._xfer_blocks( + read_spec=ReadSpec(0, [[1]], [[1]]), + dst_engine_id=w.engine_id, + request_id=request_id, + remote_request_id="decode-request", + local_xfer_side_handle=1, + remote_xfer_side_handle=2, + ) + assert handle == (101 if release_fails else None) + w._sending_transfers[request_id] = [102] + if handle is not None: + w._sending_transfers[request_id].append(handle) + w.nixl_wrapper.check_xfer_state.return_value = "DONE" + + assert w.get_transfer_results().finished_sending == set() + assert request_id in w._reqs_to_send + assert request_id not in w._sending_transfers + assert w._evict_finished_inbox.empty() + + @pytest.mark.parametrize("first_state", ["ERR", "DONE"]) + def test_push_completion_after_expiry_is_not_reported_twice(self, first_state): + """A WRITE finishing after lease expiry must not complete the request again.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.side_effect = [ + first_state, + "PROC", + "PROC", + "DONE", + ] + assert w.get_transfer_results().finished_sending == set() + + w.expected_consumer_notifications_by_req = {} + w._reqs_to_send[request_id] = time.perf_counter() - 1 + assert w.get_transfer_results().finished_sending == {request_id} + assert w._evict_finished_inbox.get_nowait() == request_id + + assert w.get_transfer_results().finished_sending == set() + assert request_id not in w._sending_transfers + assert request_id not in w._send_failures + assert w._evict_finished_inbox.empty() + # ----------------------------------------------------------------- # # Negative / error-path tests # diff --git a/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py b/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py index da8be07157..5d617a7ec3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py +++ b/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py @@ -12,6 +12,7 @@ import gc import pytest import torch +from tests.utils import wait_for_gpu_memory_to_clear from vllm.platforms import current_platform pytestmark = pytest.mark.skipif( @@ -19,6 +20,8 @@ pytestmark = pytest.mark.skipif( reason="ROCm platform required", ) +MAX_LEAK_PCT = 10 + def _mb(b: int) -> float: return b / (1024 * 1024) @@ -61,6 +64,18 @@ def _full_gpu_cleanup(): torch.accelerator.empty_cache() +def _wait_for_driver_memory_release(baseline: dict, peak: dict) -> None: + # Process exit can precede ROCm's release of the backing allocation. + peak_delta_mb = max(0.0, peak["drv_used_mb"] - baseline["drv_used_mb"]) + target_mb = baseline["drv_used_mb"] + peak_delta_mb * MAX_LEAK_PCT / 100 + wait_for_gpu_memory_to_clear( + devices=[torch.accelerator.current_device_index()], + threshold_bytes=int(target_mb * 1024**2), + timeout_s=120, + poll_interval_s=1, + ) + + @pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)]) def test_gpu_memory_nixl_hma(model_name, sw_size): """Track GPU memory through NixlConnector create/infer/shutdown cycle.""" @@ -116,7 +131,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): # shutdown + cleanup print("\n--- shutdown ---") - llm.llm_engine.engine_core.shutdown() + llm.llm_engine.engine_core.shutdown(timeout=60) _gpu_snapshot("3. after shutdown()", snap2["alloc_mb"]) del llm @@ -126,6 +141,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): torch._dynamo.reset() gc.collect() torch.accelerator.empty_cache() + _wait_for_driver_memory_release(snap0, snap2) snap_final = _gpu_snapshot("4. final", snap2["alloc_mb"]) # summary @@ -160,7 +176,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): # Peak driver memory used above baseline drv_peak = snap2["drv_used_mb"] - drv_base leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0 - max_leak_pct = 10 + max_leak_pct = MAX_LEAK_PCT assert leak_pct <= max_leak_pct, ( f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory " f"not freed after NixlConnector shutdown " @@ -196,7 +212,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): llm.generate(["hi " * 500], SamplingParams(max_tokens=1)) snap_peak = _gpu_snapshot("after generate()", snap0["alloc_mb"]) - llm.llm_engine.engine_core.shutdown() + llm.llm_engine.engine_core.shutdown(timeout=60) del llm _full_gpu_cleanup() cleanup_dist_env_and_memory() @@ -204,6 +220,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): torch._dynamo.reset() gc.collect() torch.accelerator.empty_cache() + _wait_for_driver_memory_release(snap0, snap_peak) snap_final = _gpu_snapshot("final", snap0["alloc_mb"]) drv_base = snap0["drv_used_mb"] @@ -213,7 +230,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): print("=" * 90) leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0 - max_leak_pct = 10 + max_leak_pct = MAX_LEAK_PCT assert leak_pct <= max_leak_pct, ( f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory " f"not freed after baseline shutdown " diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py new file mode 100644 index 0000000000..16358dec7e --- /dev/null +++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py @@ -0,0 +1,466 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SimpleCPUOffloadConnector metrics.""" + +from __future__ import annotations + +import dataclasses +from types import SimpleNamespace +from typing import Any + +import pytest +from prometheus_client import Counter, Gauge, Histogram + +from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import ( + _make_connector, +) +from tests.v1.simple_kv_offload.test_scheduler import ( + make_request, + make_scheduler, + make_scheduler_output, +) +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) +from vllm.v1.outputs import KVConnectorOutput +from vllm.v1.simple_kv_offload.manager import ( + BoundaryStoreStats, + LoadRequestState, + TransferMeta, +) +from vllm.v1.simple_kv_offload.metrics import ( + INFO_LABELS, + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, + _MetricType, + _StatsKey, +) + +SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES +LOAD_BLOCKS = MetricName.LOAD_BLOCKS +USED_BLOCKS = MetricName.USED_BLOCKS +PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS +INFO = MetricName.INFO + + +class _FakeMetric: + def __init__(self, **kwargs: Any): + self.kwargs = kwargs + self.children: list[_FakeMetric] = [] + self.observed: list[int | float] = [] + self.increments: list[int | float] = [] + self.set_values: list[int | float] = [] + self.labelvalues: tuple[object, ...] = () + + def labels(self, *labelvalues: object) -> _FakeMetric: + child = _FakeMetric(**self.kwargs) + child.labelvalues = labelvalues + self.children.append(child) + return child + + def observe(self, value: int | float) -> None: + self.observed.append(value) + + def inc(self, value: int | float) -> None: + self.increments.append(value) + + def set(self, value: int | float) -> None: + self.set_values.append(value) + + +def _fake_metric_types() -> dict[type, type]: + return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric} + + +def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + return SimpleCPUOffloadPromMetrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues=per_engine or {0: ["model", "0"]}, + ) + + +def _stats_payload(types: dict, data: dict) -> dict[str, Any]: + return {_StatsKey.TYPES: types, _StatsKey.DATA: data} + + +def _values(stats: SimpleCPUOffloadStats, name: str) -> dict: + return stats.data[_StatsKey.DATA][name] + + +# --------------------------------------------------------------------------- +# Stats class +# --------------------------------------------------------------------------- + + +def test_build_kv_connector_stats_none_and_empty() -> None: + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={}) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + +def test_stats_round_trip_through_serialized_dict() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.increase_counter(LOAD_BLOCKS, 4) + stats.set_gauge(USED_BLOCKS, 7) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict()) + assert isinstance(rebuilt, SimpleCPUOffloadStats) + assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3} + assert _values(rebuilt, LOAD_BLOCKS) == {(): 4} + assert _values(rebuilt, USED_BLOCKS) == {(): 7} + assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1} + + +def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",)) + stats1.set_gauge(USED_BLOCKS, 7) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",)) + stats2.set_gauge(USED_BLOCKS, 2) + + result = stats1.aggregate(stats2) + assert result is stats1 + outcomes = _values(stats1, SAVE_OUTCOMES) + assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1} + assert _values(stats1, USED_BLOCKS) == {(): 2} + + +def test_stats_aggregate_merges_types_from_other() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.set_gauge(USED_BLOCKS, 1) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(LOAD_BLOCKS, 2) + + stats1.aggregate(stats2) + assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER + assert _values(stats1, LOAD_BLOCKS) == {(): 2} + + +def test_stats_reduce_and_reset() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.set_gauge(PENDING_STORE_BLOCKS, 4) + stats.increase_counter(LOAD_BLOCKS, 9) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + reduced = stats.reduce() + assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3 + assert reduced[PENDING_STORE_BLOCKS] == 4 + assert reduced[LOAD_BLOCKS] == 9 + assert not any(key.startswith(INFO) for key in reduced) + + stats.reset() + assert stats.is_empty() + + +def test_outcome_labels_match_boundary_store_stats_fields() -> None: + """Every outcome label maps 1:1 to a BoundaryStoreStats counter field.""" + boundary_fields = { + f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published" + } + assert set(OUTCOME_TO_FIELD.values()) == boundary_fields + assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD) + + +# --------------------------------------------------------------------------- +# Prom metrics class +# --------------------------------------------------------------------------- + + +def test_prom_metrics_registers_tier1_metrics() -> None: + prom = _make_prom_metrics() + defs = prom._defs + assert set(defs) == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + # The prom class registers literal names (required by the docs + # generator); they must stay in sync with the emission-side constants. + assert {m.kwargs["name"] for m in defs.values()} == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + for metric in defs.values(): + assert metric.kwargs["documentation"] + assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"] + + assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [ + "model_name", + "engine", + "outcome", + ] + assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + assert defs[INFO].kwargs["labelnames"] == [ + "model_name", + "engine", + *INFO_LABELS, + ] + assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + + # mostrecent: multiprocess scrapes expose one series per label set, not + # one pid-tagged series per writer process. + for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO): + assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent" + + +def test_prom_metrics_observe_routes_each_series() -> None: + prom = _make_prom_metrics() + prom.observe( + _stats_payload( + types={ + SAVE_OUTCOMES: _MetricType.COUNTER, + LOAD_BLOCKS: _MetricType.COUNTER, + USED_BLOCKS: _MetricType.GAUGE, + PENDING_STORE_BLOCKS: _MetricType.GAUGE, + INFO: _MetricType.GAUGE, + }, + data={ + SAVE_OUTCOMES: {("stored",): 3}, + LOAD_BLOCKS: {(): 4}, + USED_BLOCKS: {(): 5}, + PENDING_STORE_BLOCKS: {(): 1}, + INFO: {("cpu", "false", "false", "64"): 1}, + }, + ) + ) + + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3] + assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4] + assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5] + assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1] + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1] + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == ( + "model", + "0", + "stored", + ) + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == ( + "model", + "0", + "cpu", + "false", + "false", + "64", + ) + + +def test_prom_metrics_reuses_bound_children() -> None: + prom = _make_prom_metrics() + payload = _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored",): 1}}, + ) + prom.observe(payload) + prom.observe(payload) + + child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))] + assert child.increments == [1, 1] + assert len(prom._defs[SAVE_OUTCOMES].children) == 1 + + +def test_prom_metrics_rejects_unknown_metric() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="Unknown"): + prom.observe( + _stats_payload( + types={"vllm:not_a_metric": _MetricType.COUNTER}, + data={"vllm:not_a_metric": {(): 1}}, + ) + ) + + +def test_prom_metrics_rejects_wrong_label_count() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="labels"): + prom.observe( + _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored", "extra"): 1}}, + ) + ) + + +def test_prom_metrics_routes_per_engine() -> None: + prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]}) + prom.observe( + _stats_payload( + types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}} + ), + engine_idx=1, + ) + assert (0, USED_BLOCKS, ()) not in prom._metrics + assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3] + + +def test_connector_build_prom_metrics() -> None: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + prom = SimpleCPUOffloadConnector.build_prom_metrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + assert isinstance(prom, SimpleCPUOffloadPromMetrics) + + +# --------------------------------------------------------------------------- +# Manager: interval stats +# --------------------------------------------------------------------------- + + +def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.boundary_store_stats.dropped_cpu_full += 2 + sched.boundary_store_stats.skipped_in_flight += 1 + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == { + ("stored",): 3, + ("dropped_cpu_full",): 2, + ("skipped_in_flight",): 1, + } + + # Deltas are per-interval: a second drain reports no new outcomes. + stats2 = sched.get_stats() + assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA] + + +def test_manager_reset_preserves_undrained_outcome_deltas() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.reset() + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3} + + +def test_manager_get_stats_gauges_and_info_labels() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + sched._store_event_to_blocks[0] = TransferMeta([10], [11]) + sched._pending_finished_stores.append(TransferMeta([12], [13])) + sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15]) + + stats = sched.get_stats() + assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3} + used = _values(stats, USED_BLOCKS)[()] + assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks() + + info = _values(stats, INFO) + assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1} + + # Gauges are re-reported every drain. + stats2 = sched.get_stats() + assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3} + + +def test_manager_info_labels_reflect_disk_mode() -> None: + connector = _make_connector( + extra_config={ + "kv_offload_backend": "disk", + "disk_path": "/tmp/fake-disk-metrics", + "disk_capacity_bytes": 1024**3, + "use_page_cache": True, + } + ) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[0] == "disk" + assert sched._info_labelvalues[1] == "true" + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None: + """CPU backend ignores use_page_cache; the info gauge must not claim it.""" + connector = _make_connector(extra_config={"use_page_cache": True}) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[:2] == ("cpu", "false") + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_counts_load_blocks_completed() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} + + +def test_manager_abandoned_load_counts_as_completed_after_reset() -> None: + """Loads abandoned by reset() that still finish on the worker are + counted as loaded blocks. + """ + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + assert sched.reset() is False + assert set(sched._abandoned_reqs_to_load) == {request.request_id} + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + assert sched._abandoned_reqs_to_load == {} + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 85ada84b0d..dda7c0f386 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -632,3 +632,29 @@ def make_nixl_push_scheduler( sched.blocks_per_sw = [] return sched + + +def make_moriio_writer(fake_worker: Any) -> Any: + """Build a MoRIIOWriter with internals stubbed for unit tests. + + Bypasses ``__init__`` and wires only the write/finalize state the tests + touch, including the deferred-task fields used by the routing suite. + """ + import threading + from queue import Queue + + from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( + MoRIIOWriter, + ) + + writer = MoRIIOWriter.__new__(MoRIIOWriter) + writer._worker_ref = lambda: fake_worker + writer._write_task_q = Queue() + writer._write_state_lock = threading.Lock() + writer._scheduled_writes = defaultdict(int) + writer._scheduled_layers = defaultdict(set) + writer._sealed_writes = {} + writer._deferred_tasks = [] + writer._defer_timeout = 60.0 + writer.ensure_worker_started = lambda: None + return writer diff --git a/tests/v1/logits_processors/test_loader.py b/tests/v1/logits_processors/test_loader.py new file mode 100644 index 0000000000..07f9ffbff2 --- /dev/null +++ b/tests/v1/logits_processors/test_loader.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for loading custom logits processors by FQCN. + +Kept separate from `test_custom_offline.py`, which runs end-to-end against a +downloaded model; these exercise the loader alone and need no model. +""" + +import pytest + +from vllm.v1.sample.logits_processor import _load_logitsprocs_by_fqcns + + +@pytest.mark.parametrize( + "fqcn", + [ + "invalid_fqcn_without_colon", + "invalid:fqcn:with:multiple:colons", + ], + ids=["missing-colon", "too-many-colons"], +) +def test_invalid_fqcn_rejected(fqcn: str): + with pytest.raises(ValueError, match="Invalid logits processor FQCN"): + _load_logitsprocs_by_fqcns([fqcn]) diff --git a/tests/v1/metrics/test_engine_logger_apis.py b/tests/v1/metrics/test_engine_logger_apis.py index 5741df0d56..e8fc1375e2 100644 --- a/tests/v1/metrics/test_engine_logger_apis.py +++ b/tests/v1/metrics/test_engine_logger_apis.py @@ -7,6 +7,7 @@ import pytest from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501 DummyStatLogger, ) +from tests.utils import wait_for_memory_to_settle from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger @@ -30,8 +31,15 @@ async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args): engine = AsyncLLM.from_engine_args( log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger] ) - assert isinstance(engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger) - engine.shutdown() + try: + assert isinstance( + engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger + ) + finally: + engine.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization + ) @pytest.mark.asyncio @@ -48,14 +56,18 @@ async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args): disabled_log_engine_args, stat_loggers=[DummyStatLogger] ) - assert len(engine.logger_manager.stat_loggers) == 2 - assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 - assert isinstance( - engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], - DummyStatLogger, - ) + try: + assert len(engine.logger_manager.stat_loggers) == 2 + assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 + assert isinstance( + engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], + DummyStatLogger, + ) - # log_stats is still True, since custom stat loggers are used - assert engine.log_stats - - engine.shutdown() + # log_stats is still True, since custom stat loggers are used + assert engine.log_stats + finally: + engine.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization + ) diff --git a/tests/v1/metrics/test_ray_metrics.py b/tests/v1/metrics/test_ray_metrics.py index ce4ebcbfdb..8552e896c0 100644 --- a/tests/v1/metrics/test_ray_metrics.py +++ b/tests/v1/metrics/test_ray_metrics.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pytest import ray +from tests.utils import wait_for_memory_to_settle from vllm.config.model import ModelDType from vllm.platforms import current_platform from vllm.sampling_params import SamplingParams @@ -46,31 +47,39 @@ def test_engine_log_metrics_ray( ) -> None: """Simple smoke test, verifying this can be used without exceptions. Need to start a Ray cluster in order to verify outputs.""" + engine_args = AsyncEngineArgs( + model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True + ) @ray.remote(num_gpus=1) class EngineTestActor: async def run(self): - engine_args = AsyncEngineArgs( - model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True - ) - engine = AsyncLLM.from_engine_args( engine_args, stat_loggers=[RayPrometheusStatLogger] ) - for i, prompt in enumerate(example_prompts): - results = engine.generate( - request_id=f"request-id-{i}", - prompt=prompt, - sampling_params=SamplingParams(max_tokens=max_tokens), - ) + try: + for i, prompt in enumerate(example_prompts): + results = engine.generate( + request_id=f"request-id-{i}", + prompt=prompt, + sampling_params=SamplingParams(max_tokens=max_tokens), + ) - async for _ in results: - pass + async for _ in results: + pass + finally: + engine.shutdown() # Create the actor and call the async method - actor = EngineTestActor.remote() # type: ignore[attr-defined] - ray.get(actor.run.remote()) + try: + actor = EngineTestActor.remote() # type: ignore[attr-defined] + ray.get(actor.run.remote()) + finally: + ray.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - engine_args.gpu_memory_utilization + ) def test_sanitized_opentelemetry_name(): diff --git a/tests/v1/sample/test_diffusion_gemma_reads.py b/tests/v1/sample/test_diffusion_gemma_reads.py new file mode 100644 index 0000000000..769481b06b --- /dev/null +++ b/tests/v1/sample/test_diffusion_gemma_reads.py @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-request DiffusionGemma state behind structured reads: seed canvases, +pinned positions, read-only slots and the per-slot step cap.""" + +import numpy as np +import pytest +import torch + +from vllm.model_executor.models.diffusion_gemma import ( + _MASKED_LOGIT, + DiffusionGemmaRequestStates, + _compiled_sample_step, + _concat_logprob_stashes, + _denoise_temperature, + _mask_rows_to_allowed, + sample_row_stats_reference, +) +from vllm.platforms import current_platform +from vllm.v1.outputs import LogprobsTensors + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), reason="the sampler state lives on the GPU" +) + +CL = 8 +VOCAB = 64 +MAX_REQS = 4 +MAX_STEPS = 48 + + +def _states() -> DiffusionGemmaRequestStates: + return DiffusionGemmaRequestStates( + max_num_reqs=MAX_REQS, + canvas_length=CL, + vocab_size=VOCAB, + max_denoising_steps=MAX_STEPS, + device=torch.device("cuda"), + hidden_size=4, + stability_threshold=2, + ) + + +def _slots(*idx: int) -> tuple[np.ndarray, torch.Tensor]: + slots = np.array(idx, dtype=np.int64) + return slots, torch.tensor(slots, device="cuda") + + +def test_seed_canvas_replaces_only_seeded_slots(): + states = _states() + for slot in range(3): + states.add_request(slot) + seed = list(range(CL)) + states.set_seed_canvas(1, seed) + slots, slots_gpu = _slots(0, 1, 2) + states.init_canvas(slots_gpu) + before = states.canvas[slots_gpu].clone() + + states.apply_seed_canvases(slots, slots_gpu) + + after = states.canvas[slots_gpu] + assert after[1].tolist() == seed + assert torch.equal(after[0], before[0]) + assert torch.equal(after[2], before[2]) + + +def test_apply_seed_canvases_leaves_unseeded_batches_alone(): + states = _states() + states.add_request(0) + slots, slots_gpu = _slots(0) + states.init_canvas(slots_gpu) + before = states.canvas[0].clone() + + states.apply_seed_canvases(slots, slots_gpu) + + assert torch.equal(states.canvas[0], before) + + +def test_add_request_clears_seed_and_read_only(): + states = _states() + states.add_request(0) + states.set_seed_canvas(0, [1] * CL) + states.set_read_only(0) + assert states.seeded_slots == {0} + assert states.read_only_slots == {0} + + states.add_request(0) + + assert not states.seeded_slots + assert not states.read_only_slots + assert not bool(states.has_seed[0]) + assert not bool(states.read_only[0]) + + +def test_canvas_width_resets_with_the_slot(): + states = _states() + states.add_request(0) + states.canvas_width_np[0] = 4 + states.set_seed_canvas(0, [7, 7, 7, 7]) + assert states.seed_canvas[0, :4].tolist() == [7, 7, 7, 7] + + states.add_request(0) + + assert states.canvas_width_np[0] == CL + + +def test_remove_request_forgets_the_slot(): + states = _states() + states.add_request(0) + states.set_seed_canvas(0, [1] * CL) + states.set_read_only(0) + + states.remove_request(0) + + assert not states.seeded_slots + assert not states.read_only_slots + + +def _denoise_once( + states: DiffusionGemmaRequestStates, + slots: list[int], + compute_sc: bool = True, + width: int = CL, + embed_weight: torch.Tensor | None = None, + embed_dtype: torch.dtype = torch.float32, +) -> tuple[torch.Tensor, torch.Tensor]: + """One compiled denoise step over ``slots`` with flat logits, so nothing + converges by stability or confidence and only the step cap can end it. + ``width`` below CL runs the step on [:, :width] views, as the sampler + does for a narrow tile.""" + n = len(slots) + device = states.device + decode_slots = torch.tensor(slots, dtype=torch.int64, device=device) + decode_idx = torch.arange(n, dtype=torch.int64, device=device) + sampled = torch.zeros(n, CL, dtype=torch.int32, device=device)[:, :width] + num_sampled = torch.zeros(n, dtype=torch.int32, device=device) + temp = _denoise_temperature(states.step, decode_slots, float(MAX_STEPS), 0.5, 1.0) + argmax, sample, entropy, probs = sample_row_stats_reference( + torch.zeros(n * width, VOCAB, device=device), + temp, + width, + embed_dtype if compute_sc else None, + ) + _compiled_sample_step( + sample.view(n, width), + argmax.view(n, width), + entropy.view(n, width), + probs.view(n, width, -1) if probs is not None else None, + decode_slots, + decode_idx, + decode_slots, + torch.full((n,), width, dtype=torch.int64, device=device), + states.canvas[:, :width], + states.argmax_canvas[:, :width], + states.step, + states.is_encoder_phase, + states.confident, + states.self_conditioning_embeds[:, :width], + ( + torch.zeros(VOCAB, 4, dtype=embed_dtype, device=device) + if embed_weight is None + else embed_weight + ), + torch.tensor(1.0, dtype=embed_dtype, device=device), + states.accepted_canvas_history[:, :, :width], + states.accepted_canvas_history_len, + states.max_steps, + states.pin_mask[:, :width], + states.seed_canvas[:, :width], + states.read_only, + sampled, + num_sampled, + torch.zeros(MAX_REQS, CL, dtype=torch.int64, device=device), + confidence_threshold=0.1, + vocab_size=VOCAB, + CL=width, + ST=states.stability_threshold, + entropy_bound=0.1, + sc_vocab_start=0, + sc_vocab_end=VOCAB, + tp_size=1, + tp_group_name="", + compute_sc=compute_sc, + ) + return sampled, num_sampled + + +def test_pinned_positions_hold_their_seed_through_a_step(): + states = _states() + states.add_request(0) + states.is_encoder_phase[0] = False + seed = list(range(10, 10 + CL)) + states.set_seed_canvas(0, seed) + states.canvas[0] = torch.tensor(seed, device="cuda") + states.set_pins(0, [0, 1, 2, 3]) + + # Flat logits accept nothing, so every free position is renoised. + _denoise_once(states, [0], embed_weight=torch.ones(VOCAB, 4, device="cuda")) + + assert states.canvas[0, :4].tolist() == seed[:4] + # The soft embed is zero at pinned positions and non-zero elsewhere. + sc = states.self_conditioning_embeds[0] + assert not sc[:4].any() + assert sc[4:].any() + + +def test_add_request_clears_pins(): + states = _states() + states.add_request(0) + states.set_seed_canvas(0, [1] * CL) + states.set_pins(0, [2, 3]) + assert states.pin_mask[0].tolist() == [False, False, True, True] + [False] * 4 + + states.add_request(0) + + assert not states.pin_mask[0].any() + + +def test_single_step_tile_skips_self_conditioning(): + states = _states() + states.add_request(0) + states.is_encoder_phase[0] = False + states.self_conditioning_embeds[0] = 1.0 + + _denoise_once(states, [0], compute_sc=False) + + assert not states.self_conditioning_embeds[0].any() + + +@pytest.mark.parametrize("stance", ["default", "force_eager"]) +def test_self_conditioning_stores_a_bf16_model_in_the_fp32_buffer(stance): + # The model's embeddings are bf16 while the buffer is fp32. Compiled code + # casts on the store, but eager does not, and the step runs eager once + # torch.compile hits its recompile limit. + states = _states() + states.add_request(0) + states.is_encoder_phase[0] = False + + with torch.compiler.set_stance(stance): + _denoise_once( + states, + [0], + embed_weight=torch.ones(VOCAB, 4, dtype=torch.bfloat16, device="cuda"), + embed_dtype=torch.bfloat16, + ) + + assert states.self_conditioning_embeds.dtype == torch.float32 + # uniform probs @ an all-ones embedding + assert torch.allclose( + states.self_conditioning_embeds[0], torch.ones(CL, 4, device="cuda") + ) + + +def test_narrow_tile_leaves_the_rest_of_the_canvas_alone(): + states = _states() + states.add_request(0) + states.is_encoder_phase[0] = False + states.canvas[0] = 5 + states.argmax_canvas[0] = 5 + + _denoise_once(states, [0], width=4) + + # Columns past the width are untouched; the step counted. + assert states.canvas[0, 4:].tolist() == [5] * (CL - 4) + assert states.argmax_canvas[0, 4:].tolist() == [5] * (CL - 4) + assert states.step[0] == 1 + + +def test_step_cap_is_per_slot(): + states = _states() + for slot in (0, 1): + states.add_request(slot) + states.is_encoder_phase[slot] = False + states.max_steps[0] = 1 + + _denoise_once(states, [0, 1]) + + # Slot 0 hit its cap and moves to commit. Slot 1 keeps denoising. + assert states.is_encoder_phase[:2].tolist() == [True, False] + assert states.step[:2].tolist() == [1, 1] + + +def _stash(rows: int, width: int, first_id: int) -> LogprobsTensors: + ids = torch.arange(first_id, first_id + rows * width).reshape(rows, width) + return LogprobsTensors( + logprob_token_ids=ids, + logprobs=-ids.float(), + selected_token_ranks=torch.zeros(rows, dtype=torch.int64), + ) + + +def test_stashes_of_different_widths_join(): + # A read that asked for 10 label ids and a generation that asked for 2 + # logprobs converged in different steps, so their stashes differ in width. + wide, narrow = _stash(2, 11, 100), _stash(3, 3, 0) + + out = _concat_logprob_stashes([narrow, wide], [0, 3]) + + assert out.logprob_token_ids.shape == (5, 11) + assert out.logprobs.shape == (5, 11) + assert out.cu_num_generated_tokens == [0, 3] + # The narrow rows keep their values and pad with id 0 at -inf. + assert torch.equal(out.logprob_token_ids[:3, :3], narrow.logprob_token_ids) + assert not out.logprob_token_ids[:3, 3:].any() + assert torch.isneginf(out.logprobs[:3, 3:]).all() + assert torch.equal(out.logprobs[3:], wide.logprobs) + + +@pytest.mark.parametrize("width", [4, CL]) +@pytest.mark.parametrize("steps", [1, 2]) +def test_read_emits_at_convergence_while_generation_waits_for_commit(width, steps): + states = _states() + for slot in (2, 0): + states.add_request(slot) + states.is_encoder_phase[slot] = False + states.max_steps[slot] = steps + states.set_read_only(2) + + for step in range(steps): + sampled, counts = _denoise_once(states, [2, 0], width=width) + assert counts.tolist() == ([width, 0] if step == steps - 1 else [0, 0]) + + assert torch.equal(sampled[0], states.argmax_canvas[2, :width].int()) + assert not states.is_encoder_phase[2] + assert states.is_encoder_phase[0] + assert not states.self_conditioning_embeds[2].any() + + sampled, counts = _denoise_once(states, [0], width=width) + assert counts.tolist() == [width] + assert torch.equal(sampled[0], states.argmax_canvas[0, :width].int()) + + +def test_batch_allowed_needs_one_shared_set(): + states = _states() + states.constrained[0] = (1, 2, 3) + states.constrained[1] = (4, 5) + + # Mixed sets, or a slot with no set, fall back to per-row masking. + assert states.batch_allowed([0, 1]) is None + assert states.batch_allowed([0, 2]) is None + assert states.batch_allowed([]) is None + + shared = states.batch_allowed([0]) + assert shared.tolist() == [1, 2, 3] + assert shared.dtype == torch.int64 + assert states.batch_allowed([0, 0]) is shared + assert states.allowed_tensor((1, 2, 3)) is shared + + +def test_mask_rows_to_allowed_masks_only_constrained_rows(): + logits = torch.randn(5, VOCAB, device="cuda") + before = logits.clone() + first = torch.tensor([3, 7], device="cuda") + third = torch.tensor([0], device="cuda") + + out = _mask_rows_to_allowed(logits, [0, 2, 3], [2, 1, 2], [first, None, third]) + + assert out is not logits + assert torch.equal(logits, before) + # Masked columns hold the finite sentinel, so entropy stays finite. + kept = out > _MASKED_LOGIT + assert torch.isfinite(out).all() + assert kept[0:2].sum(dim=1).tolist() == [2, 2] + assert kept[0:2][:, first].all() + assert torch.equal(out[0:2][:, first], before[0:2][:, first]) + assert torch.equal(out[2], before[2]) + assert kept[3:5].sum(dim=1).tolist() == [1, 1] + assert torch.equal(out[3:5][:, third], before[3:5][:, third]) + # The masked row's softmax is the distribution renormalized over the set. + torch.testing.assert_close( + out[0].softmax(dim=-1)[first], before[0, first].softmax(dim=-1) + ) + + +def test_mask_rows_to_allowed_is_a_no_op_without_constrained_rows(): + logits = torch.randn(3, VOCAB, device="cuda") + assert _mask_rows_to_allowed(logits, [0, 1], [1, 2], [None, None]) is logits diff --git a/tests/v1/sample/test_diffusion_sampler_stats.py b/tests/v1/sample/test_diffusion_sampler_stats.py new file mode 100644 index 0000000000..589c28c0a6 --- /dev/null +++ b/tests/v1/sample/test_diffusion_sampler_stats.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The fused row statistics of the DiffusionGemma sampler against the PyTorch +reference: argmax, entropy and softmax must match. A zero temperature must +sample the argmax and a positive one must sample from the distribution.""" + +import math + +import pytest +import torch + +from vllm.model_executor.models.diffusion_gemma_sampler import ( + sample_row_stats, + sample_row_stats_reference, +) + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), reason="the fused kernel needs a GPU" +) + + +@pytest.mark.parametrize("vocab", [5000, 9001]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_stats_match_reference(vocab: int, dtype: torch.dtype): + torch.manual_seed(0) + num_decode, cl = 9, 4 + logits = (torch.randn(num_decode * cl, vocab, device="cuda") * 3).to(dtype) + temps = torch.tensor([0.0, 0.7, 2.0, 1.0, 0.0, 0.3, 1.5, 1.0, 0.0], device="cuda") + argmax, sample, entropy, probs = sample_row_stats( + logits, temps, cl, seed=1234, probs_dtype=torch.bfloat16 + ) + ref_argmax, _, ref_entropy, ref_probs = sample_row_stats_reference( + logits, temps, cl, probs_dtype=torch.bfloat16 + ) + assert torch.equal(argmax, ref_argmax) + torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3) + torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2) + assert probs.dtype == torch.bfloat16 + assert probs.shape == logits.shape + # A padded (all-zero) row is uniform: max entropy, argmax 0. + zero = torch.zeros(1, vocab, device="cuda") + a0, _, e0, _ = sample_row_stats(zero, temps[:1] + 1.0, 1, 1, None) + assert a0.item() == 0 + assert abs(e0.item() - torch.log(torch.tensor(float(vocab))).item()) < 1e-3 + + +def test_zero_temperature_is_greedy_and_probs_optional(): + torch.manual_seed(0) + logits = torch.randn(12, 4097, device="cuda") + temps = torch.zeros(12, device="cuda") + argmax, sample, entropy, probs = sample_row_stats(logits, temps, 1, 7, None) + assert torch.equal(sample, argmax) + assert probs is None + assert torch.equal(argmax, logits.argmax(dim=-1)) + # Greedy entropy is that of the reference's clamped temperature. + _, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None) + torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3) + + +def test_positive_temperature_samples_from_the_distribution(): + torch.manual_seed(0) + rows, vocab = 2000, 4099 + # A peaked row samples its argmax almost always and a flat row rarely. + peaked = torch.zeros(rows, vocab, device="cuda") + peaked[:, 17] = 16.0 # p(17) = e^16 / (e^16 + 4098) > 0.999 + flat = torch.zeros(rows, vocab, device="cuda") + temps = torch.ones(rows, device="cuda") + _, s_peaked, _, _ = sample_row_stats(peaked, temps, 1, 99, None) + _, s_flat, _, _ = sample_row_stats(flat, temps, 1, 99, None) + assert (s_peaked == 17).float().mean().item() > 0.99 + assert s_flat.unique().numel() > rows // 2 + # The same seed repeats the draws and another seed changes them. + _, s_again, _, _ = sample_row_stats(flat, temps, 1, 99, None) + _, s_other, _, _ = sample_row_stats(flat, temps, 1, 100, None) + assert torch.equal(s_flat, s_again) + assert not torch.equal(s_flat, s_other) + + +def test_empty_batch(): + logits = torch.empty(0, 128, device="cuda") + temps = torch.empty(0, device="cuda") + argmax, sample, entropy, probs = sample_row_stats( + logits, temps, 1, 1, torch.bfloat16 + ) + assert argmax.numel() == sample.numel() == entropy.numel() == 0 + assert probs is not None and probs.shape == (0, 128) + + +def test_masked_logits_keep_a_finite_entropy(): + """top_k/top_p leave -inf logits. Masked columns gave NaN entropy, and a + row with one live column has zero entropy.""" + logits = torch.full((4, 5000), float("-inf"), device="cuda") + logits[:, 3] = 0.0 + logits[2, 9] = 0.0 # two live columns: entropy log 2 + temps = torch.tensor([1.0, 0.0, 1.0, 0.7], device="cuda") + argmax, sample, entropy, probs = sample_row_stats( + logits, temps, 1, 5, torch.float32 + ) + assert torch.isfinite(entropy).all() + assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6 + assert abs(entropy[2].item() - math.log(2.0)) < 1e-4 + assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda")) + assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0 + _, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None) + assert torch.isfinite(ref_entropy).all() + + +def test_leading_masked_block_keeps_a_finite_entropy(): + """top_k leaves most of the vocabulary -inf, so whole leading blocks can be + masked before the first live logit. The running max was -inf through + them, and -inf - -inf poisoned the row with NaN.""" + logits = torch.full((3, 9001), float("-inf"), device="cuda") + logits[:, 8500] = 0.0 # the only live column, past the first block + logits[1, 8600] = 0.0 # two live columns: entropy log 2 + logits[2, 4] = -1.0 # a live column in the first block as well + temps = torch.tensor([1.0, 1.0, 0.7], device="cuda") + argmax, sample, entropy, probs = sample_row_stats( + logits, temps, 1, 5, torch.float32 + ) + _, _, ref_entropy, ref_probs = sample_row_stats_reference( + logits, temps, 1, torch.float32 + ) + assert torch.isfinite(entropy).all() and torch.isfinite(probs).all() + assert entropy[0].item() < 1e-6 + assert abs(entropy[1].item() - math.log(2.0)) < 1e-4 + torch.testing.assert_close(entropy, ref_entropy, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(probs, ref_probs, atol=1e-5, rtol=1e-4) + assert argmax.tolist() == [8500, 8500, 8500] + assert sample[0].item() == 8500 diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index 8ff16e5599..c0b9792456 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -25,7 +25,7 @@ def _flashinfer_topk_topp_supported() -> bool: Mirrors the gate in `TopKTopPSampler.__init__`: CUDA + flashinfer importable + GPU compute capability supported by the FlashInfer - backend. + backend + more than 16 SMs. """ if not current_platform.is_cuda(): return False @@ -38,7 +38,10 @@ def _flashinfer_topk_topp_supported() -> bool: capability = current_platform.get_device_capability() if capability is None: return False - return FlashInferBackend.supports_compute_capability(capability) + return FlashInferBackend.supports_compute_capability(capability) and ( + current_platform.num_compute_units(torch.accelerator.current_device_index()) + > 16 + ) FLASHINFER_TOPK_TOPP_SUPPORTED = _flashinfer_topk_topp_supported() @@ -164,79 +167,6 @@ def test_topk_impl_equivalence(): assert torch.allclose(result1, result2) -@pytest.mark.skip( - reason="FlashInfer top-k/top-p renorm comparison fails; " - "needs investigation of tolerance threshold or " - "interface differences between Python and FlashInfer implementations" -) -def test_flashinfer_sampler(): - """This test verifies that the FlashInfer top-k and top-p sampling - implementation produces the same results as the Python implementation. - - NOTE: FlashInfer did not directly expose an interface for fused top-k and - top-p prob renorm (it did provide fused sampling but we cannot compare - sampling results due to randomness), so we will compare the probability - renormed consequently by top-k and then top-p of FlashInfer implementation. - """ - try: - from flashinfer.sampling import top_k_renorm_probs, top_p_renorm_probs - - is_flashinfer_available = True - except ImportError: - is_flashinfer_available = False - - FLASHINFER_ENABLED = current_platform.is_cuda() and is_flashinfer_available - - if not FLASHINFER_ENABLED: - pytest.skip("FlashInfer not installed or not available on this platform.") - - torch.set_default_device(DEVICE_TYPE) - generator = Generator(device=DEVICE_TYPE).manual_seed(42) - - # Generate random logits - logits = torch.rand((BATCH_SIZE, VOCAB_SIZE), generator=generator) - - # Generate various top-k and top-p values - k_values = torch.randint(1, 1000, (BATCH_SIZE,), generator=generator) - p_values = ( - torch.rand((BATCH_SIZE,), generator=generator) * 0.5 + 0.5 - ) # range in [0.5, 1.0] - - # Sometimes disable top-k (k=vocab_size) - k_values.masked_fill_( - torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool), - VOCAB_SIZE, - ) - - # Sometimes disable top-p (p=1.0) - p_values.masked_fill_( - torch.randint(0, 2, (BATCH_SIZE,), generator=generator, dtype=torch.bool), 1.0 - ) - - python_logits = apply_top_k_top_p_pytorch( - logits=logits.clone(), - k=k_values, - p=p_values, - ) - python_probs = torch.softmax(python_logits, dim=-1) - - # FlashInfer only exposed renorm interfaces for probs so convert first - flashinfer_probs = torch.softmax(logits.clone(), dim=-1) - flashinfer_probs = top_k_renorm_probs( - probs=flashinfer_probs, - top_k=k_values, - ) - flashinfer_probs = top_p_renorm_probs( - probs=flashinfer_probs, - top_p=p_values, - ) - - # Compare the results - assert torch.allclose(python_probs, flashinfer_probs, atol=2e-2), ( - "FlashInfer and Python sampling implementations do not match!" - ) - - @pytest.mark.skipif( not FLASHINFER_TOPK_TOPP_SUPPORTED, reason="Requires CUDA and flashinfer", diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index ed07718de6..3817a15e3e 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler( def test_hybrid_store_uses_resolved_group_block_sizes() -> None: """Replicated groups must not be scaled by the DCP world size. - ``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP - size and every other spec 1. Reconstructing a group's geometry as + The spec's ``dcp_sharded`` flag controls its effective block size. + Reconstructing a group's geometry as ``spec.block_size * cp_world_size`` over-scales the replicated groups, so the eager store scan believes far fewer of their blocks are ready and silently offloads only a fraction of them. diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 116f4a7c36..392b64d992 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -1,15 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from contextlib import nullcontext from types import SimpleNamespace import numpy as np import pytest from vllm.config.compilation import CUDAGraphMode -from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder from vllm.v1.worker.gpu.async_utils import StepTimingSample -from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, @@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe ) -def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch): - class Backend: - @classmethod - def supports_device_cpu_query_lens_mismatch(cls): - return True +@pytest.mark.parametrize( + "target_support,target_bound,device_offsets,additional,error", + [ + pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded" + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + None, + True, + None, + "TargetBackend allows none", + id="unsupported", + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + 7, + True, + None, + "up to 8, but TargetBackend allows at most 7", + id="too-narrow", + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + 8, + False, + None, + "trims verification requests", + id="host-query-lens", + ), + pytest.param( + AttentionCGSupport.ALWAYS, + None, + True, + (AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"), + "EncoderBackend allows none", + id="additional-group", + ), + ], +) +def test_manager_checks_target_varlen_cudagraph_bound( + monkeypatch, target_support, target_bound, device_offsets, additional, error +): + """Target builders must replay varlen decode graphs of the verification + width; draft-only groups are not checked.""" - class Builder: - def __init__(self, support): - self.support = support + def group(backend_name, layer_name, support, bound=None, mismatch=True): + class Builder(AttentionMetadataBuilder): + _cudagraph_support = support - def get_cudagraph_support(self, *_args): - return self.support + def __init__(self): + pass - def group(layer_name, support): - builder = Builder(support) + def build(self, common_prefix_len, common_attn_metadata, fast_build=False): + raise NotImplementedError + + if bound is not None: + + @classmethod + def get_varlen_cudagraph_max_query_len(cls, *_args): + return bound + + backend = type( + backend_name, + (), + {"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)}, + ) + builder = Builder() return SimpleNamespace( layer_names=[layer_name], - backend=Backend, + backend=backend, kv_cache_spec=None, get_metadata_builder=lambda _index: builder, ) groups = [ [ - group("target", AttentionCGSupport.ALWAYS), - group("draft", AttentionCGSupport.UNIFORM_BATCH), + group( + "TargetBackend", "target", target_support, target_bound, device_offsets + ), + group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH), ] ] - runner_support = AttentionCGSupportInfo( - AttentionCGSupport.UNIFORM_BATCH, "DraftBackend" - ) created = object() monkeypatch.setattr( adaptive_module, @@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc lambda *_args, **_kwargs: created, ) - manager = maybe_create_adaptive_verification_manager( - enable_adaptive_verification=True, - attn_groups=groups, - attn_cg_support=runner_support, - req_states=object(), - query_start_loc=object(), - num_bonus_tokens=1, - max_total_logits=1, - vllm_config=None, - target_layer_names={"target"}, - ) - - assert manager is created - assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH + with pytest.raises(ValueError, match=error) if error else nullcontext(): + manager = maybe_create_adaptive_verification_manager( + enable_adaptive_verification=True, + attn_groups=groups, + req_states=SimpleNamespace(num_speculative_steps=7), + query_start_loc=object(), + num_bonus_tokens=1, + max_total_logits=1, + vllm_config=None, + target_layer_names={"target"}, + additional_attn_cg_support=additional, + ) + assert manager is created def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): diff --git a/tests/v1/spec_decode/test_backup_token_async_spec.py b/tests/v1/spec_decode/test_backup_token_async_spec.py deleted file mode 100644 index 9340503ea4..0000000000 --- a/tests/v1/spec_decode/test_backup_token_async_spec.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression tests for the backup token fix in prepare_next_token_ids_padded. - -Fixes #38098: with async scheduling, seq_lens_cpu is inflated by unaccepted -draft token placeholders, causing get_token_id() to return -1. -""" - -from __future__ import annotations - -import numpy as np -import pytest -import torch - - -class _FakeRequest: - def __init__(self, prompt_tokens: list[int], output_tokens: list[int]): - self.num_prompt_tokens = len(prompt_tokens) - self._prompt = prompt_tokens - self._output = output_tokens - - @property - def num_tokens(self) -> int: - return self.num_prompt_tokens + len(self._output) - - def get_token_id(self, idx: int) -> int: - if idx < self.num_prompt_tokens: - return self._prompt[idx] - out_idx = idx - self.num_prompt_tokens - if out_idx < len(self._output): - return self._output[out_idx] - return -1 # out of range - - -class _FakeInputBatch: - def __init__( - self, - req_ids: list[str], - num_tokens_no_spec: list[int], - vocab_size: int = 32000, - ): - self.req_ids = req_ids - self.num_reqs = len(req_ids) - self.vocab_size = vocab_size - self.num_tokens_no_spec = np.array(num_tokens_no_spec, dtype=np.int64) - - -def _make_requests( - req_ids: list[str], - prompt_lens: list[int], - output_lens: list[int], -) -> dict[str, _FakeRequest]: - requests = {} - for rid, plen, olen in zip(req_ids, prompt_lens, output_lens): - requests[rid] = _FakeRequest(list(range(plen)), list(range(1000, 1000 + olen))) - return requests - - -def _backup_buggy( - seq_lens_cpu: torch.Tensor, - requests: dict[str, _FakeRequest], - batch: _FakeInputBatch, -) -> list[int]: - """Old logic: uses seq_lens_cpu directly (may be inflated).""" - n = batch.num_reqs - return [ - requests[batch.req_ids[i]].get_token_id(int(seq_lens_cpu[i])) for i in range(n) - ] - - -def _backup_fixed( - requests: dict[str, _FakeRequest], - batch: _FakeInputBatch, -) -> list[int]: - """New logic: uses num_tokens_no_spec - 1 (last committed token).""" - n = batch.num_reqs - idx = (batch.num_tokens_no_spec[:n] - 1).tolist() - return [requests[batch.req_ids[i]].get_token_id(int(idx[i])) for i in range(n)] - - -class TestBackupTokenAsyncSpec: - def test_no_inflation_fixed_returns_last_token(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - # idx = 5-1 = 4 → output[1] = 1001 - assert _backup_fixed(requests, batch) == [1001, 1001] - - def test_inflation_buggy_returns_placeholder(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - # inflated by 3 spec tokens → idx 8 is out of range - seq_lens = torch.tensor([8, 8], dtype=torch.int64) - assert _backup_buggy(seq_lens, requests, batch) == [-1, -1] - - def test_inflation_fixed_returns_correct_token(self): - req_ids = ["r0", "r1"] - requests = _make_requests(req_ids, [3, 3], [2, 2]) - batch = _FakeInputBatch(req_ids, [5, 5]) - assert _backup_fixed(requests, batch) == [1001, 1001] - - def test_mixed_inflation_per_request(self): - req_ids = ["r0", "r1", "r2"] - requests = { - "r0": _FakeRequest([0, 1], [1000, 1001, 1002]), - "r1": _FakeRequest([0, 1, 2, 3], [2000]), - "r2": _FakeRequest([0], [3000, 3001, 3002, 3003]), - } - batch = _FakeInputBatch(req_ids, [5, 5, 5]) - seq_lens = torch.tensor([7, 9, 5], dtype=torch.int64) - - assert _backup_buggy(seq_lens, requests, batch) == [-1, -1, -1] - assert _backup_fixed(requests, batch) == [1002, 2000, 3003] - - def test_prefill_only_request(self): - """No output tokens yet — backup should be the last prompt token.""" - req_ids = ["r0"] - requests = {"r0": _FakeRequest([10, 20, 30], [])} - batch = _FakeInputBatch(req_ids, [3]) - # idx = 3-1 = 2 → prompt[2] = 30 - assert _backup_fixed(requests, batch) == [30] - - @pytest.mark.parametrize("num_spec_tokens", [1, 2, 3, 4, 5]) - def test_various_spec_token_counts(self, num_spec_tokens: int): - req_ids = ["r0"] - requests = {"r0": _FakeRequest([0, 1, 2], list(range(1000, 1005)))} - batch = _FakeInputBatch(req_ids, [8]) - # idx = 8-1 = 7 → output[4] = 1004 - assert _backup_fixed(requests, batch) == [1004] - - def test_buggy_code_was_always_off_by_one(self): - """The original code used seq_len as index, which is always one past - the end of output_token_ids even without async inflation.""" - req_ids = ["r0"] - requests = {"r0": _FakeRequest([0, 1, 2], [1000, 1001])} - batch = _FakeInputBatch(req_ids, [5]) - - # no inflation: seq_len == num_tokens == 5 → idx 5 is out of range - seq_lens = torch.tensor([5], dtype=torch.int64) - assert _backup_buggy(seq_lens, requests, batch) == [-1] - assert _backup_fixed(requests, batch) == [1001] - - # with inflation: still -1, fixed still correct - seq_lens_inf = torch.tensor([8], dtype=torch.int64) - assert _backup_buggy(seq_lens_inf, requests, batch) == [-1] - assert _backup_fixed(requests, batch) == [1001] diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index 72379fa7d7..c508768b39 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -6,6 +6,9 @@ from types import SimpleNamespace import pytest import torch +from vllm.model_executor.models.qwen3_dflash import ( + _add_global_draft_layer_exclusions, +) from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator @@ -85,6 +88,24 @@ def test_grouped_conv_triton_matches_reference( ) +def test_draft_quant_exclusions_include_global_layer_indices(): + quant_config = SimpleNamespace( + exclude_modules=[ + "layers.0.mlp_conv*", + "*layers.4.self_attn.q_proj", + "layers.88.already_global", + "lilicorr.layers.0.mlp.0", + ] + ) + + _add_global_draft_layer_exclusions(quant_config, 88, 5) + + assert "layers.88.mlp_conv*" in quant_config.exclude_modules + assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules + assert quant_config.exclude_modules.count("layers.88.already_global") == 1 + assert "lilicorr.layers.88.mlp.0" not in quant_config.exclude_modules + + def test_selector_edges_match_sequential_reference(): torch.manual_seed(1) batch, steps, top_k, rank = 2, 4, 3, 5 @@ -169,10 +190,13 @@ def test_selector_asks_for_fp32_proposal_logits(): @pytest.mark.skip_global_cleanup -def test_dflash2_model_decoder_layer_cls(monkeypatch): +@pytest.mark.parametrize("variant", ["dflash2", "lilicorr", "lilicorr_plain"]) +def test_candidate_model_decoder_layer_cls(monkeypatch, variant): from types import SimpleNamespace from vllm.config import set_current_vllm_config + from vllm.model_executor.models.lilicorr import LiLiCorr + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3DecoderLayer from vllm.model_executor.models.qwen3_dflash2 import ( DFlash2Qwen3DecoderLayer, DFlash2Qwen3Model, @@ -249,8 +273,17 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch): dflash_config={ "selector_rank": 4, "selector_top_k": 3, - "conv_kernel_size": 3, - "conv_group_size": 2, + "conv_kernel_size": 0 if variant == "lilicorr_plain" else 3, + "conv_group_size": 0 if variant == "lilicorr_plain" else 2, + "block_size": 5, + "lilicorr_candidate_topk": 4, + "lilicorr_hidden_size": 8, + "lilicorr_num_layers": 2, + "lilicorr_num_heads": 2, + "lilicorr_mlp_ratio": 2.0, + "lilicorr_factor_dim": 4, + "lilicorr_vector_eps": 1e-6, + "lilicorr_logit_scale": 3.0, "use_aux_hidden_state": False, }, ) @@ -277,8 +310,150 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch): # 3. Instantiate the model under meta device to avoid parameter allocation issues with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): - model = DFlash2Qwen3Model(vllm_config=vllm_config) + model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr + model = model_cls(vllm_config=vllm_config) # 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass) assert len(model.layers) == 2 - assert isinstance(model.layers[0], DFlash2Qwen3DecoderLayer) + expected = ( + DFlashQwen3DecoderLayer + if variant == "lilicorr_plain" + else DFlash2Qwen3DecoderLayer + ) + assert type(model.layers[0]) is expected + + +def test_conv_projections_use_draft_quant_config(monkeypatch): + from torch import nn + + from vllm.distributed import parallel_state + from vllm.model_executor.layers.quantization import modelopt + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3DecoderLayer + from vllm.model_executor.models.qwen3_dflash2 import DFlash2Qwen3DecoderLayer + from vllm.model_executor.models.utils import AutoWeightsLoader + + monkeypatch.setattr( + parallel_state, "_TP", SimpleNamespace(rank_in_group=0, world_size=1) + ) + monkeypatch.setattr( + DFlashQwen3DecoderLayer, + "__init__", + lambda self, *a, **kw: nn.Module.__init__(self), + ) + # Exercise the real quantized parameter allocation without selecting a GPU kernel. + monkeypatch.setattr( + modelopt, + "select_linear_kernel", + lambda *a, **kw: SimpleNamespace(input_quant_key=lambda: None), + ) + quant_config = modelopt.ModelOptNvFp4Config( + quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True + ) + layer = DFlash2Qwen3DecoderLayer( + SimpleNamespace( + speculative_config=SimpleNamespace(num_speculative_tokens=7), + model_config=SimpleNamespace(dtype=torch.bfloat16), + ), + config=SimpleNamespace( + hidden_size=16, + dflash_config={"conv_kernel_size": 2, "conv_group_size": 2}, + ), + layer_idx=0, + prefix="model.layers.0", + quant_config=quant_config, + ) + for name in ("attention_conv", "mlp_conv"): + module = getattr(layer, name) + projection = module.kernel_projection + assert projection.weight.dtype == torch.uint8 + assert projection.weight.shape == (32, 8) + weight = torch.ones_like(projection.weight) + AutoWeightsLoader(module).load_weights([("kernel_projection.weight", weight)]) + torch.testing.assert_close(projection.weight, weight) + assert module.base_kernel.dtype == torch.bfloat16 + + +def test_context_kv_uses_quantized_projection_fallback(monkeypatch): + from torch import nn + + from vllm.model_executor.models import qwen3_dflash + + class Projection(nn.Module): + def __init__(self, packed_weight): + super().__init__() + self.register_buffer("packed_weight", packed_weight) + self.quant_method = object() + self.calls = 0 + + def forward(self, hidden_states): + self.calls += 1 + return torch.nn.functional.linear(hidden_states, self.packed_weight), None + + monkeypatch.setattr( + qwen3_dflash.ops, + "rms_norm", + lambda output, hidden_states, weight, eps: output.copy_(hidden_states), + ) + context_states = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + projections = [ + Projection( + torch.tensor( + [ + [0.0, 0.0], + [0.0, 0.0], + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + [1.0, -1.0], + ] + ) + ), + Projection( + torch.tensor( + [ + [0.0, 0.0], + [0.0, 0.0], + [2.0, 0.0], + [0.0, 2.0], + [-1.0, 0.0], + [0.0, -1.0], + ] + ) + ), + ] + model = SimpleNamespace( + hidden_norm=SimpleNamespace(weight=nn.Parameter(torch.ones(2))), + _rms_norm_eps=1e-6, + ) + layers_attn = [ + SimpleNamespace( + qkv_proj=projection, + q_size=2, + k_norm=SimpleNamespace(weight=nn.Parameter(torch.ones(2))), + ) + for projection in projections + ] + qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers( + model, layers_attn, has_bias=False + ) + assert model._fused_kv_weight is None + assert all(not hasattr(projection, "weight") for projection in projections) + + actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv( + model, + context_states, + num_ctx=2, + num_layers=2, + num_kv_heads=1, + head_dim=2, + ) + + expected_k = torch.tensor( + [[[1.0, 2.0], [3.0, 4.0]], [[2.0, 4.0], [6.0, 8.0]]] + ).unsqueeze(2) + expected_v = torch.tensor( + [[[3.0, -1.0], [7.0, -1.0]], [[-1.0, -2.0], [-3.0, -4.0]]] + ).unsqueeze(2) + torch.testing.assert_close(actual_k, expected_k) + torch.testing.assert_close(actual_v, expected_v) + assert [projection.calls for projection in projections] == [1, 1] diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 16d6d8e516..1d004c3667 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from types import SimpleNamespace +from unittest.mock import patch import numpy as np import pytest @@ -9,7 +10,7 @@ import torch from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( - prepare_dflash_inputs, + DFlashSpeculator, ) pytestmark = pytest.mark.skipif( @@ -24,6 +25,7 @@ def _run_prepare( cp_rank: int = 0, cp_size: int = 1, cp_interleave: int = 1, + draft_dcp_size: int | None = None, ): device = torch.device("cuda") max_num_reqs = 4 @@ -40,7 +42,9 @@ def _run_prepare( ) input_batch = SimpleNamespace( num_reqs=1, + num_tokens=4, num_scheduled_tokens=np.array([4], dtype=np.int32), + seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]), positions=torch.tensor(target_positions, dtype=torch.int64, device=device), query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device), idx_mapping=torch.tensor([2], dtype=torch.int32, device=device), @@ -74,36 +78,65 @@ def _run_prepare( next_prefill_tokens = torch.zeros_like(last_sampled) block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device) - prepare_dflash_inputs( - input_buffers, - query_slot_mapping, - context_positions, - context_slot_mapping, - sample_indices, - sample_pos, - sample_idx_mapping, - temperature, - seeds, - input_batch, - torch.tensor([1], dtype=torch.int32, device=device), - torch.tensor([2], dtype=torch.int32, device=device), - last_sampled, - next_prefill_tokens, - input_temperature, - input_seeds, - block_table, - 4, - cp_rank, - cp_size, - cp_interleave, - 123, - num_speculative_steps, - num_speculative_steps, - max_num_reqs, - max_num_tokens, - 128, + class InputsPrepared(Exception): + pass + + draft = SimpleNamespace( + input_buffers=input_buffers, + context_positions=context_positions, + _context_slot_mappings=context_slot_mapping.unsqueeze(0), + sample_indices=sample_indices, + sample_pos=sample_pos, + sample_idx_mapping=sample_idx_mapping, + temperature=temperature, + seeds=seeds, + hidden_states=torch.zeros(4, 1, device=device), + prepare_context_anchor=lambda *args: None, + query_cudagraph_manager=None, + dp_size=1, + dp_rank=0, + pcp_manager=None, + draft_kv_cache_group_id=0, + draft_kv_cache_group_ids=[0], + _layer_group_idx=None, + block_tables=SimpleNamespace( + slot_mappings=query_slot_mapping.unsqueeze(0), + input_block_tables=[block_table], + kernel_block_sizes=[4], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=cp_interleave, + ), + dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size, + parallel_drafting_token_id=123, + num_query_per_req=num_speculative_steps, + num_speculative_steps=num_speculative_steps, + max_num_reqs=max_num_reqs, + max_num_tokens=max_num_tokens, + max_model_len=128, sample_from_anchor=True, ) + with ( + patch( + "vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp", + side_effect=InputsPrepared, + ), + pytest.raises(InputsPrepared), + ): + DFlashSpeculator.propose( + draft, + input_batch, + {}, + {}, + draft.hidden_states, + None, + torch.tensor([1], dtype=torch.int32, device=device), + torch.tensor([2], dtype=torch.int32, device=device), + last_sampled, + next_prefill_tokens, + input_temperature, + input_seeds, + ) torch.accelerator.synchronize() return SimpleNamespace( input_buffers=input_buffers, @@ -118,12 +151,17 @@ def _run_prepare( ) -def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): +@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)]) +def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size): # Positions 10/11 use physical block 7. Rejected positions 12/13 would use # block 8, but must be PAD context rather than contaminating draft KV. out = _run_prepare( target_positions=[10, 11, 12, 13], block_table_values=[0, 0, 7, 8, 9, 10, 11, 12], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=2, + draft_dcp_size=1, ) assert out.context_positions[:4].tolist() == [10, 11, 0, 0] diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py index 0cc78f3371..2c4b951be7 100644 --- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py +++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py @@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace from unittest.mock import patch import numpy as np +import pytest import torch from vllm.config.compilation import CUDAGraphMode @@ -53,6 +54,7 @@ def _make_fake_speculator( arange_np=np.arange(max_num_reqs + 1, dtype=np.int32), draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool), block_tables=fake_block_tables, + dcp_size=1, input_buffers=fake_input_buffers, attn_groups=[], kv_cache_config=SimpleNamespace(kv_cache_groups=[]), @@ -152,23 +154,24 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len(): assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32)) -def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): +@pytest.mark.parametrize("draft_dcp_size", [1, 2]) +def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size): fake = _make_fake_speculator() + fake.dcp_size = draft_dcp_size fake.block_tables.cp_size = 2 fake.block_tables.cp_rank = 1 fake.block_tables.cp_interleave = 4 fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16]) def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave): + assert draft_dcp_size > 1 assert seq_lens is fake.input_buffers.seq_lens assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4) out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32)) out[num_reqs:].zero_() return out - with patch.object( - base_speculator, "maybe_prepare_dcp_local_seq_lens", fake_prepare - ): + with patch.object(base_speculator, "prepare_dcp_local_seq_lens", fake_prepare): captured = _run_build( fake, num_reqs=3, @@ -178,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): ) local = captured["dcp_local_seq_lens"] + if draft_dcp_size == 1: + assert local is None + return assert isinstance(local, torch.Tensor) assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr() assert local.tolist() == [1, 4, 8, 0] diff --git a/tests/v1/spec_decode/test_extract_hidden_states.py b/tests/v1/spec_decode/test_extract_hidden_states.py index e330d95880..7eb8eca023 100644 --- a/tests/v1/spec_decode/test_extract_hidden_states.py +++ b/tests/v1/spec_decode/test_extract_hidden_states.py @@ -7,7 +7,7 @@ from unittest import mock import numpy as np import pytest import torch -from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig, PretrainedConfig +from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig, PreTrainedConfig from tests.v1.attention.utils import ( BatchSpec, @@ -336,7 +336,7 @@ def test_propose_different_layer_counts(num_hidden_layers): # --------------------------------------------------------------------------- -class _DummyVLMConfig(PretrainedConfig): +class _DummyVLMConfig(PreTrainedConfig): """Minimal composite config that mimics VLMs like Kimi-K2.5 or LLaVA. The text model's parameters (hidden_size, num_attention_heads, …) live @@ -345,11 +345,11 @@ class _DummyVLMConfig(PretrainedConfig): model_type = "test_vlm" - def __init__(self, text_config: PretrainedConfig, **kwargs): + def __init__(self, text_config: PreTrainedConfig, **kwargs): self.text_config = text_config super().__init__(architectures=["LlamaForCausalLM"], **kwargs) - def get_text_config(self, decoder: bool = False) -> PretrainedConfig: + def get_text_config(self, decoder: bool = False) -> PreTrainedConfig: del decoder return self.text_config @@ -403,7 +403,7 @@ def test_extract_hidden_states_config_preserves_vlm_text_config(): eagle_aux_hidden_state_layer_ids=[1, 2], ) - # The fix: text_config is still a PretrainedConfig, not a dict. + # The fix: text_config is still a PreTrainedConfig, not a dict. assert isinstance(extract_config.text_config, LlamaConfig) extracted = get_hf_text_config(extract_config) @@ -473,7 +473,7 @@ def test_extract_hidden_states_speculative_config_vlm(): def test_extract_hidden_states_config_invalid_text_config(): """A nested text_config missing required attrs must still be rejected.""" - broken_text_config = PretrainedConfig(hidden_size=128) + broken_text_config = PreTrainedConfig(hidden_size=128) vlm_config = _DummyVLMConfig(text_config=broken_text_config) extract_config = ExtractHiddenStatesConfig( diff --git a/tests/v1/spec_decode/test_lilicorr.py b/tests/v1/spec_decode/test_lilicorr.py new file mode 100644 index 0000000000..14bad45f6b --- /dev/null +++ b/tests/v1/spec_decode/test_lilicorr.py @@ -0,0 +1,777 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from typing import Any + +import pytest +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.config import ( + CompilationConfig, + DeviceConfig, + LoadConfig, + VllmConfig, + set_current_vllm_config, +) +from vllm.distributed import parallel_state +from vllm.model_executor.layers import logits_processor +from vllm.model_executor.layers.linear import LinearBase +from vllm.model_executor.models.lilicorr import LiLiCorrHead +from vllm.transformers_utils.configs.eagle import EAGLEConfig +from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler +from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator + + +@pytest.fixture(autouse=True) +def head_tp_metadata(monkeypatch): + # Single-rank unit tests only need TP metadata. + monkeypatch.setattr( + parallel_state, "_TP", SimpleNamespace(rank_in_group=0, world_size=1) + ) + + +def _config(**overrides): + config = dict( + candidate_topk=4, + hidden_size=8, + num_layers=2, + num_heads=2, + mlp_ratio=2.0, + factor_dim=4, + vector_eps=1e-6, + logit_scale=3.0, + ) + config.update(overrides) + return {f"lilicorr_{key}": value for key, value in config.items()} + + +def _reference_scores(head, embeddings, log_probs, hidden, anchor, valid): + """Unfused exported head: MHA attention, concatenated factors, separate edges.""" + batch, slots, top_k = log_probs.shape + rank = torch.arange(top_k, device=log_probs.device) / max(top_k - 1, 1) + top1 = (torch.arange(top_k, device=log_probs.device) == 0).float() + features = torch.stack( + ( + log_probs, + log_probs.exp(), + log_probs - log_probs.amax(-1, keepdim=True), + rank.expand_as(log_probs), + top1.expand_as(log_probs), + ), + -1, + ) + x = head.token_proj(embeddings) + head.pass_hidden_proj(hidden).unsqueeze(-2) + features = head.feature_norm(features.to(x.dtype)) + x = x + head.feature_mlp.down_proj(F.silu(head.feature_mlp.up_proj(features)[0]))[0] + x = x + head.slot_embedding[:, 0, :slots] + head.rank_embedding[:, 0] + x = x.flatten(1, 2) + positions = torch.arange(slots, device=x.device).repeat_interleave(top_k) + relative = positions[:, None] - positions[None, :] + bias = head.relative_slot_bias[:, relative + head.block_size - 1] + bias = bias + (relative == 0) * head.same_slot_bias[:, None, None] + bias = bias.repeat(batch, 1, 1) + for layer in head.layers: + attention = nn.MultiheadAttention( + head.hidden_size, + head.num_heads, + batch_first=True, + dtype=x.dtype, + device=x.device, + ) + attention.load_state_dict(layer.attn.state_dict()) + normalized = layer.attn_norm(x) + x = ( + x + + attention( + normalized, normalized, normalized, attn_mask=bias, need_weights=False + )[0] + ) + x = x + layer.mlp.down_proj(F.silu(layer.mlp.up_proj(layer.mlp_norm(x))[0]))[0] + x = head.output_norm(x).view(batch, slots, top_k, -1) + a = head.anchor_norm(head.context_proj(anchor) * valid[:, None]) + expanded = a[:, None, None].expand_as(x) + factors = F.silu(head.factor_input_proj(torch.cat((x, expanded, x * expanded), -1))) + incoming = F.normalize(head.in_head(factors), dim=-1, eps=head.vector_eps) + outgoing = F.normalize(head.out_head(factors), dim=-1, eps=head.vector_eps) + a = F.normalize(head.anchor_out_head(a), dim=-1, eps=head.vector_eps) + start = (a[:, None] * incoming[:, 0]).sum(-1) + pairs = outgoing[:, :-1] @ incoming[:, 1:].transpose(-1, -2) + return ( + head.logit_scale + * torch.cat((start[:, None, None].expand(-1, 1, top_k, -1), pairs), 1).float() + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +@pytest.mark.parametrize( + "head_width,slots,dtype", + [ + pytest.param(8, 3, torch.float32, id="projected-multi-slot"), + pytest.param(16, 1, torch.float32, id="identity-shortened-single-slot"), + pytest.param(8, 2, torch.bfloat16, id="bf16-shortened-multi-slot"), + ], +) +def test_lilicorr_matches_exported_head(head_width, slots, dtype): + device = torch.device("cuda") + torch.manual_seed(7) + with ( + torch.device(device), + set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cuda"), + compilation_config=CompilationConfig(mode=0), + ) + ), + ): + head = LiLiCorrHead( + model_hidden_size=16, + block_size=4, + rms_norm_eps=1e-6, + config=_config(hidden_size=head_width), + ).to(dtype) + # vLLM linear parameters are initialized by the checkpoint loader. + with torch.no_grad(), torch.device(device): + for module in head.modules(): + if isinstance(module, LinearBase): + for parameter in module.parameters(): + parameter.weight_loader( + parameter, torch.randn_like(parameter) * 0.1 + ) + for layer in head.layers: + layer.attn.in_proj_weight.normal_(std=0.1) + layer.attn.in_proj_bias.normal_(std=0.1) + head.relative_slot_bias.normal_(std=0.1) + head.same_slot_bias.normal_(std=0.1) + head.slot_embedding.normal_(std=0.1) + head.rank_embedding.normal_(std=0.1) + head.materialize_inference_buffers(device, dtype) + inputs = ( + torch.randn(2, slots, 4, 16, dtype=dtype), + torch.randn(2, slots, 23).log_softmax(-1).topk(4).values, + torch.randn(2, slots, 16, dtype=dtype), + torch.randn(2, 16, dtype=dtype), + torch.tensor([True, False]), + ) + actual = head(*inputs) + expected = _reference_scores(head, *inputs) + tolerance = 0.04 if dtype == torch.bfloat16 else 2e-6 + torch.testing.assert_close(actual, expected, atol=tolerance, rtol=tolerance) + + +def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch): + from vllm.model_executor.models.lilicorr import LiLiCorr + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3Model + + def init_backbone(self, **kwargs): + nn.Module.__init__(self) + self.quant_config = None + + monkeypatch.setattr(DFlashQwen3Model, "__init__", init_backbone) + spec = SimpleNamespace( + num_speculative_tokens=1, + draft_model_config=SimpleNamespace( + hf_config=SimpleNamespace( + hidden_size=16, + rms_norm_eps=1e-6, + dflash_config=_config() | {"block_size": 4}, + ) + ), + ) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + for slots in (1, 3): + spec.num_speculative_tokens = slots + head = LiLiCorr( + vllm_config=SimpleNamespace(speculative_config=spec) + ).lilicorr + assert head.slot_embedding.shape == (1, 1, 3, 1, 8) + assert head.relative_slot_bias.shape == (2, 7) + spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False + with pytest.raises(ValueError, match="lilicorr_enabled"): + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True + spec.num_speculative_tokens = 4 + with pytest.raises(ValueError, match="trained block_size"): + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + + +@pytest.mark.parametrize("tp_size", [1, 2]) +def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_size): + """Candidate features must include mass outside top-k and on other TP ranks.""" + + class Gather: + def __init__(self, calls): + self.calls = calls + self.index = 0 + + def __call__(self, value, dim=-1): + expected_input, result = self.calls[self.index] + assert dim == -1 + torch.testing.assert_close(value, expected_input) + self.index += 1 + return result + + shards = [ + torch.tensor([[1.0, 3.0, 2.0, 100.0], [4.0, 1.0, 2.0, 100.0]]), + torch.tensor([[6.0, 5.0, 0.0, 100.0], [3.0, 6.0, 5.0, 100.0]]), + ][:tp_size] + for rank in range(tp_size): + fake = SimpleNamespace( + scale=1.0, + soft_cap=None, + _apply_head=lambda *args, rank=rank: shards[rank].clone(), + ) + lm_head = SimpleNamespace( + tp_size=tp_size, + shard_indices=SimpleNamespace( + num_org_vocab_padding=1, org_vocab_start_index=rank * 3 + ), + ) + valid_shards = [shard[:, :3] for shard in shards] + local_topk = [shard.topk(2) for shard in valid_shards] + gather_calls = [ + ( + valid_shards[rank].logsumexp(-1, keepdim=True), + torch.cat( + [shard.logsumexp(-1, keepdim=True) for shard in valid_shards], + -1, + ), + ), + ( + local_topk[rank].values, + torch.cat([topk.values for topk in local_topk], -1), + ), + ( + local_topk[rank].indices + rank * 3, + torch.cat( + [ + topk.indices + shard_rank * 3 + for shard_rank, topk in enumerate(local_topk) + ], + -1, + ), + ), + ] + all_gather = Gather(gather_calls) + + monkeypatch.setattr( + logits_processor, + "tensor_model_parallel_all_gather", + all_gather, + ) + monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1)) + ids, values = logits_processor.LogitsProcessor.get_top_k_tokens( + fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True + ) + expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2) + torch.testing.assert_close(ids, expected.indices) + torch.testing.assert_close(values, expected.values) + assert all_gather.index == (3 if tp_size > 1 else 0) + + +def test_context_anchor_is_last_committed_normalized_feature(): + hidden = torch.arange(1, 25, dtype=torch.float32).view(6, 4) + norm = nn.RMSNorm(4) + spec = SimpleNamespace( + hidden_states=hidden, + model=SimpleNamespace(model=SimpleNamespace(hidden_norm=norm)), + anchor_hidden=torch.full((4, 4), 123.0), + anchor_valid=torch.ones(4, dtype=torch.bool), + ) + batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) + torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]])) + assert spec.anchor_valid.tolist() == [True, True, False, False] + assert not spec.anchor_hidden[2:].any() + # A smaller reordered batch must not retain an old request's anchor. + batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) + torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1])) + assert not spec.anchor_hidden[1:].any() + + +@pytest.mark.parametrize("lilicorr", [False, True]) +def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lilicorr): + """Both heads share the walk, including a padded request and adaptive scores.""" + from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator + from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator + + key = "lilicorr_candidate_topk" if lilicorr else "selector_top_k" + + def init_base(self, config, device): + self.draft_model_config = SimpleNamespace( + hf_config=SimpleNamespace(hidden_size=4, dflash_config={key: 4}) + ) + self.max_num_reqs = 3 + self.num_speculative_steps = 2 + self.num_query_per_req = 3 + self.dtype = torch.float32 + + monkeypatch.setattr(DFlashSpeculator, "__init__", init_base) + cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator + spec = cls(None, torch.device("cpu")) + hidden = torch.arange(36).float().view(9, 4) + spec._run_model = lambda *args: hidden + spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8]) + spec.sample_pos = torch.arange(6) + spec.sample_idx_mapping = torch.tensor([1, 1, 0, 0, -1, -1]) + spec.sample_col = torch.tensor([0, 1, 0, 1, 0, 1]) + spec.temperature = torch.ones(3) + spec.seeds = torch.arange(3) + spec.draft_tokens = torch.empty(3, 2, dtype=torch.long) + spec.draft_logits = None + spec.use_fp64_gumbel = False + spec.enable_adaptive_verification = True + spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) + candidates = torch.arange(24).view(6, 4) + log_probs = candidates.float().log_softmax(-1) + scores = torch.randn(3, 2, 4, 4) + scoring_inputs: list[torch.Tensor] = [] + sampling_inputs: list[Any] = [] + adaptive_inputs: list[torch.Tensor] = [] + + def score(*args): + scoring_inputs.extend(args) + return scores + + def sample(*args): + sampling_inputs.extend(args) + spec.candidate_sampler.scores.copy_(scores[:, :, 0]) + + spec.model = SimpleNamespace( + compute_candidates=lambda h: (candidates, log_probs), + model=SimpleNamespace(lilicorr=score, candidate_selector=score), + ) + spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4) + if lilicorr: + spec.anchor_hidden.copy_(torch.arange(12).view(3, 4)) + spec.anchor_valid[:2] = True + spec.candidate_sampler.sample = sample + spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args) + spec._generate_draft(3, 9, None, None, None) + + expected_ids = candidates.view(3, 2, 4) + expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids + torch.testing.assert_close(scoring_inputs[0], expected_first) + torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4)) + torch.testing.assert_close( + scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4) + ) + if lilicorr: + torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden) + torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid) + else: + torch.testing.assert_close(scoring_inputs[3], torch.tensor([0, 3, 6])) + torch.testing.assert_close(sampling_inputs[0], expected_ids) + assert sampling_inputs[1] is scores + assert sampling_inputs[4] is spec.sample_idx_mapping + torch.testing.assert_close(adaptive_inputs[0], scores[:, :, 0].flatten(0, 1)) + torch.testing.assert_close(adaptive_inputs[1], spec.sample_idx_mapping) + torch.testing.assert_close(adaptive_inputs[2], spec.sample_col) + + +def test_lilicorr_config_preserves_architecture(): + from transformers import Qwen3Config + + config = Qwen3Config(architectures=["LiLiCorrDraftModel"]) + wrapped = EAGLEConfig(config, method="dflash") + assert wrapped.architectures == ["LiLiCorrDraftModel"] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("probabilistic", [False, True]) +def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic): + """Check the shared sampler's actual q, stale support, and graph padding.""" + device = torch.device("cuda") + batch, steps, top_k, vocab = 3, 3, 4, 32 + sampler = CandidateSampler(batch, steps, top_k, device) + candidates = ( + torch.arange(steps * top_k, device=device) + .view(1, steps, top_k) + .repeat(batch, 1, 1) + ) + scores = torch.randn(batch, steps, top_k, top_k, device=device) + tokens = torch.full((batch, steps), -1, dtype=torch.long, device=device) + cache = ( + torch.full((batch, steps, vocab), -float("inf"), device=device) + if probabilistic + else None + ) + positions = torch.arange(batch * steps * 2, device=device)[::2] + mappings = torch.tensor( + [2] * steps + [0] * steps + [-1] * steps, device=device + ).repeat_interleave(2)[::2] + temperature = torch.tensor([0.0, 1.0, 0.8], device=device) + seeds = torch.tensor([11, 22, 33], device=device) + + def run(): + sampler.sample( + candidates, + scores, + batch, + positions, + mappings, + temperature, + seeds, + tokens, + cache, + False, + ) + + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + old_candidates = candidates.clone() + candidates.add_(12) + graph.replay() + for row, state in ((0, 2), (1, 0)): + previous = 0 + for step in range(steps): + realized = scores[row, step, previous] + torch.testing.assert_close(sampler.scores[row, step], realized) + index = (candidates[row, step] == tokens[row, step]).nonzero().item() + if not probabilistic or temperature[state] == 0: + assert index == realized.argmax().item() + if cache is not None: + torch.testing.assert_close( + cache[state, step, candidates[row, step]], realized + ) + assert cache[state, step, old_candidates[row, step]].isneginf().all() + previous = index + assert (tokens[2] == -1).all() + if cache is not None: + assert cache[1].isneginf().all() + + +@pytest.mark.parametrize( + ("convolution", "mismatch", "quantized"), + [ + pytest.param(False, None, False, id="valid-plain"), + pytest.param(True, None, False, id="valid-convolution"), + pytest.param(False, "missing_head", False, id="missing-head"), + pytest.param(False, "extra_head", False, id="extra-head"), + pytest.param(True, "missing_conv", False, id="missing-convolution"), + pytest.param(False, "extra_conv", False, id="unexpected-convolution"), + pytest.param(False, None, True, id="optional-w4a16-input-scale"), + pytest.param(False, "missing_weight", True, id="missing-quantized-weight"), + pytest.param(False, "missing_bias", True, id="missing-quantized-bias"), + pytest.param(False, "missing_weight_scale", True, id="missing-block-scale"), + pytest.param(False, "missing_weight_scale_2", True, id="missing-global-scale"), + ], +) +def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads( + monkeypatch, convolution, mismatch, quantized +): + from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM + + wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) + nn.Module.__init__(wrapper) + wrapper.model = LiLiCorr.__new__(LiLiCorr) + nn.Module.__init__(wrapper.model) + wrapper.has_own_lm_head = False + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + wrapper.model.lilicorr = LiLiCorrHead( + model_hidden_size=16, block_size=4, rms_norm_eps=1e-6, config=_config() + ) + layer = nn.Module() + if convolution: + layer.attention_conv = nn.Linear(2, 2, bias=False) + wrapper.model.layers = nn.ModuleList([layer]) + if quantized: + from vllm.config.quantization import QuantSpec + from vllm.model_executor.layers.quantization.modelopt import ( + CkptCtx, + ModelOptLinearMethod, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kNvfp4Static, + ) + + # Match the W4A16 NVFP4 checkpoint contract without GPU kernel setup. + projection = wrapper.model.lilicorr.pass_hidden_proj + projection.quant_method = ModelOptLinearMethod( + QuantSpec(weight=kNvfp4Static, activation=None), CkptCtx(group_size=16) + ) + projection.weight = nn.Parameter( + torch.zeros(8, 8, dtype=torch.uint8), requires_grad=False + ) + for name, value in ( + ("weight_scale", torch.ones(8, 1).to(torch.float8_e4m3fn)), + ("weight_scale_2", torch.ones(1)), + ("input_scale", torch.full((1,), torch.nan)), + ): + projection.register_parameter( + name, nn.Parameter(value, requires_grad=False) + ) + weights = dict(wrapper.model.state_dict()) + if quantized: + del weights["lilicorr.pass_hidden_proj.input_scale"] + if mismatch == "missing_head": + del weights["lilicorr.slot_embedding"] + elif mismatch in ( + "missing_weight", + "missing_bias", + "missing_weight_scale", + "missing_weight_scale_2", + ): + del weights[f"lilicorr.pass_hidden_proj.{mismatch.removeprefix('missing_')}"] + elif mismatch == "extra_head": + weights["lilicorr.untrained.weight"] = torch.zeros(1) + elif mismatch == "missing_conv": + del weights["layers.0.attention_conv.weight"] + elif mismatch == "extra_conv": + weights["layers.0.mlp_conv.untrained"] = torch.zeros(1) + + # Keep production loading and mapping, bypass only backbone GPU KV setup. + wrapper.model.use_aux_hidden_state = True + wrapper.model.has_separate_mask_embedding = False + monkeypatch.setattr(wrapper, "_read_mask_embedding", lambda: None) + monkeypatch.setattr( + wrapper.model, "_build_fused_kv_buffers", lambda: None, raising=False + ) + supplied = [("model." + name, value) for name, value in weights.items()] + if mismatch: + message = ( + "no module or parameter" + if mismatch.startswith("extra") + else "coverage mismatch" + ) + with pytest.raises(ValueError, match=message): + wrapper.load_weights(supplied) + else: + wrapper.load_weights(supplied) + assert wrapper.model.lilicorr._attn_bias is not None + + +@pytest.fixture +def owned_head_model(monkeypatch): + from vllm.model_executor.layers.quantization import modelopt + from vllm.model_executor.models import qwen3_dflash + from vllm.model_executor.models.lilicorr import LiLiCorrForCausalLM + + quant_config = modelopt.ModelOptNvFp4Config( + quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True + ) + monkeypatch.setattr( + modelopt, + "select_linear_kernel", + lambda *a, **kw: SimpleNamespace(input_quant_key=lambda: None), + ) + monkeypatch.setattr(qwen3_dflash, "get_draft_quant_config", lambda _: quant_config) + + class Backbone(nn.Module): + def __init__(self, **kwargs): + super().__init__() + self.quant_config = quant_config + self.use_aux_hidden_state = True + self.has_separate_mask_embedding = False + self.lilicorr = LiLiCorrHead( + model_hidden_size=16, block_size=4, rms_norm_eps=1e-6, config=_config() + ) + + def _build_fused_kv_buffers(self): + pass + + monkeypatch.setattr(LiLiCorrForCausalLM, "model_cls", Backbone) + monkeypatch.setattr(LiLiCorrForCausalLM, "_read_mask_embedding", lambda _: None) + + def build(owned=True): + config = SimpleNamespace( + vocab_size=64, + draft_vocab_size=64, + hidden_size=16, + has_own_lm_head=owned, + num_hidden_layers=0, + dflash_config={}, + ) + vllm_config = SimpleNamespace( + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(hf_config=config), + draft_load_config=None, + attention_backend=None, + kv_cache_dtype=None, + ), + model_config=SimpleNamespace( + get_total_num_hidden_layers=lambda: 0, + get_vocab_size=lambda: 64, + ), + attention_config=SimpleNamespace(), + cache_config=SimpleNamespace(), + load_config=LoadConfig(), + ) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + model = LiLiCorrForCausalLM(vllm_config=vllm_config) + return model, vllm_config + + return build + + +@pytest.mark.parametrize("missing", [None, "weight", "weight_scale", "weight_scale_2"]) +def test_owned_nvfp4_lm_head_loads_required_checkpoint_tensors( + owned_head_model, missing +): + model, _ = owned_head_model() + assert model.lm_head.weight.dtype == torch.uint8 + assert model.lm_head.weight.shape == (64, 8) + weights = { + name: torch.ones_like(value) for name, value in model.state_dict().items() + } + del weights["lm_head.input_scale"] # Deprecated and unused for W4A16. + if missing: + del weights[f"lm_head.{missing}"] + with pytest.raises(ValueError, match=f"lm_head.{missing}"): + model.load_weights(weights.items()) + else: + model.load_weights(weights.items()) + assert model.has_own_lm_head + for name in ("weight", "weight_scale", "weight_scale_2"): + torch.testing.assert_close( + getattr(model.lm_head, name), weights[f"lm_head.{name}"] + ) + + +@pytest.mark.parametrize("owned", [False, True]) +def test_selected_lm_head_survives_dflash_and_lilicorr_sharing( + monkeypatch, owned_head_model, owned +): + from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, + ) + from vllm.v1.worker.gpu.spec_decode.dflash import utils as dflash_utils + from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils + + model, config = owned_head_model(owned) + own_head = model.lm_head + target_head = owned_head_model(owned)[0].lm_head + # Identical packed bytes do not imply identical quantized heads. + with torch.no_grad(): + own_head.weight.fill_(1) + target_head.weight.fill_(1) + if owned: + own_head.weight_scale_2.fill_(1) + target_head.weight_scale_2.fill_(2) + embedding = VocabParallelEmbedding(64, 16) + target = SimpleNamespace( + model=SimpleNamespace(embed_tokens=embedding), lm_head=target_head + ) + monkeypatch.setattr( + eagle_utils, "get_pp_group", lambda: SimpleNamespace(world_size=1) + ) + monkeypatch.setattr( + dflash_utils, "replace", lambda obj, **kw: SimpleNamespace(**(vars(obj) | kw)) + ) + monkeypatch.setattr(dflash_utils, "get_pp_safe_draft_load_config", lambda c: c) + monkeypatch.setattr(dflash_utils, "get_model", lambda **kw: model) + if owned: + + def no_weight_comparison(*args): + raise AssertionError("Explicit ownership must bypass raw-weight comparison") + + monkeypatch.setattr(dflash_utils, "_should_share", no_weight_comparison) + spec = LiLiCorrSpeculator.__new__(LiLiCorrSpeculator) + spec.vllm_config = config + assert spec.load_draft_model(target, set()) is model + assert model.lm_head is (own_head if owned else target_head) + assert spec.target_embeddings is embedding + if owned: + target.lm_head = None + assert spec.load_draft_model(target, set()).lm_head is own_head + + +def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch): + from vllm.model_executor.layers import linear + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + calls = [] + configured = [] + quant_config = object() + + class PackedMethod(UnquantizedLinearMethod): + def apply(self, layer, x, bias=None): + calls.append(layer.prefix) + # Packed storage has a different name/dtype from the activations. + assert x.dtype == torch.float32 + return F.linear(x, layer.packed_weight.float() * layer.weight_scale, bias) + + def resolve(config, layer, prefix): + assert config is quant_config + configured.append(prefix) + return PackedMethod() + + monkeypatch.setattr(linear, "resolve_quant_method", resolve) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + head = LiLiCorrHead( + model_hidden_size=16, + block_size=4, + rms_norm_eps=1e-6, + config=_config(hidden_size=8), + quant_config=quant_config, + prefix="model.lilicorr", + ) + with torch.no_grad(): + for module in head.modules(): + if isinstance(module, LinearBase): + if module.quant_config is None: + module.weight.normal_(std=0.1) + module.bias.zero_() + continue + module.register_parameter( + "packed_weight", + nn.Parameter( + torch.randint(-4, 5, module.weight.shape, dtype=torch.int8), + requires_grad=False, + ), + ) + module.register_parameter( + "weight_scale", + nn.Parameter( + torch.tensor(0.05), + requires_grad=False, + ), + ) + del module.weight + module.bias.zero_() + for layer in head.layers: + layer.attn.in_proj_weight.normal_(std=0.1) + layer.attn.in_proj_bias.zero_() + head.materialize_inference_buffers(torch.device("cpu"), torch.float32) + result = head( + torch.randn(2, 3, 4, 16), + torch.randn(2, 3, 4).log_softmax(-1), + torch.randn(2, 3, 16), + torch.randn(2, 16), + torch.tensor([True, False]), + ) + assert result.shape == (2, 3, 4, 4) + assert torch.isfinite(result).all() + for name in ("factor_input_proj", "out_head", "in_head"): + assert getattr(head, name).quant_config is None + assert f"model.lilicorr.{name}" not in configured + assert set(calls) == set(configured) + assert len(configured) == len(set(configured)) + for name, module in head.named_modules(): + if isinstance(module, LinearBase): + assert module.prefix == f"model.lilicorr.{name}" diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index 8bc791e963..e3803509e3 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -5,6 +5,7 @@ from unittest import mock import pytest import torch +import torch.nn as nn from tests.v1.attention.utils import ( BatchSpec, @@ -31,6 +32,31 @@ mimo_7b_dir = "XiaomiMiMo/MiMo-7B-Base" DEVICE_TYPE = current_platform.device_type +def test_glm_mtp_defers_lm_head(default_vllm_config): + from vllm.model_executor.models import deepseek_mtp + from vllm.models.glm5next.common import mtp + + config = mock.MagicMock( + hidden_size=16, + rms_norm_eps=1e-5, + index_topk=8, + index_kpool=4, + ) + vllm_config = mock.MagicMock() + vllm_config.speculative_config.draft_model_config.hf_config = config + vllm_config.scheduler_config.max_num_batched_tokens = 4 + + with ( + mock.patch.object(mtp, "Glm5NextDecoderLayer", return_value=nn.Identity()), + mock.patch.object(deepseek_mtp, "ParallelLMHead") as parallel_lm_head, + mock.patch.object(mtp.current_platform, "device_type", "cpu"), + ): + layer = mtp.Glm5NextMultiTokenPredictorLayer(vllm_config, "model.layers.1") + + parallel_lm_head.assert_not_called() + assert layer.shared_head.head is None + + def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer: """Create an MTP proposer with unified model configuration.""" model_config = ModelConfig( diff --git a/tests/v1/spec_decode/test_speculators_correctness.py b/tests/v1/spec_decode/test_speculators_correctness.py index 4cc66d5cd3..7d7ea53f71 100644 --- a/tests/v1/spec_decode/test_speculators_correctness.py +++ b/tests/v1/spec_decode/test_speculators_correctness.py @@ -37,7 +37,7 @@ DFLASH_CONFIG = SpeculatorTestConfig( acceptance_len_rtol=0.15, expected_per_pos_acceptance_rates=(0.795, 0.611, 0.429, 0.282), per_pos_rtol=0.15, - quantization="fp8", + quantization="fp8_per_tensor", ) PEAGLE_CONFIG = SpeculatorTestConfig( diff --git a/tests/v1/spec_decode/test_vocab_mapping.py b/tests/v1/spec_decode/test_vocab_mapping.py index b8773c5a55..25c99ad049 100644 --- a/tests/v1/spec_decode/test_vocab_mapping.py +++ b/tests/v1/spec_decode/test_vocab_mapping.py @@ -2,9 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +import torch from transformers import AutoTokenizer -from vllm.v1.spec_decode.vocab_mapping import _detect_space_prefix +import vllm.utils.gpu_sync_debug as gsd +from vllm.utils.gpu_sync_debug import with_gpu_sync_check +from vllm.v1.spec_decode.vocab_mapping import VocabMapping, _detect_space_prefix @pytest.mark.parametrize( @@ -46,3 +49,79 @@ def test_detect_space_prefix_empty_encode(): result = _detect_space_prefix(EmptyTokenizer()) assert result == ("Ġ", "▁") + + +class _FakeTokenizer: + """Minimal tokenizer exposing only what ``VocabMapping`` needs.""" + + def __init__(self, vocab, unk_token_id=None, eos_token_id=None): + self._vocab = dict(vocab) + self.unk_token_id = unk_token_id + self.eos_token_id = eos_token_id + + def get_vocab(self): + return dict(self._vocab) + + def encode(self, text, **kwargs): + raise RuntimeError("no encode") + + def convert_ids_to_tokens(self, ids): + raise RuntimeError("no convert") + + +@pytest.fixture(params=["cpu", "cuda"]) +def vocab_mapping(request): + device = request.param + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("requires CUDA") + + # Intersection: {a, b}. x/y/z are target-only, u/v are draft-only + target = _FakeTokenizer( + vocab={"a": 0, "b": 1, "x": 2, "y": 3, "z": 4}, unk_token_id=0 + ) + draft = _FakeTokenizer(vocab={"a": 0, "b": 1, "u": 2, "v": 3}, unk_token_id=2) + return VocabMapping(target, draft, 5, 4, device) + + +@pytest.mark.parametrize( + "target_ids,expected", + [ + ([0, 1, 2, 3, 4], [0, 1, 2, 2, 2]), # mixed + ([0, 1], [0, 1]), # all present + ([2, 3, 4], [2, 2, 2]), # all missing + ], +) +def test_map_target_to_draft_ids(monkeypatch, vocab_mapping, target_ids, expected): + device = vocab_mapping.device + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error" if device == "cuda" else None) + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + + ids = torch.tensor(target_ids, dtype=torch.int32, device=device) + out = with_gpu_sync_check(lambda: vocab_mapping.map_target_to_draft_ids(ids))() + + assert out.tolist() == expected + assert out.dtype == torch.int32, f"expected torch.int32, got {out.dtype}" + assert out.device.type == device, f"expected {device} device, got {out.device}" + assert (out != -1).all() + + +@pytest.mark.parametrize( + "draft_ids,expected", + [ + ([0, 1, 2, 3], [0, 1, 0, 0]), # mixed + ([0, 1], [0, 1]), # all present + ([2, 3], [0, 0]), # all missing + ], +) +def test_map_draft_to_target_ids(monkeypatch, vocab_mapping, draft_ids, expected): + device = vocab_mapping.device + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error" if device == "cuda" else None) + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + + ids = torch.tensor(draft_ids, dtype=torch.int32, device=device) + out = with_gpu_sync_check(lambda: vocab_mapping.map_draft_to_target_ids(ids))() + + assert out.tolist() == expected + assert out.dtype == torch.int32, f"expected torch.int32, got {out.dtype}" + assert out.device.type == device, f"expected {device} device, got {out.device}" + assert (out != -1).all() diff --git a/tests/v1/structured_output/test_backend_xgrammar_lark.py b/tests/v1/structured_output/test_backend_xgrammar_lark.py new file mode 100644 index 0000000000..6b13f270d9 --- /dev/null +++ b/tests/v1/structured_output/test_backend_xgrammar_lark.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lark grammars in the xgrammar backend. + +`structured_outputs.grammar` accepts either EBNF or Lark. The xgrammar backend +parses Lark natively, both when validating a request and when compiling it, so +the request's grammar is never rewritten and frontends that skip Python +request validation get the same behavior. +""" + +import pytest +from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoiceEnum +from transformers import AutoTokenizer + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.tokenizers.mistral import MistralTokenizer +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import ( + XgrammarBackend, + validate_xgrammar_grammar, +) + +pytestmark = pytest.mark.cpu_test + +TOKENIZER = "openai-community/gpt2" +VOCAB_SIZE = 50257 +MISTRAL_TOKENIZER = "mistralai/Ministral-3-3B-Instruct-2512" + +# Uses Lark features beyond plain rules: a regex terminal, `%import common`, +# and a rule alias. The Codex `apply_patch` tool grammar has the same shape. +LARK_GRAMMAR = """ +start: "id=" NUMBER tag? LF? +tag: " #" /[a-z]+/ -> label +NUMBER: /[0-9]+/ +%import common.LF +""" + +EBNF_GRAMMAR = 'root ::= "id=" [0-9]+' + + +def _validate( + grammar: str, backend: str = "xgrammar", tokenizer: object | None = None +) -> StructuredOutputsParams: + params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) + params._validate_structured_outputs( + _StubModelConfig(), + StructuredOutputsConfig(backend=backend), + tokenizer=tokenizer or object(), + ) + assert params.structured_outputs is not None + return params.structured_outputs + + +class _StubModelConfig: + is_diffusion = False + + +GRAMMARS = [ + pytest.param(LARK_GRAMMAR, id="lark"), + pytest.param(EBNF_GRAMMAR, id="ebnf"), +] + + +@pytest.mark.parametrize("grammar", GRAMMARS) +def test_validation_accepts_grammar_without_rewriting_it(grammar: str): + assert _validate(grammar).grammar == grammar + + +def test_auto_backend_selects_xgrammar_for_lark(): + assert _validate(LARK_GRAMMAR, backend="auto")._backend == "xgrammar" + + +@pytest.mark.parametrize("mode", [ToolChoiceEnum.required, ToolChoiceEnum.none]) +def test_auto_backend_selects_guidance_for_mistral_lark(mode: ToolChoiceEnum): + """The Mistral tool parser generates Lark grammars for llguidance, which + keeps special tokens such as `[TOOL_CALLS]` out of regex matches.""" + tokenizer = MistralTokenizer.from_pretrained(MISTRAL_TOKENIZER) + factory = tokenizer.grammar_factory + tool = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + grammar = factory.get_lark_from_jinja( + template=factory.select_jinja_template(), + mode=mode, + tools=[Tool.from_openai(tool)], + json_schema=None, + parallel_tool_calls=True, + json_only=False, + ) + assert _validate(grammar, "auto", tokenizer)._backend == "guidance" + + +@pytest.mark.parametrize( + "grammar", + [ + pytest.param('rule: "hello"', id="missing-start-rule"), + pytest.param("start: 'hello'", id="single-quoted-literal"), + ], +) +def test_validation_rejects_invalid_lark(grammar: str): + params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) + with pytest.raises(VLLMValidationError, match="Invalid grammar specification"): + validate_xgrammar_grammar(params) + + +@pytest.fixture(scope="module") +def backend() -> XgrammarBackend: + vllm_config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar") + ) + return XgrammarBackend( + vllm_config, + tokenizer=AutoTokenizer.from_pretrained(TOKENIZER), + vocab_size=VOCAB_SIZE, + ) + + +@pytest.mark.parametrize("grammar", GRAMMARS) +@pytest.mark.parametrize( + "text, accepted", + [("id=42", True), ("id=x", False)], +) +def test_compile_grammar_matches_lark_and_ebnf( + backend: XgrammarBackend, grammar: str, text: str, accepted: bool +): + compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, grammar) + token_ids = backend.tokenizer.encode(text, add_special_tokens=False) + assert compiled.accept_tokens("req", token_ids) is accepted + + +def test_compile_grammar_uses_lark_semantics(backend: XgrammarBackend): + compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, LARK_GRAMMAR) + token_ids = backend.tokenizer.encode("id=7 #ok\n", add_special_tokens=False) + assert compiled.accept_tokens("req", token_ids) + assert compiled.is_terminated() or compiled.accept_tokens( + "req", [backend.tokenizer.eos_token_id] + ) diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 17bda09b3e..a1da205fe0 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token) EOS = "<|eos|>" # resolved to tokenizer.eos_token_id JSON_SCHEMA = '{"type": "object"}' BACKENDS = ("xgrammar", "guidance") +REGEX_BACKENDS = (*BACKENDS, "outlines") +DIGITS_REGEX = "[0-9]+" MAX_WAIT_SECONDS = 5 NUM_SPEC_TOKENS = 8 @@ -88,6 +90,7 @@ def _build_harness( reasoning_ended: bool | None = None, enable_in_reasoning: bool = False, reasoning_parser_kwargs: dict | None = None, + structured_outputs: StructuredOutputsParams | None = None, ) -> tuple[StructuredOutputManager, Request]: vllm_config = VllmConfig( model_config=ModelConfig(tokenizer=TOKENIZER), @@ -106,7 +109,8 @@ def _build_harness( manager.reasoner_cls = MockReasoner sampling_params = SamplingParams( - structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA) + structured_outputs=structured_outputs + or StructuredOutputsParams(json=JSON_SCHEMA) ) sampling_params.structured_outputs._backend = backend sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) @@ -412,3 +416,96 @@ def test_initial_constraint_activation( structured_req = request.structured_output_request assert structured_req is not None assert structured_req.reasoner is None + + +@pytest.mark.parametrize("backend", REGEX_BACKENDS) +@pytest.mark.parametrize( + ("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"), + [ + pytest.param(("1", "2"), None, "CCC", False, id="all_valid"), + # EOS is only valid once the regex has matched; rows after EOS are + # unconstrained since the request stops there. + pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"), + pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"), + ], +) +def test_regex_flow( + tokenizer, + backend: str, + raw_drafts: tuple[str, ...], + expected_validated: tuple[str, ...] | None, + expected_row_pattern: str, + terminated: bool, +): + manager, request = _build_harness( + tokenizer, + backend, + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + validated = _to_token_ids( + tokenizer, raw_drafts if expected_validated is None else expected_validated + ) + _run_real_flow( + manager, + request, + raw_drafts=_to_token_ids(tokenizer, raw_drafts), + expected_validated=validated, + expected_row_pattern=expected_row_pattern, + # accept_tokens() latches reasoning_ended once it accepts any token. + expected_reasoning=True if validated else None, + expect_terminated=terminated, + ) + + +@pytest.mark.parametrize("backend", REGEX_BACKENDS) +def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str): + """A draft the grammar rejects during bitmask fill must not unconstrain + the remaining rows or the bonus row.""" + manager, request = _build_harness( + tokenizer, + backend, + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + drafts = _to_token_ids(tokenizer, ("1", "z", "2")) + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + assert bitmask is not None + assert _row_pattern(bitmask) == "CCCC" + assert (bitmask[2] == bitmask[1]).all() + assert (bitmask[3] == bitmask[1]).all() + + +def test_outlines_termination(tokenizer): + """outlines_core never advances on EOS; termination must still be + tracked exactly once per accepted EOS and undone by rollback.""" + manager, request = _build_harness( + tokenizer, + "outlines", + use_reasoner=False, + structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX), + ) + structured_req = request.structured_output_request + assert structured_req is not None + grammar = structured_req.grammar + assert grammar is not None + one, eos = _to_token_ids(tokenizer, ("1", EOS)) + + assert not grammar.accept_tokens(request.request_id, [eos]) + assert grammar.accept_tokens(request.request_id, [one]) + # Polling must not change the result. + assert not any(grammar.is_terminated() for _ in range(3)) + assert grammar.accept_tokens(request.request_id, [eos]) + assert all(grammar.is_terminated() for _ in range(3)) + assert grammar.num_processed_tokens == 2 + + grammar.rollback(2) + assert not grammar.is_terminated() + assert grammar.num_processed_tokens == 0 + assert grammar.validate_tokens([eos]) == [] + assert grammar.accept_tokens(request.request_id, [one, eos, one]) + assert grammar.is_terminated() diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 99481a1293..1bff5ae466 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -8,6 +8,7 @@ never addressed by the logical view. """ from types import SimpleNamespace +from typing import Any import numpy as np import pytest @@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size( class _FakeMetadataBuilder: - def __init__(self, support: AttentionCGSupport): + def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None): self.support = support + self.varlen_bound = varlen_bound def get_cudagraph_support(self, *_args): return self.support + def get_varlen_cudagraph_max_query_len(self, *_args): + return self.varlen_bound + class _TargetBackend: @classmethod @@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): ) +def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds(): + """ALWAYS passes without a bound, other builders need one at least as wide as + the requested length, and NEVER fails whatever bound a builder reports.""" + config: Any = SimpleNamespace() + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + ) + + def group( + backend: Any, + layer_name: str, + support: AttentionCGSupport, + bound: int | None = None, + ): + builder: Any = _FakeMetadataBuilder(support, bound) + attn_group = AttentionGroup(backend, [layer_name], spec, 0) + attn_group.metadata_builders = [builder] + return attn_group + + target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS) + draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8) + never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8) + + unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend + assert unsupported([[target, draft]], config, 8) is None + assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8) + assert ( + unsupported([[target, draft]], config, 9, checked_layer_names={"target"}) + is None + ) + assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None) + + def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch): """Fast prefill applies to the contiguous suffix of KV-sharing layers. diff --git a/tests/v1/worker/test_cp_utils.py b/tests/v1/worker/test_cp_utils.py index b38ae4d063..d477ace479 100644 --- a/tests/v1/worker/test_cp_utils.py +++ b/tests/v1/worker/test_cp_utils.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + import pytest import torch @@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention +def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch): + from vllm.v1.worker import cp_utils + + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=1, + decode_context_parallel_size=4, + cp_kv_cache_interleave_size=16, + ), + speculative_config=object(), + ) + target = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=4, + supports_mtp_with_cp_non_trivial_interleave_size=True, + need_to_return_lse_for_decode=True, + ) + ) + draft = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=1, + supports_mtp_with_cp_non_trivial_interleave_size=False, + need_to_return_lse_for_decode=False, + ) + ) + layers = {"target": target, "draft": draft} + monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers) + cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"}) + with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"): + cp_utils.check_attention_cp_compatibility(config) + + def test_skip_gate_only_for_zero_context(): assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32)) assert not should_skip_dcp_context_attention( diff --git a/tests/v1/worker/test_dspark_utils.py b/tests/v1/worker/test_dspark_utils.py index 94655f192c..1014051d64 100644 --- a/tests/v1/worker/test_dspark_utils.py +++ b/tests/v1/worker/test_dspark_utils.py @@ -2,8 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field +from types import SimpleNamespace +import pytest +import torch + +from vllm.config import ParallelConfig +from vllm.v1.kv_cache_interface import FullAttentionSpec +from vllm.v1.worker.gpu import model_runner +from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator @dataclass @@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically(): assert draft_config.eplb_config.num_redundant_experts == 0 assert not draft_config.enable_elastic_ep assert draft_config.eplb_config is not target_config.eplb_config + + +@pytest.mark.parametrize("pcp_size", [1, 4]) +@pytest.mark.parametrize("dcp_size", [1, 4]) +@pytest.mark.parametrize("use_mla", [False, True]) +def test_draft_context_parallelism_without_changing_target( + monkeypatch, pcp_size, dcp_size, use_mla +): + target_parallel = ParallelConfig( + tensor_parallel_size=4, + prefill_context_parallel_size=pcp_size, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + distributed_executor_backend="mp", + ) + target_config = SimpleNamespace( + parallel_config=target_parallel, + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(use_mla=use_mla) + ), + ) + + class CapturedConfig(Exception): + pass + + def capture_init(self, config, device): + raise CapturedConfig(config) + + monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init) + with pytest.raises(CapturedConfig) as captured: + DFlashSpeculator(target_config, device=None) + draft_parallel = captured.value.args[0].parallel_config + assert draft_parallel.tensor_parallel_size == 4 + assert draft_parallel.prefill_context_parallel_size == 1 + assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1) + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert target_parallel.prefill_context_parallel_size == pcp_size + assert target_parallel.decode_context_parallel_size == dcp_size + assert target_parallel.cp_kv_cache_interleave_size == 16 + + +@pytest.mark.parametrize("configured_dcp", [1, 4]) +def test_attention_uses_draft_dcp_setting_inside_target_process_group( + monkeypatch, configured_dcp +): + import vllm.config as config_module + from vllm.distributed import parallel_state + from vllm.v1.attention.backend import AttentionImplBase + + config = SimpleNamespace( + parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp) + ) + monkeypatch.setattr( + config_module, "get_current_vllm_config_or_none", lambda: config + ) + + monkeypatch.setattr( + parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2) + ) + impl = AttentionImplBase() + assert impl.dcp_world_size == configured_dcp + assert impl.dcp_rank == (0 if configured_dcp == 1 else 2) + + +@pytest.mark.parametrize("draft_dcp_size", [1, 4]) +def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size): + spec = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16 + ) + monkeypatch.setattr( + model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec} + ) + runner = object.__new__(model_runner.GPUModelRunner) + runner.vllm_config = None + runner.dcp_size = 4 + runner.speculator = object.__new__(DFlashSpeculator) + runner.speculator.dcp_size = draft_dcp_size + runner.speculator.draft_attn_layer_names = {"draft"} + + specs = runner.get_kv_cache_spec() + + assert specs["target"].dcp_sharded + assert specs["draft"].dcp_sharded == (draft_dcp_size == 4) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index f73023cee3..36ba9a4b16 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -19,6 +19,7 @@ from vllm.model_executor.models.mistral_large_3_eagle import ( ) from vllm.v1.attention.backends import flash_attn as flash_attn_module from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata +from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module @@ -69,6 +70,50 @@ class _TextOnlyDraftModel(torch.nn.Module): raise AssertionError("embed_input_ids should not be called during loading") +@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL]) +def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode): + def build(common_prefix_len, common_attn_metadata): + return split_decodes_and_prefills( + common_attn_metadata, + decode_threshold=1, + require_uniform=True, + treat_short_extends_as_decodes=False, + ) + + speculator = object.__new__(_TestSpeculator) + speculator.arange_np = torch.arange(5, dtype=torch.int32).numpy() + speculator.max_model_len = speculator.draft_max_seq_len = 32 + speculator.draft_is_prefilling = torch.zeros(4, dtype=torch.bool) + speculator.input_buffers = SimpleNamespace( + query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32), + seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32), + ) + speculator.dcp_size = 1 + speculator.block_tables = SimpleNamespace( + cp_size=1, + input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)], + slot_mappings=torch.tensor([[10, 20, -1, -1]]), + ) + speculator.kv_cache_config = SimpleNamespace(kv_cache_groups=[object()]) + speculator.attn_groups = [ + [ + SimpleNamespace( + get_metadata_builder=lambda _: SimpleNamespace(build=build), + layer_names=["draft"], + ) + ] + ] + num_reqs_padded = 4 if cg_mode == CUDAGraphMode.FULL else 2 + metadata = speculator._build_uniform_attn_metadata( + batch_desc=BatchExecutionDescriptor(cg_mode, 4, num_reqs_padded), + num_reqs=2, + num_query_per_req=1, + seq_lens_cpu_upper_bound=torch.tensor([10, 20], dtype=torch.int32), + step=1, + ) + assert metadata["draft"] == (num_reqs_padded, 0, num_reqs_padded, 0) + + def _mock_base_model_load(monkeypatch): monkeypatch.setattr( base_spec_module, @@ -139,6 +184,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected) parallel_config=SimpleNamespace( data_parallel_size=1, data_parallel_rank=0, + decode_context_parallel_size=1, ), ) diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py index 5c245d1c4d..5c2231a1f2 100644 --- a/tests/v1/worker/test_gpu_block_table.py +++ b/tests/v1/worker/test_gpu_block_table.py @@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups(): @pytest.mark.parametrize("cp_rank", range(4)) def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): - """DCP interleave is expressed in logical-block token coordinates.""" + """Only sharded groups use DCP interleave in logical-block coordinates.""" device = torch.device("cuda") block_tables = BlockTables( - block_sizes=[128], + block_sizes=[128, 128], max_num_reqs=1, max_num_batched_tokens=1024, - max_num_blocks_per_group=[2], + max_num_blocks_per_group=[2, 8], device=device, - kernel_block_sizes=[64], + kernel_block_sizes=[64, 64], + dcp_sharded=[True, False], cp_size=4, cp_rank=cp_rank, cp_interleave=128, ) block_tables.append_block_ids( req_index=0, - new_block_ids=([5, 9],), + new_block_ids=([5, 9], list(range(10, 18))), overwrite=True, ) block_tables.apply_staged_writes() @@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): query_start_loc, positions, num_tokens_padded=1024, - )[0] + ) expected = torch.full((1024,), -1, dtype=torch.int64, device=device) first_start = cp_rank * 128 @@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): expected[second_start : second_start + 128] = torch.arange( 9 * 128, 10 * 128, dtype=torch.int64, device=device ) - assert torch.equal(actual, expected) + assert torch.equal(actual[0], expected) + assert torch.equal(actual[1], positions + 10 * 128) def test_v1_block_table_move_row_clears_vacated_row(): diff --git a/tests/v1/worker/test_gpu_input_batch.py b/tests/v1/worker/test_gpu_input_batch.py index 2fe3567739..f37e824e6c 100644 --- a/tests/v1/worker/test_gpu_input_batch.py +++ b/tests/v1/worker/test_gpu_input_batch.py @@ -372,6 +372,116 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis _compare_objs(input_batch, ref_input_batch) +@pytest.mark.parametrize("device", DEVICES) +def test_condense_clears_stale_allowed_token_ids_mask(device: str): + """condense() must clear the mask row a constrained request is moved out + of. Otherwise a later request that reuses that row without + allowed_token_ids inherits the stale whitelist. + See https://github.com/vllm-project/vllm/issues/43894. + """ + batch_size = 4 + allowed_token_id = 13 + input_batch = InputBatch( + max_num_reqs=batch_size, + max_model_len=1024, + max_num_batched_tokens=1024, + device=torch.device(device), + vocab_size=VOCAB_SIZE, + block_sizes=[1], + kernel_block_sizes=[1], + max_num_blocks_per_req=[1024], + ) + + def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState: + return CachedRequestState( + req_id=f"req_id_{suffix}", + prompt_token_ids=[1, 2, 3], + sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids), + pooling_params=None, + mm_features=[], + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + # Only req_id_2 is constrained, and it lands on the highest row. + assert input_batch.add_request(_make_req(0)) == 0 + assert input_batch.add_request(_make_req(1)) == 1 + assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2 + + mask = input_batch.allowed_token_ids_mask_cpu_tensor + assert mask is not None + # The constrained row masks every token except the single allowed id. + assert not mask[2][allowed_token_id].item() + assert int(mask[2].sum().item()) == VOCAB_SIZE - 1 + + # Free row 0, then condense: req_id_2 slides from row 2 down into row 0. + input_batch.remove_request("req_id_0") + input_batch.condense() + assert input_batch.req_id_to_index["req_id_2"] == 0 + assert int(mask[2].sum().item()) == 0 + + # A new unrestricted request reuses row 2 and must stay unconstrained. + assert input_batch.add_request(_make_req(3)) == 2 + assert "req_id_3" not in input_batch.has_allowed_token_ids + assert int(mask[2].sum().item()) == 0 + + +@pytest.mark.parametrize("device", DEVICES) +def test_swap_states_preserves_allowed_token_ids_mask(device: str): + """swap_states() must exchange the two allowed_token_ids_mask rows. + + Swapping row views via a tuple assignment + (t[i1], t[i2] = t[i2], t[i1]) aliases them, so both rows collapse to the + second row's contents and one request silently loses its token + restriction. This is the sibling of the condense() leak fixed for + https://github.com/vllm-project/vllm/issues/43894. + """ + allowed_token_id = 13 + input_batch = InputBatch( + max_num_reqs=2, + max_model_len=1024, + max_num_batched_tokens=1024, + device=torch.device(device), + vocab_size=VOCAB_SIZE, + block_sizes=[1], + kernel_block_sizes=[1], + max_num_blocks_per_req=[1024], + ) + + def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState: + return CachedRequestState( + req_id=f"req_id_{suffix}", + prompt_token_ids=[1, 2, 3], + sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids), + pooling_params=None, + mm_features=[], + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + # Row 0 is constrained to a single token; row 1 is unconstrained. + assert input_batch.add_request(_make_req(0, [allowed_token_id])) == 0 + assert input_batch.add_request(_make_req(1)) == 1 + + mask = input_batch.allowed_token_ids_mask_cpu_tensor + assert mask is not None + assert int(mask[0].sum().item()) == VOCAB_SIZE - 1 + assert not mask[0][allowed_token_id].item() + assert int(mask[1].sum().item()) == 0 + + input_batch.swap_states(0, 1) + + # The constrained request now sits at row 1 and must keep its mask; the + # unconstrained request now sits at row 0 and must stay open. + assert int(mask[1].sum().item()) == VOCAB_SIZE - 1 + assert not mask[1][allowed_token_id].item() + assert int(mask[0].sum().item()) == 0 + + def _construct_pooling_request(req_id_suffix: int, pooling_params=None): from vllm.pooling_params import PoolingParams @@ -506,3 +616,98 @@ def test_pooling_metadata_token_id_buffers( assert metadata.get_prompt_token_ids_cpu()[0].tolist() == req.prompt_token_ids else: assert metadata.prompt_token_ids_cpu is None + + +PROMPT_EMBEDS_HIDDEN = 8 +PROMPT_EMBEDS_LEN = 4 + + +def _make_input_batch(is_pooling_model: bool = False) -> InputBatch: + return InputBatch( + max_num_reqs=2, + max_model_len=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS, + max_num_batched_tokens=MAX_PROMPT_SIZE + NUM_OUTPUT_TOKENS, + device=torch.device("cpu"), + vocab_size=VOCAB_SIZE, + block_sizes=[16], + kernel_block_sizes=[16], + max_num_blocks_per_req=[64], + is_pooling_model=is_pooling_model, + ) + + +def _make_embeds_request(req_id: str, pooling: bool = False) -> CachedRequestState: + from vllm.pooling_params import PoolingParams + + return CachedRequestState( + req_id=req_id, + prompt_token_ids=None, + prompt_embeds=torch.randn(PROMPT_EMBEDS_LEN, PROMPT_EMBEDS_HIDDEN), + mm_features=[], + sampling_params=None if pooling else SamplingParams(), + pooling_params=PoolingParams(task="classify") if pooling else None, + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + +def _make_token_request(req_id: str) -> CachedRequestState: + return CachedRequestState( + req_id=req_id, + prompt_token_ids=[10, 11, 12, 13], + mm_features=[], + sampling_params=SamplingParams(), + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + +def test_remove_request_releases_prompt_embeds(): + """A finished prompt-embeds request must not keep its tensor referenced by + the persistent batch; an empty batch owns no prompt embeds.""" + input_batch = _make_input_batch() + req = _make_embeds_request("embeds-req") + input_batch.add_request(req) + req_index = input_batch.req_id_to_index[req.req_id] + assert req_index in input_batch.req_prompt_embeds + + input_batch.remove_request(req.req_id) + input_batch.condense() + + assert req_index not in input_batch.req_prompt_embeds + assert not input_batch.req_prompt_embeds + + +def test_reused_slot_does_not_retain_prompt_embeds(): + """Reusing a vacated slot with a token-id request must not keep the previous + occupant's prompt-embeds tensor alive.""" + input_batch = _make_input_batch() + embeds_req = _make_embeds_request("embeds-req") + input_batch.add_request(embeds_req) + slot = input_batch.req_id_to_index[embeds_req.req_id] + + input_batch.remove_request(embeds_req.req_id) + input_batch.condense() + + token_req = _make_token_request("token-req") + input_batch.add_request(token_req) + assert input_batch.req_id_to_index[token_req.req_id] == slot + assert slot not in input_batch.req_prompt_embeds + + +def test_pooling_model_releases_prompt_embeds(): + """The cleanup must run before remove_request's pooling-model early return.""" + input_batch = _make_input_batch(is_pooling_model=True) + req = _make_embeds_request("pool-embeds-req", pooling=True) + input_batch.add_request(req) + req_index = input_batch.req_id_to_index[req.req_id] + assert req_index in input_batch.req_prompt_embeds + + input_batch.remove_request(req.req_id) + input_batch.condense() + + assert req_index not in input_batch.req_prompt_embeds diff --git a/tests/v1/worker/test_gpu_input_batch_v2.py b/tests/v1/worker/test_gpu_input_batch_v2.py index 13a92d2c0f..920277cb81 100644 --- a/tests/v1/worker/test_gpu_input_batch_v2.py +++ b/tests/v1/worker/test_gpu_input_batch_v2.py @@ -81,7 +81,7 @@ def test_make_dummy_padding_controls_moe_routing(monkeypatch, is_padding: bool): assert bool((topk_ids >= 0).all()) is not is_padding -def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): +def test_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): """The batch must view the caller-owned buffer, sliced to padded length. Runtime (and capture) paths all funnel through this helper; attention @@ -115,7 +115,7 @@ def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): return fake_kernel monkeypatch.setattr(cp_utils, "_dcp_local_seq_lens_kernel", FakeKernel()) - batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens( + batch.dcp_local_seq_lens = cp_utils.prepare_dcp_local_seq_lens( buffers.dcp_local_seq_lens, batch.seq_lens, batch.num_reqs, @@ -130,37 +130,10 @@ def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): assert batch.dcp_local_seq_lens.tolist() == [1, 2, 0, 0] -def test_maybe_prepare_dcp_local_seq_lens_clears_stale_metadata(monkeypatch): - """dcp_size == 1 resets the field to None instead of leaving a leftover. - - DCP is toggled per deployment, and batches are recycled; a value from an - earlier DCP batch would otherwise be consumed as if it were current. - """ - buffers = InputBuffers(max_num_reqs=2, max_num_tokens=2, device=torch.device("cpu")) - batch = InputBatch.make_dummy(1, 1, buffers) - batch.dcp_local_seq_lens = buffers.dcp_local_seq_lens[:1] - - def fail_if_called(*args, **kwargs): - raise AssertionError("kernel must not run with dcp_size == 1") - - monkeypatch.setattr(cp_utils, "_dcp_local_seq_lens_kernel", fail_if_called) - batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens( - buffers.dcp_local_seq_lens, - batch.seq_lens, - batch.num_reqs, - 1, - 0, - 1, - num_reqs_padded=batch.num_reqs_after_padding, - ) - - assert batch.dcp_local_seq_lens is None - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="triton kernel needs CUDA") @pytest.mark.parametrize("dcp_size", [2, 4]) @pytest.mark.parametrize("cp_interleave", [1, 16]) -def test_maybe_prepare_dcp_local_seq_lens_matches_reference( +def test_prepare_dcp_local_seq_lens_matches_reference( dcp_size: int, cp_interleave: int ): """Every rank's local lengths must equal the reference torch formula.""" @@ -172,7 +145,7 @@ def test_maybe_prepare_dcp_local_seq_lens_matches_reference( for dcp_rank in range(dcp_size): buffers.dcp_local_seq_lens.fill_(-1) - batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens( + batch.dcp_local_seq_lens = cp_utils.prepare_dcp_local_seq_lens( buffers.dcp_local_seq_lens, batch.seq_lens, batch.num_reqs, diff --git a/tests/v1/worker/test_gpu_kv_connector.py b/tests/v1/worker/test_gpu_kv_connector.py index 03c483dce1..5e56e8a317 100644 --- a/tests/v1/worker/test_gpu_kv_connector.py +++ b/tests/v1/worker/test_gpu_kv_connector.py @@ -96,4 +96,4 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch): connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type] - assert events == ["handle", "bind", "start", "clear"] + assert events == ["handle", "bind", "start", "wait", "clear"] diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 32341ce944..e77d278116 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.platforms import current_platform +from vllm.platforms.interface import Platform from vllm.sampling_params import SamplingParams from vllm.utils.mem_constants import GiB_bytes from vllm.utils.system_utils import update_environment_variables @@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option(): select_common_block_size(48, [backend_a, backend_b]) +def _mock_backend(supported: list, *, exact: bool = False): + """Backend accepting multiples of ``supported``, or only those exact sizes + when ``exact`` (as CPU_MLA does).""" + from vllm.v1.attention.backend import AttentionBackend + + class _MockBackendCls(AttentionBackend): + @staticmethod + def get_name() -> str: + return "MOCK_EXACT" if exact else "MOCK" + + @staticmethod + def get_supported_kernel_block_sizes(): + return list(supported) + + if exact: + + @classmethod + def supports_block_size(cls, block_size: int | None) -> bool: + return block_size is None or block_size in supported + + return _MockBackendCls + + +@pytest.mark.parametrize( + "backends,expected", + [ + # A lone backend keeps the default; MultipleOf(8) makes a + # minimum-based regression return 8 instead of 16. + ([[MultipleOf(8)]], 16), + # Sparse-MLA main backend (32 or 64) beside an indexer supporting only + # 64: the main backend alone picks 32, which the indexer rejects at + # select_common_block_size time. + ([[32, 64], [64]], 64), + ([[32, 64]], 32), + # Neither divides the other, so only the LCM satisfies both. + ([[32], [48]], 96), + ], +) +def test_preferred_block_size_satisfies_every_backend(backends, expected): + classes = [_mock_backend(s) for s in backends] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected + + +def test_preferred_block_size_searches_past_an_exact_size_backend(): + # Extending greedily picks lcm(16, 32) = 32, which the exact backend + # rejects; 96 is accepted by both. + classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96 + + +def test_preferred_block_size_rejects_backends_with_no_common_size(): + classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])] + with pytest.raises(ValueError, match="share no supported KV cache block size"): + Platform._preferred_block_size_for_backends(classes, 16, None) + + def test_set_active_mm_loras_builds_tower_and_connector_mappings(): model = Mock() model.get_mm_lora_token_counts.side_effect = ( diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 83cd1e9d07..308c9fe9ea 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -21,20 +21,6 @@ from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.model_runner import GPUModelRunner -def test_prepare_padding_mask_marks_sequence_parallel_padding(): - runner = GPUModelRunner.__new__(GPUModelRunner) - runner.input_buffers = SimpleNamespace(is_padding=torch.empty(8, dtype=torch.bool)) - - mask = runner._prepare_padding_mask(1, 8) - - assert mask.tolist() == [False, True, True, True, True, True, True, True] - assert mask.data_ptr() == runner.input_buffers.is_padding.data_ptr() - - mask = runner._prepare_padding_mask(0, 8) - - assert mask.all() - - def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): runner = GPUModelRunner.__new__(GPUModelRunner) runner.max_model_len = 262144 diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index 2bd7be2cfa..4612d49d92 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import replace from types import SimpleNamespace -from unittest.mock import MagicMock import numpy as np import pytest @@ -76,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills(): ) +@pytest.mark.parametrize("draft_uses_mla", [False, True]) +@pytest.mark.parametrize("dcp_size", [1, 2, 4]) +def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size): + config = _make_config(CUDAGraphMode.NONE) + config.parallel_config.prefill_context_parallel_size = 4 + config.parallel_config.decode_context_parallel_size = dcp_size + config.model_config.hf_text_config.index_topk = 1 + config.speculative_config = SimpleNamespace( + use_dspark=lambda: True, + draft_model_config=SimpleNamespace(use_mla=draft_uses_mla), + ) + if draft_uses_mla and dcp_size == 2: + with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"): + PCPManager.validate_config(config, supports_mm_inputs=False) + return + PCPManager.validate_config(config, supports_mm_inputs=False) + + def test_replicated_decode_piecewise_graph_padding(monkeypatch): manager = PCPManager( pcp_world_size=2, @@ -170,37 +187,55 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs GPU kernels") @pytest.mark.parametrize( ("cg_mode", "num_reqs", "expected_tokens", "expected_reqs"), [ - (CUDAGraphMode.NONE, 4, None, None), - (CUDAGraphMode.PIECEWISE, None, 8, None), + (CUDAGraphMode.NONE, 4, 2, 2), + (CUDAGraphMode.PIECEWISE, None, 8, 2), (CUDAGraphMode.FULL, 4, 8, 4), ], ) def test_partition_padding_is_derived_from_batch_descriptor( cg_mode, num_reqs, expected_tokens, expected_reqs ): - manager = MagicMock() - input_batch = MagicMock() - manager.partition_batch.return_value = input_batch - batch_desc = BatchExecutionDescriptor( - cg_mode=cg_mode, - num_tokens=8, - num_reqs=num_reqs, + """partition_batch derives graph padding from the batch descriptor: no + padding without graphs, token padding only for PIECEWISE, and request + padding on top for FULL.""" + device = torch.device("cuda:0") + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=device, + max_num_reqs=4, + max_num_tokens=8, + ) + global_batch = _make_global_decode_batch( + [16, 24], InputBuffers(4, 8, device), device ) - result = pcp_manager_module.maybe_partition_pcp_batch( - manager, - input_batch, - batch_desc, + local_batch = manager.partition_batch( + global_batch, + BatchExecutionDescriptor( + cg_mode=cg_mode, + num_tokens=8, + num_reqs=num_reqs, + ), ) - assert result is input_batch - manager.partition_batch.assert_called_once_with( - input_batch, - padded_num_tokens=expected_tokens, - padded_num_reqs=expected_reqs, + assert local_batch.num_tokens_after_padding == expected_tokens + assert local_batch.num_reqs_after_padding == expected_reqs + + +def test_dummy_draft_does_not_reuse_previous_graph_batch(): + manager, _ = _make_capture_manager(torch.ones((4, 2), dtype=torch.int32)) + dummy_batch = InputBatch.make_dummy(1, 4, manager.input_buffers) + manager.draft_prefill_batch = replace(dummy_batch) + + manager.prepare_draft_prefill(dummy_batch, dummy_batch.input_ids) + + assert ( + manager.get_draft_input_buffers(manager.input_buffers) is manager.input_buffers ) @@ -472,8 +507,11 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): local_batch = manager.partition_batch( global_batch, - padded_num_tokens=4, - padded_num_reqs=4, + BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.FULL, + num_tokens=4, + num_reqs=4, + ), ) assert local_batch.dcp_local_seq_lens is None @@ -489,7 +527,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): # What execute_model does next: derive DCP metadata from the final batch # on the PCP-owned buffers. - local_batch.dcp_local_seq_lens = gpu_cp_utils.maybe_prepare_dcp_local_seq_lens( + local_batch.dcp_local_seq_lens = gpu_cp_utils.prepare_dcp_local_seq_lens( manager.input_buffers.dcp_local_seq_lens, local_batch.seq_lens, local_batch.num_reqs, diff --git a/tests/v1/worker/test_gpu_profiler.py b/tests/v1/worker/test_gpu_profiler.py index cb3b177f41..5b407e80e9 100644 --- a/tests/v1/worker/test_gpu_profiler.py +++ b/tests/v1/worker/test_gpu_profiler.py @@ -611,6 +611,15 @@ _requires_cuda_for_proton = pytest.mark.skipif( reason="Proton profiling tests require an NVIDIA CUDA platform.", ) +# Proton subscribes to CUPTI itself, and a process gets one CUPTI subscriber. +# CI's kernel-launch recorder is another, injected through CUDA_INJECTION64_PATH +# on recording runs, and there Proton's cuptiSubscribe fails with error 39. +_requires_no_injected_cupti_tool = pytest.mark.skipif( + bool(os.environ.get("CUDA_INJECTION64_PATH")), + reason="Another CUPTI tool is injected (CUDA_INJECTION64_PATH); " + "Proton has to be the only one.", +) + @_requires_cuda_for_proton class TestProtonConfig: @@ -1176,6 +1185,7 @@ def test_proton_initializes_before_cuda_graph_capture(): @_requires_cuda_for_proton +@_requires_no_injected_cupti_tool @pytest.mark.parametrize("context", ["shadow", "python"]) @pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"]) def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format): diff --git a/tests/v1/worker/test_gpu_ubatch_slicing.py b/tests/v1/worker/test_gpu_ubatch_slicing.py index 9d3a92e623..ebf334e45d 100644 --- a/tests/v1/worker/test_gpu_ubatch_slicing.py +++ b/tests/v1/worker/test_gpu_ubatch_slicing.py @@ -756,7 +756,7 @@ def test_microbatches_recompute_dcp_lens_from_truncated_seq_lens(dcp_rank: int): [1, 1, 10, 1, 1], [64, 96, 512, 32, 48] ) # execute_model has populated the merged batch's DCP metadata already. - input_batch.dcp_local_seq_lens = gpu_cp_utils.maybe_prepare_dcp_local_seq_lens( + input_batch.dcp_local_seq_lens = gpu_cp_utils.prepare_dcp_local_seq_lens( buffers.dcp_local_seq_lens, input_batch.seq_lens, input_batch.num_reqs, diff --git a/tests/v1/worker/test_gpu_worker.py b/tests/v1/worker/test_gpu_worker.py index 88cbee6802..b63681deca 100644 --- a/tests/v1/worker/test_gpu_worker.py +++ b/tests/v1/worker/test_gpu_worker.py @@ -6,7 +6,10 @@ from types import SimpleNamespace from unittest.mock import patch import pytest +import torch +from tests.utils import create_new_process_for_each_test +from vllm.platforms import current_platform from vllm.utils.mem_constants import GiB_bytes from vllm.v1.worker import gpu_worker, startup_plan from vllm.v1.worker.gpu_worker import maybe_rocm_profiling_fallback @@ -15,6 +18,140 @@ from vllm.v1.worker.startup_plan import ( maybe_save_startup_plan, ) + +def test_load_model_preserves_compiled_graphs_at_runtime(monkeypatch): + """Profiling must use serving's thread count to keep Dynamo guards valid.""" + from torch._dynamo.testing import CompileCounter + + monkeypatch.delenv("OMP_NUM_THREADS", raising=False) + monkeypatch.setattr(gpu_worker, "has_ec_transfer", lambda: False) + monkeypatch.setattr( + gpu_worker, "set_current_vllm_config", lambda config: nullcontext() + ) + loading_threads = [] + worker = SimpleNamespace( + vllm_config=SimpleNamespace(weight_transfer_config=None), + model_runner=SimpleNamespace( + load_model=lambda **kwargs: loading_threads.append(torch.get_num_threads()) + ), + _maybe_get_memory_pool_context=lambda **kwargs: nullcontext(), + _scoped_allocator_max_split=lambda **kwargs: nullcontext(), + ) + original_threads = torch.get_num_threads() + try: + torch.set_num_threads(2) + gpu_worker.Worker.load_model(worker) + assert loading_threads == [2] + + counter = CompileCounter() + compiled = torch.compile(lambda x: x + 1, backend=counter, fullgraph=True) + x = torch.ones(2) + compiled(x) + gpu_worker.set_torch_threads_for_runtime() + torch.testing.assert_close(compiled(x), x + 1) + assert counter.frame_count == 1 + finally: + torch.set_num_threads(original_threads) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike() + or not torch.accelerator.is_available() + or torch.cuda.memory.get_allocator_backend() != "native", + reason="needs the native CUDA or ROCm allocator", +) +# A fresh allocator: blocks cached by earlier tests could serve the large buffer. +@create_new_process_for_each_test("spawn") +def test_scoped_max_split_keeps_freed_large_blocks_releasable(): + """A small allocation made after a large buffer is freed must not pin the + buffer's segment: the profiling run (determine_available_memory) grows + workspaces this way, and a pinned segment survives empty_cache() and is + counted as consumed memory.""" + large = 512 * 1024 * 1024 + small = 2 * 1024 * 1024 # large pool, so it is served by splitting cached blocks + + def reserved_while_small_is_live(scope) -> int: + torch.accelerator.empty_cache() + with scope: + buf = torch.empty(large, dtype=torch.uint8, device="cuda") + del buf + tensor = torch.empty(small, dtype=torch.uint8, device="cuda") + torch.accelerator.empty_cache() + reserved = torch.accelerator.memory_reserved() + del tensor + torch.accelerator.empty_cache() + return reserved + + baseline = torch.accelerator.memory_reserved() + # Without the limit the small tensor is split off the freed block and pins it. + assert reserved_while_small_is_live(nullcontext()) - baseline >= large + scoped = gpu_worker.Worker._scoped_allocator_max_split( + SimpleNamespace(), max_split_size_mb=20 + ) + assert reserved_while_small_is_live(scoped) - baseline < large + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike() + or not torch.accelerator.is_available() + or torch.cuda.memory.get_allocator_backend() != "native", + reason="needs the native CUDA or ROCm allocator", +) +@pytest.mark.parametrize("fail", [False, True]) +@pytest.mark.parametrize("snapshot_fallback", [False, True]) +@pytest.mark.parametrize("suffix", ["", ", "]) +def test_scoped_max_split_preserves_allocator_settings( + monkeypatch, fail, snapshot_fallback, suffix +): + """Preserve allocator settings across successful and failed profiling runs.""" + if snapshot_fallback: + monkeypatch.delattr( + torch._C, "_accelerator_getAllocatorSettings", raising=False + ) + + def settings(): + return torch.cuda.memory._snapshot()["allocator_settings"] + + original = settings()["PYTORCH_CUDA_ALLOC_CONF"] + configured = ( + "max_split_size_mb:128,garbage_collection_threshold:0.8," + "roundup_power2_divisions:[256:1,512:2,>:4],max_non_split_rounding_mb:32" + ) + try: + torch._C._accelerator_setAllocatorSettings(configured + suffix) + before = settings() + expected_error = pytest.raises(RuntimeError, match="profiling failed") + with ( + expected_error if fail else nullcontext(), + gpu_worker.Worker._scoped_allocator_max_split(SimpleNamespace(), 20), + ): + scoped = settings() + assert scoped["max_split_size"] == 20 * 1024 * 1024 + for key in ("garbage_collection_threshold", "roundup_power2_divisions"): + assert scoped[key] == before[key] + if fail: + raise RuntimeError("profiling failed") + assert settings() == before + finally: + torch._C._accelerator_setAllocatorSettings(original) + + +def test_scoped_max_split_ignores_async_allocator(monkeypatch): + """Async allocators ignore max_split and may not support memory snapshots.""" + monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr( + torch.cuda.memory, "get_allocator_backend", lambda: "cudaMallocAsync" + ) + with ( + patch.object(torch.cuda.memory, "_snapshot") as snapshot, + patch.object(torch._C, "_accelerator_setAllocatorSettings") as set_settings, + gpu_worker.Worker._scoped_allocator_max_split(SimpleNamespace(), 20), + ): + pass + snapshot.assert_not_called() + set_settings.assert_not_called() + + # Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and # saved by Worker.determine_available_memory / compile_or_warm_up_model. @@ -221,3 +358,32 @@ def test_execute_model_waits_previous_pp_send_before_forward( assert log == ["wait:prev-tensor", "forward", "isend"] assert worker._pp_send_work == [tensor_handle] + + +def test_jit_monitor_activation_follows_enable_jit_warmup( + monkeypatch: pytest.MonkeyPatch, +): + """The post-warmup JIT monitor must stay off when JIT warmup is disabled + (e.g. by enforce_eager): runtime compilation is then expected, and + warning/erroring on it would be noise.""" + from vllm.utils import jit_monitor + + calls = [] + monkeypatch.setattr(jit_monitor, "activate", lambda **kwargs: calls.append(kwargs)) + + def worker(enable_jit_warmup): + return SimpleNamespace( + vllm_config=SimpleNamespace( + kernel_config=SimpleNamespace(enable_jit_warmup=enable_jit_warmup) + ), + observability_config=SimpleNamespace( + jit_monitor_mode="warn", jit_monitor_verbose=False + ), + ) + + gpu_worker.Worker._maybe_activate_jit_monitor(worker(True)) + assert calls == [{"mode": "warn", "verbose": False}] + + calls.clear() + gpu_worker.Worker._maybe_activate_jit_monitor(worker(False)) + assert calls == [] diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index 0cdd91e0d7..19f77c13e6 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -4,9 +4,16 @@ from types import SimpleNamespace from unittest.mock import Mock +import numpy as np import pytest import torch +from tests.v1.attention.test_gdn_metadata_builder import ( + BLOCK_SIZE, + DEVICE, + _create_gdn_builder, +) +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata from vllm.config.compilation import CUDAGraphMode from vllm.platforms import current_platform from vllm.v1.attention.backends.recoverssm_metadata import ( @@ -21,7 +28,7 @@ from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None: state = object.__new__(MambaHybridModelState) state.vllm_config = SimpleNamespace(num_speculative_tokens=0) - state.max_model_len = 8192 + state.model_config = SimpleNamespace(max_model_len=8192) state._align_mode = False state.recoverssm = None @@ -34,6 +41,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(), query_start_loc=torch.tensor([0, 1], dtype=torch.int32), num_scheduled_tokens=torch.tensor([1], dtype=torch.int32), + max_query_len=None, seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), seq_lens=torch.tensor([1537], dtype=torch.int32), is_prefilling_np=torch.tensor([False]).numpy(), @@ -58,6 +66,79 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non assert build_attn_metadata.call_args.kwargs["positions"] is positions +def test_padded_prompt_tail_builds_as_spec_decode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A one-token prompt tail over prior state, padded with K placeholder + drafts (e.g. a P/D decode-node arrival), must reach the GDN builder as a + spec-decode row. Built as a prefill, the placeholder tokens are folded into + the recurrent state and can't be rolled back. + """ + k = 3 + state = object.__new__(MambaHybridModelState) + state.vllm_config = SimpleNamespace(num_speculative_tokens=k) + state.model_config = SimpleNamespace(max_model_len=8192) + state._align_mode = False + state.recoverssm = None + state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32) + + # A verify decode, a padded prompt tail (128 of 129 prompt tokens already + # computed), and a fresh prompt chunk of the same length. + query_lens = [k + 1] * 3 + seq_lens = [50, 128 + k + 1, k + 1] + is_prefilling = [False, True, True] + query_start_loc = np.array([0, 4, 8, 12], dtype=np.int32) + input_batch = SimpleNamespace( + num_reqs=3, + num_tokens=12, + num_reqs_after_padding=3, + num_tokens_after_padding=12, + idx_mapping=torch.arange(3), + query_start_loc_np=query_start_loc, + query_start_loc=torch.from_numpy(query_start_loc), + num_scheduled_tokens=np.array(query_lens, dtype=np.int32), + num_draft_tokens_per_req=np.array([k, k, 0], dtype=np.int32), + max_query_len=None, + seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + is_prefilling_np=np.array(is_prefilling), + prefill_len_np=np.array([40, 129, 100], dtype=np.int32), + num_computed_prefill_tokens_np=np.array([40, 128, 0], dtype=np.int32), + dcp_local_seq_lens=None, + positions=torch.zeros(12, dtype=torch.int64), + prompt_lens=None, + ) + build_attn_metadata = Mock(return_value={}) + monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata) + state.prepare_attn( + input_batch=input_batch, + cudagraph_mode=CUDAGraphMode.NONE, + block_tables=(), + slot_mappings=torch.empty(0, dtype=torch.int64), + attn_groups=[], + kv_cache_config=Mock(), + ) + mamba_metadata = build_attn_metadata.call_args.kwargs[ + "model_specific_attn_metadata" + ] + + builder = _create_gdn_builder(num_speculative_tokens=k) + common = create_common_attn_metadata( + BatchSpec(seq_lens=seq_lens, query_lens=query_lens), BLOCK_SIZE, DEVICE + ).replace(is_prefilling=torch.tensor(is_prefilling)) + meta = builder.build( + common_prefix_len=0, + common_attn_metadata=common, + num_accepted_tokens=mamba_metadata.num_accepted_tokens, + num_decode_draft_tokens_cpu=mamba_metadata.num_decode_draft_tokens_cpu, + ) + + # Only the fresh prompt chunk needs the prefill kernels. + assert meta.num_spec_decodes == 2 + assert meta.num_prefills == 1 + assert meta.num_prefill_tokens == k + 1 + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") @pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)]) def test_postprocess_state_scalar_with_int32_mapping( diff --git a/tests/v1/worker/test_prompt_embeds_state.py b/tests/v1/worker/test_prompt_embeds_state.py index 1893d0f347..5fd2b02005 100644 --- a/tests/v1/worker/test_prompt_embeds_state.py +++ b/tests/v1/worker/test_prompt_embeds_state.py @@ -15,16 +15,20 @@ import pytest import torch pytest.importorskip("triton") -if not torch.cuda.is_available(): + +from vllm.platforms import current_platform + +if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): pytest.skip( - "CUDA required for prompt-embeds overlay tests", allow_module_level=True + "An accelerator is required for prompt-embeds overlay tests", + allow_module_level=True, ) from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState HIDDEN = 24 MAX_NUM_REQS = 8 -DEVICE = torch.device("cuda") +DEVICE = torch.device(current_platform.device_type) @dataclass diff --git a/tests/v1/worker/test_sleep_mode_backend.py b/tests/v1/worker/test_sleep_mode_backend.py index 686331f2d7..a8b8d61ae2 100644 --- a/tests/v1/worker/test_sleep_mode_backend.py +++ b/tests/v1/worker/test_sleep_mode_backend.py @@ -4,7 +4,7 @@ These cover the registry/factory contract and capability flags. They do not touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU -in ``tests/basic_correctness/test_cumem.py``. +in ``tests/basic_correctness/memory/``. """ import pytest diff --git a/tests/watermarking/__init__.py b/tests/watermarking/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/watermarking/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/watermarking/generate_goldens.py b/tests/watermarking/generate_goldens.py new file mode 100644 index 0000000000..3cf5df1d42 --- /dev/null +++ b/tests/watermarking/generate_goldens.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Regenerate tests/watermarking/watermarking_goldens.json.""" + +import json +from pathlib import Path + +from tests.watermarking.golden_candidates import golden_payload, validate_golden_guards + +GOLDENS_PATH = Path(__file__).with_name("watermarking_goldens.json") + + +def main() -> None: + payload = golden_payload() + validate_golden_guards(payload["candidates"]) + GOLDENS_PATH.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"wrote {len(payload['candidates'])} candidates to {GOLDENS_PATH}") + + +if __name__ == "__main__": + main() diff --git a/tests/watermarking/golden_candidates.py b/tests/watermarking/golden_candidates.py new file mode 100644 index 0000000000..a06ef00c84 --- /dev/null +++ b/tests/watermarking/golden_candidates.py @@ -0,0 +1,885 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Deterministic watermarking golden candidates.""" + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, replace +from pathlib import Path +from typing import get_args + +import torch + +from vllm.config.watermarking import ( + WatermarkConfig, + WatermarkContextScope, + WatermarkingAlgorithm, + WatermarkPRFName, + derive_watermark_key, +) +from vllm.v1.watermarking import ( + DualKeyGumbelWatermarkDetector, + GumbelWatermarkDetector, + WatermarkDetection, + WatermarkDetector, + create_prf, + create_watermarker, +) +from vllm.v1.worker.gpu.sample.watermark import repeated_context_mask + +GOLDEN_SCHEMA_VERSION = 3 + +# JSON floats round-trip exactly, but libm differs by an ULP across builds. +GOLDEN_FLOAT_RTOL = 1e-9 +GOLDEN_FLOAT_ATOL = 1e-300 + +P_VALUE_RATIO_MIN_DISTANCE = 1e-6 +REPETITIVE_MIN_KEY_PERTURBATION_TOKENS = 8 +REPETITIVE_MAX_P_VALUE_RATIO = 0.5 +ROUTING_BOUNDARY_MIN_DISTANCE = 1e-6 + +WATERMARK_CONFIG_FIELDS = ( + "algorithm", + "alpha", + "context_width", + "deduplicate_contexts", + "deduplicate_contexts_max_history", + "prf", + "allow_target_only_watermarking", +) + +# round(((i + 1) * (5**0.5 - 1) / 2) % 1.0, 4) for i in range(64) +ROUTING_UNIFORMS: tuple[float, ...] = ( + 0.618, 0.2361, 0.8541, 0.4721, 0.0902, 0.7082, 0.3262, 0.9443, + 0.5623, 0.1803, 0.7984, 0.4164, 0.0344, 0.6525, 0.2705, 0.8885, + 0.5066, 0.1246, 0.7426, 0.3607, 0.9787, 0.5967, 0.2148, 0.8328, + 0.4508, 0.0689, 0.6869, 0.305, 0.923, 0.541, 0.1591, 0.7771, + 0.3951, 0.0132, 0.6312, 0.2492, 0.8673, 0.4853, 0.1033, 0.7214, + 0.3394, 0.9574, 0.5755, 0.1935, 0.8115, 0.4296, 0.0476, 0.6656, + 0.2837, 0.9017, 0.5197, 0.1378, 0.7558, 0.3738, 0.9919, 0.6099, + 0.2279, 0.846, 0.464, 0.082, 0.7001, 0.3181, 0.9361, 0.5542, +) # fmt: skip + + +class GoldenFormatError(ValueError): + """The stored goldens do not have the shape this module expects.""" + + +class GoldenGuardError(ValueError): + """A candidate is too weak or too unstable to be frozen as a golden.""" + + +@dataclass(frozen=True) +class DeterministicGenerationFixture: + vocabulary_size: int = 128 + num_tokens: int = 64 + logit_denominator: int = 16 + logit_modulus: int = 131 + dominant_token: int | None = None + dominant_bias: int = 0 + prompt: tuple[int, ...] = () + routing_uniforms: tuple[float, ...] = ROUTING_UNIFORMS + + def logits(self, position: int) -> torch.Tensor: + token_ids = torch.arange(self.vocabulary_size, dtype=torch.int64) + values = (token_ids * 37 + position * 17) % self.logit_modulus - 56 + if self.dominant_token is not None: + values[self.dominant_token] += self.dominant_bias + return (values.to(torch.float32) / self.logit_denominator).unsqueeze(0) + + +@dataclass(frozen=True) +class WatermarkingSchemeConfig: + context_width: int = 4 + generation_alpha: float = 0.1 + detection_alpha: float = 0.1 + generation_deduplicate_contexts: WatermarkContextScope = "single_turn" + generation_deduplicate_contexts_max_history: int | None = 8192 + detection_deduplicate_contexts: bool = True + p_value_threshold: float = 0.01 + + +@dataclass(frozen=True) +class GenerationTrace: + routing_draws: int + key_b_routed: int | None + key_b_effective: int | None + dedup_skips: int + partial_context_skips: int + + +class _DeterministicSampler: + def __init__(self, routing_uniforms: tuple[float, ...]) -> None: + self.routing_uniforms = routing_uniforms + self.routing_position = 0 + self.key_b_routed = 0 + + def __call__(self, logits: torch.Tensor) -> torch.Tensor: + if logits.shape[-1] != 2: + return logits.argmax(dim=-1) + + probability_a = logits.softmax(dim=-1)[:, 0] + uniform = self.routing_uniforms[ + self.routing_position % len(self.routing_uniforms) + ] + self.routing_position += 1 + routed = (probability_a <= uniform).to(torch.int64) + self.key_b_routed += int(routed.sum()) + return routed + + +def routing_boundary(alpha: float) -> float: + """Return the realised key-A probability the deterministic sampler sees.""" + logits = torch.tensor([1 - alpha, alpha], dtype=torch.float32).log() + return float(logits.softmax(dim=-1)[0]) + + +def _create_gumbel_detector( + key: int, + config: WatermarkingSchemeConfig, + prf: WatermarkPRFName, +) -> WatermarkDetector: + return GumbelWatermarkDetector( + key=key, + context_width=config.context_width, + p_value_threshold=config.p_value_threshold, + prf=prf, + deduplicate_contexts=config.detection_deduplicate_contexts, + ) + + +def _create_dual_key_gumbel_detector( + key: int, + config: WatermarkingSchemeConfig, + prf: WatermarkPRFName, +) -> WatermarkDetector: + return DualKeyGumbelWatermarkDetector( + key=key, + context_width=config.context_width, + p_value_threshold=config.p_value_threshold, + prf=prf, + deduplicate_contexts=config.detection_deduplicate_contexts, + alpha=config.detection_alpha, + ) + + +DETECTOR_FACTORIES = { + "gumbel": _create_gumbel_detector, + "dual_key_gumbel": _create_dual_key_gumbel_detector, +} + + +@dataclass(frozen=True) +class WatermarkingCandidate: + id: str + scheme: WatermarkingAlgorithm + scheme_config: WatermarkingSchemeConfig + prf: WatermarkPRFName + key: int + detection_key: int + fixture: DeterministicGenerationFixture = DeterministicGenerationFixture() + + def configuration(self) -> dict[str, object]: + prf_version = getattr(create_prf(self.prf, self.key), "version", None) + if not isinstance(prf_version, str): + raise ValueError(f"PRF {self.prf} does not define a version") + return { + "scheme": self.scheme, + "scheme_config": { + "context_width": self.scheme_config.context_width, + "generation_alpha": self.scheme_config.generation_alpha, + "detection_alpha": self.scheme_config.detection_alpha, + "generation_deduplicate_contexts": ( + self.scheme_config.generation_deduplicate_contexts + ), + "generation_deduplicate_contexts_max_history": ( + self.scheme_config.generation_deduplicate_contexts_max_history + ), + "detection_deduplicate_contexts": ( + self.scheme_config.detection_deduplicate_contexts + ), + "p_value_threshold": self.scheme_config.p_value_threshold, + }, + "prf": self.prf, + "prf_version": prf_version, + "key": str(self.key), + "detection_key": str(self.detection_key), + "fixture": { + "vocabulary_size": self.fixture.vocabulary_size, + "num_tokens": self.fixture.num_tokens, + "logit_denominator": self.fixture.logit_denominator, + "logit_modulus": self.fixture.logit_modulus, + "dominant_token": self.fixture.dominant_token, + "dominant_bias": self.fixture.dominant_bias, + "prompt": list(self.fixture.prompt), + }, + } + + def resolved(self) -> dict[str, object]: + """Return the vLLM-side state this candidate resolves to. + + An allowlist rather than the whole dataclass, so a new production field + is a deliberate golden change: WATERMARK_CONFIG_FIELDS names the fields + recorded here and test_goldens.py checks it still covers + WatermarkConfig. Keys exceed 2**53 and are decimal strings. + """ + config = self._watermark_config() + detector = self._detector() + key_b_prf = getattr(detector, "key_b_prf", None) + dual_key = self.scheme == "dual_key_gumbel" + return { + "watermark_config": { + field: getattr(config, field) for field in WATERMARK_CONFIG_FIELDS + }, + "derived_keys": { + "key_a": str(derive_watermark_key(self.key, b"key_a")) + if dual_key + else None, + "key_b": str(derive_watermark_key(self.key, b"key_b")) + if dual_key + else None, + }, + "detector": { + "type": type(detector).__name__, + "context_width": detector.context_width, + "p_value_threshold": detector.p_value_threshold, + "deduplicate_contexts": detector.deduplicate_contexts, + "alpha": getattr(detector, "alpha", None), + "prf_key": str(detector.prf.key), + "key_b_prf_key": None if key_b_prf is None else str(key_b_prf.key), + }, + } + + def generate(self) -> list[int]: + return self.generate_with_trace()[0] + + def generate_with_trace(self) -> tuple[list[int], GenerationTrace]: + watermarker = create_watermarker(self._watermark_config()) + sampler = _DeterministicSampler(self.fixture.routing_uniforms) + generated: list[int] = [] + dedup_skips = 0 + partial_context_skips = 0 + key_b_effective = 0 + + for position in range(self.fixture.num_tokens): + context = self._context(generated) + skip_mask = self._skip_mask(generated, context) + skipped = skip_mask is not None and bool(skip_mask.item()) + if skipped: + dedup_skips += 1 + if len(generated) < self.scheme_config.context_width: + partial_context_skips += 1 + routed_before = sampler.key_b_routed + sample = watermarker.sample( + self.fixture.logits(position), + context, + sampler, + skip_mask, + ) + if sampler.key_b_routed > routed_before and not skipped: + key_b_effective += 1 + generated.append(int(sample.token_ids.item())) + + routes = self.scheme == "dual_key_gumbel" and ( + 0 < self.scheme_config.generation_alpha < 1 + ) + trace = GenerationTrace( + routing_draws=sampler.routing_position, + key_b_routed=sampler.key_b_routed if routes else None, + key_b_effective=key_b_effective if routes else None, + dedup_skips=dedup_skips, + partial_context_skips=partial_context_skips, + ) + return generated, trace + + def detect(self, token_ids: list[int]) -> WatermarkDetection: + return self._detector().detect(token_ids) + + def golden_entry( + self, + detection_token_ids: list[int] | None = None, + ) -> dict: + """Generate, detect and trace this candidate into one golden entry. + + ``detection_token_ids`` runs detection over stored tokens instead of + the freshly generated ones, so a token drift and a detection drift are + reported independently. + """ + token_ids, trace = self.generate_with_trace() + scored = token_ids if detection_token_ids is None else detection_token_ids + detection = self.detect(scored) + threshold = self.scheme_config.p_value_threshold + return { + "configuration": self.configuration(), + "resolved": self.resolved(), + "generation": token_ids, + "trace": _trace_payload(trace), + "detection": { + "score": detection.score, + "p_value": detection.p_value, + "p_value_ratio": detection.p_value / threshold, + "num_scored_tokens": detection.num_scored_tokens, + "is_watermarked": detection.is_watermarked, + }, + } + + def _watermark_config(self) -> WatermarkConfig: + return WatermarkConfig( + algorithm=self.scheme, + key=self.key, + alpha=self.scheme_config.generation_alpha, + context_width=self.scheme_config.context_width, + deduplicate_contexts=(self.scheme_config.generation_deduplicate_contexts), + deduplicate_contexts_max_history=( + self.scheme_config.generation_deduplicate_contexts_max_history + ), + prf=self.prf, + ) + + def _detector(self) -> WatermarkDetector: + return DETECTOR_FACTORIES[self.scheme]( + self.detection_key, + self.scheme_config, + self.prf, + ) + + def _context(self, generated: list[int]) -> torch.Tensor: + width = self.scheme_config.context_width + padded = [-1] * width + generated + return torch.tensor([padded[-width:]], dtype=torch.int64) + + def _skip_mask( + self, generated: list[int], context: torch.Tensor + ) -> torch.Tensor | None: + scope = self.scheme_config.generation_deduplicate_contexts + if scope == "none": + return None + + prompt_len = len(self.fixture.prompt) + storage = torch.zeros( + (1, prompt_len + self.fixture.num_tokens), + dtype=torch.int64, + ) + if prompt_len: + storage[0, :prompt_len] = torch.tensor(self.fixture.prompt) + if generated: + storage[0, prompt_len : prompt_len + len(generated)] = torch.tensor( + generated + ) + return repeated_context_mask( + storage, + torch.tensor([0]), + torch.tensor([prompt_len]), + torch.tensor([prompt_len + len(generated)]), + context, + self.scheme_config.generation_deduplicate_contexts_max_history, + include_prompt=scope == "all", + skip_partial_context=scope == "all", + ) + + +def _trace_payload(trace: GenerationTrace) -> dict: + return { + "routing_draws": trace.routing_draws, + "key_b_routed": trace.key_b_routed, + "key_b_effective": trace.key_b_effective, + "dedup_skips": trace.dedup_skips, + "partial_context_skips": trace.partial_context_skips, + } + + +BALANCED_FIXTURE = DeterministicGenerationFixture() + +# The first five tokens of BALANCED_FIXTURE under scope "all", so position 4 repeats. +PROMPT_FIXTURE = DeterministicGenerationFixture(prompt=(46, 42, 38, 34, 122)) + +REPETITIVE_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=16, + num_tokens=48, + dominant_token=3, + dominant_bias=52, +) + +MIDDLE_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=8, + num_tokens=64, + dominant_token=3, + dominant_bias=16, +) + +# Few enough scored tokens that the p_value lands just under its threshold. +NEAR_THRESHOLD_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=16, + num_tokens=8, +) + + +def _candidate( + id: str, + scheme: WatermarkingAlgorithm, + key: int, + *, + prf: WatermarkPRFName, + detection_key: int | None = None, + context_width: int = 4, + generation_alpha: float = 0.1, + detection_alpha: float | None = None, + generation_deduplicate_contexts: WatermarkContextScope = "single_turn", + generation_deduplicate_contexts_max_history: int | None = 8192, + detection_deduplicate_contexts: bool = True, + p_value_threshold: float = 0.01, + fixture: DeterministicGenerationFixture = BALANCED_FIXTURE, +) -> WatermarkingCandidate: + return WatermarkingCandidate( + id=id, + scheme=scheme, + scheme_config=WatermarkingSchemeConfig( + context_width=context_width, + generation_alpha=generation_alpha, + detection_alpha=( + generation_alpha if detection_alpha is None else detection_alpha + ), + generation_deduplicate_contexts=generation_deduplicate_contexts, + generation_deduplicate_contexts_max_history=( + generation_deduplicate_contexts_max_history + ), + detection_deduplicate_contexts=detection_deduplicate_contexts, + p_value_threshold=p_value_threshold, + ), + prf=prf, + key=key, + detection_key=key if detection_key is None else detection_key, + fixture=fixture, + ) + + +WATERMARKING_CANDIDATES = ( + _candidate("gumbel-philox-key0-cw1", "gumbel", 0, prf="philox", context_width=1), + _candidate("gumbel-philox-key42-cw4", "gumbel", 42, prf="philox"), + _candidate( + "gumbel-philox-key42-cw4-wrong-key", + "gumbel", + 42, + prf="philox", + detection_key=43, + ), + _candidate( + "gumbel-philox-key2p32-cw5-no-dedup", + "gumbel", + 2**32, + prf="philox", + context_width=5, + generation_deduplicate_contexts="none", + detection_deduplicate_contexts=False, + ), + _candidate( + "gumbel-philox-keymax-cw16-all", + "gumbel", + 2**64 - 1, + prf="philox", + context_width=16, + generation_deduplicate_contexts="all", + ), + _candidate( + "gumbel-philox-key42-cw4-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + ), + _candidate( + "gumbel-philox-key42-cw4-all-prompt", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=PROMPT_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-history8", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts_max_history=8, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-history-none", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts_max_history=None, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-no-dedup", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="none", + generation_deduplicate_contexts_max_history=None, + detection_deduplicate_contexts=False, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + generation_deduplicate_contexts_max_history=None, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-near-threshold", + "gumbel", + 42, + prf="philox", + p_value_threshold=0.0015, + fixture=NEAR_THRESHOLD_FIXTURE, + ), + _candidate( + "gumbel-philox-repeated-cw1", + "gumbel", + 42, + prf="philox", + context_width=1, + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "gumbel-philox-repeated-cw4-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-key0-cw1-alpha0", + "dual_key_gumbel", + 0, + prf="philox", + context_width=1, + generation_alpha=0.0, + ), + _candidate( + "dual-key-gumbel-philox-key0-cw4-alpha0.1", + "dual_key_gumbel", + 0, + prf="philox", + ), + _candidate( + "dual-key-gumbel-philox-key42-cw4-alpha-mismatch", + "dual_key_gumbel", + 42, + prf="philox", + generation_alpha=0.1, + detection_alpha=0.2, + ), + _candidate( + "dual-key-gumbel-philox-key2p32-cw5-alpha0.5", + "dual_key_gumbel", + 2**32, + prf="philox", + context_width=5, + generation_alpha=0.5, + ), + _candidate( + "dual-key-gumbel-philox-keymax-cw16-alpha1", + "dual_key_gumbel", + 2**64 - 1, + prf="philox", + context_width=16, + generation_alpha=1.0, + ), + _candidate( + "dual-key-gumbel-philox-keymax-cw4-alpha0.4", + "dual_key_gumbel", + 2**64 - 1, + prf="philox", + generation_alpha=0.4, + ), + _candidate( + "dual-key-gumbel-philox-key7-cw4-near-threshold", + "dual_key_gumbel", + 7, + prf="philox", + p_value_threshold=0.003, + fixture=NEAR_THRESHOLD_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-repeated-cw1", + "dual_key_gumbel", + 42, + prf="philox", + context_width=1, + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-repeated-cw4-all", + "dual_key_gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=REPETITIVE_FIXTURE, + ), +) + +DEDUPLICATION_TWINS = ( + ( + "gumbel-philox-key42-cw4-mid-history-none", + "gumbel-philox-key42-cw4-mid-no-dedup", + ), +) + + +def validate_golden_guards(entries: Mapping[str, dict]) -> None: + candidates_by_id = { + candidate.id: candidate for candidate in WATERMARKING_CANDIDATES + } + combinations = { + (candidate.scheme, candidate.prf) for candidate in WATERMARKING_CANDIDATES + } + expected_combinations = configured_algorithm_prf_combinations() + if combinations != expected_combinations: + missing_combinations = sorted(expected_combinations - combinations) + unexpected_combinations = sorted(combinations - expected_combinations) + raise GoldenGuardError( + "candidate scheme/PRF coverage mismatch: " + f"missing {missing_combinations}, unexpected {unexpected_combinations}" + ) + + missing = sorted(set(candidates_by_id) - set(entries)) + if missing: + raise GoldenGuardError(f"golden entries are missing candidates: {missing}") + + for candidate in WATERMARKING_CANDIDATES: + entry = entries[candidate.id] + ratio = entry["detection"]["p_value_ratio"] + if abs(ratio - 1.0) < P_VALUE_RATIO_MIN_DISTANCE: + raise GoldenGuardError( + f"{candidate.id}: p_value is within " + f"{P_VALUE_RATIO_MIN_DISTANCE:g} of p_value_threshold " + f"(ratio {ratio!r}), so is_watermarked is not reproducible" + ) + + if candidate.fixture == REPETITIVE_FIXTURE: + alternate_tokens = replace(candidate, key=candidate.key ^ 1).generate() + changed_tokens = sum( + actual != alternate + for actual, alternate in zip( + entry["generation"], alternate_tokens, strict=True + ) + ) + if changed_tokens < REPETITIVE_MIN_KEY_PERTURBATION_TOKENS: + raise GoldenGuardError( + f"{candidate.id}: key ^ 1 changes only {changed_tokens} tokens; " + f"expected at least {REPETITIVE_MIN_KEY_PERTURBATION_TOKENS}" + ) + full_context_skips = ( + entry["trace"]["dedup_skips"] - entry["trace"]["partial_context_skips"] + ) + if full_context_skips < 1: + raise GoldenGuardError( + f"{candidate.id}: repetitive fixture has no full-context " + "deduplication skip" + ) + if ratio > REPETITIVE_MAX_P_VALUE_RATIO: + raise GoldenGuardError( + f"{candidate.id}: p_value_ratio {ratio!r} exceeds " + f"{REPETITIVE_MAX_P_VALUE_RATIO}" + ) + + alpha = candidate.scheme_config.generation_alpha + if candidate.scheme == "dual_key_gumbel" and 0 < alpha < 1: + boundary = routing_boundary(alpha) + distance = min( + abs(uniform - boundary) + for uniform in candidate.fixture.routing_uniforms + ) + if distance < ROUTING_BOUNDARY_MIN_DISTANCE: + raise GoldenGuardError( + f"{candidate.id}: routing uniform is only {distance!r} from " + f"the realised boundary; expected at least " + f"{ROUTING_BOUNDARY_MIN_DISTANCE:g}" + ) + + for enabled_id, disabled_id in DEDUPLICATION_TWINS: + enabled = candidates_by_id[enabled_id] + disabled = candidates_by_id[disabled_id] + expected_disabled = replace( + enabled, + id=disabled_id, + scheme_config=replace( + enabled.scheme_config, + generation_deduplicate_contexts="none", + detection_deduplicate_contexts=False, + ), + ) + if disabled != expected_disabled: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id} must differ only in generation " + "and detection deduplication" + ) + + enabled_entry = entries[enabled_id] + disabled_entry = entries[disabled_id] + if enabled_entry["trace"]["dedup_skips"] < 1: + raise GoldenGuardError(f"{enabled_id}: expected at least one dedup skip") + if disabled_entry["trace"]["dedup_skips"] != 0: + raise GoldenGuardError(f"{disabled_id}: expected zero dedup skips") + if enabled_entry["generation"] == disabled_entry["generation"]: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id}: generations must differ" + ) + enabled_scored = enabled_entry["detection"]["num_scored_tokens"] + disabled_scored = disabled_entry["detection"]["num_scored_tokens"] + if enabled_scored == disabled_scored: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id}: detector scored-token counts " + "must differ" + ) + + +def golden_payload() -> dict: + candidates = { + candidate.id: candidate.golden_entry() for candidate in WATERMARKING_CANDIDATES + } + return { + "schema_version": GOLDEN_SCHEMA_VERSION, + "candidates": candidates, + } + + +def floats_match(actual: float, expected: float) -> bool: + return math.isclose( + actual, + expected, + rel_tol=GOLDEN_FLOAT_RTOL, + abs_tol=GOLDEN_FLOAT_ATOL, + ) + + +def _relative_difference(actual: float, expected: float) -> float: + if expected == 0.0: + return 0.0 if actual == 0.0 else math.inf + return abs(actual - expected) / abs(expected) + + +def _compare_float( + field: str, + actual: float, + expected: float, + differences: list[str], +) -> None: + if not floats_match(actual, expected): + differences.append( + f"{field}: expected {expected!r}, got {actual!r}, " + f"relative difference {_relative_difference(actual, expected):.3e}" + ) + + +def _compare_exact( + field: str, + actual: object, + expected: object, + differences: list[str], +) -> None: + if actual != expected: + differences.append(f"{field}: expected {expected!r}, got {actual!r}") + + +def _compare_mapping( + field: str, + actual: Mapping[str, object], + expected: Mapping[str, object], + differences: list[str], +) -> None: + for key in sorted(set(actual) | set(expected)): + if actual.get(key) != expected.get(key): + differences.append( + f"{field}.{key}: expected {expected.get(key)!r}, " + f"got {actual.get(key)!r}" + ) + + +def compare_entries(produced: dict, golden: dict) -> list[str]: + """Return one line per field that differs, empty when the two agree. + + Configuration, resolved state, tokens, trace, + num_scored_tokens and is_watermarked are compared exactly; score, p_value + and p_value_ratio are compared with GOLDEN_FLOAT_RTOL. + """ + differences: list[str] = [] + for field in ("configuration", "resolved", "trace"): + _compare_mapping(field, produced[field], golden[field], differences) + + if produced["generation"] != golden["generation"]: + first_difference = next( + ( + index + for index, (actual, expected) in enumerate( + zip(produced["generation"], golden["generation"]) + ) + if actual != expected + ), + min(len(produced["generation"]), len(golden["generation"])), + ) + differences.append( + f"generation: first differs at index {first_difference} " + f"({len(produced['generation'])} tokens generated, " + f"{len(golden['generation'])} in the golden): " + f"expected {golden['generation']}, got {produced['generation']}" + ) + + detection = produced["detection"] + expected_detection = golden["detection"] + for field in ("score", "p_value", "p_value_ratio"): + _compare_float( + f"detection.{field}", + detection[field], + expected_detection[field], + differences, + ) + for field in ("num_scored_tokens", "is_watermarked"): + _compare_exact( + f"detection.{field}", + detection[field], + expected_detection[field], + differences, + ) + return differences + + +def compare_golden(candidate: WatermarkingCandidate, golden: dict) -> list[str]: + """Return one line per field of ``candidate`` that drifted from ``golden``.""" + return compare_entries(candidate.golden_entry(golden["generation"]), golden) + + +def read_goldens(path: Path) -> dict[str, dict]: + """Read the goldens file and return its candidate mapping.""" + payload = json.loads(path.read_text(encoding="utf-8")) + version = payload.get("schema_version") + if version != GOLDEN_SCHEMA_VERSION: + raise GoldenFormatError( + f"{path}: schema_version {version!r}, expected {GOLDEN_SCHEMA_VERSION}" + ) + return payload["candidates"] + + +def configured_algorithms() -> set[str]: + return set(get_args(WatermarkingAlgorithm)) + + +def configured_prfs() -> set[str]: + return set(get_args(WatermarkPRFName)) + + +def configured_algorithm_prf_combinations() -> set[tuple[str, str]]: + return { + (algorithm, prf) + for algorithm in configured_algorithms() + for prf in configured_prfs() + } diff --git a/tests/watermarking/test_goldens.py b/tests/watermarking/test_goldens.py new file mode 100644 index 0000000000..5e12a9af88 --- /dev/null +++ b/tests/watermarking/test_goldens.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Check the watermarking goldens still reproduce.""" + +import dataclasses +from pathlib import Path + +import pytest + +from tests.watermarking.golden_candidates import ( + DETECTOR_FACTORIES, + GOLDEN_FLOAT_RTOL, + WATERMARK_CONFIG_FIELDS, + WATERMARKING_CANDIDATES, + compare_golden, + configured_algorithm_prf_combinations, + configured_algorithms, + read_goldens, + routing_boundary, + validate_golden_guards, +) +from vllm.config.watermarking import WatermarkConfig + +GOLDENS_PATH = Path(__file__).with_name("watermarking_goldens.json") + +CANDIDATES_BY_ID = {candidate.id: candidate for candidate in WATERMARKING_CANDIDATES} + +NEGATIVE_CANDIDATE_ID = "gumbel-philox-key42-cw4-wrong-key" + + +@pytest.fixture(scope="module") +def goldens() -> dict[str, dict]: + return read_goldens(GOLDENS_PATH) + + +def test_goldens_cover_every_candidate(goldens: dict[str, dict]): + candidate_ids = set(CANDIDATES_BY_ID) + + assert len(candidate_ids) == len(WATERMARKING_CANDIDATES) + assert set(goldens) == candidate_ids, "golden contract does not match candidates" + candidate_combinations = { + (candidate.scheme, candidate.prf) for candidate in WATERMARKING_CANDIDATES + } + assert candidate_combinations == configured_algorithm_prf_combinations() + assert set(DETECTOR_FACTORIES) == configured_algorithms() + + +def test_golden_fixture_guards(goldens: dict[str, dict]): + validate_golden_guards(goldens) + + +def test_resolved_records_every_watermark_config_field(): + assert {field.name for field in dataclasses.fields(WatermarkConfig)} == ( + set(WATERMARK_CONFIG_FIELDS) | {"key"} + ) + + +@pytest.mark.parametrize( + "candidate", + WATERMARKING_CANDIDATES, + ids=lambda candidate: candidate.id, +) +def test_watermarking_candidate_golden(candidate, goldens: dict[str, dict]): + differences = compare_golden(candidate, goldens[candidate.id]) + + assert not differences, "\n".join( + [ + f"{candidate.id}: golden drift " + f"(tolerance {GOLDEN_FLOAT_RTOL:g} on score and p_value)", + *differences, + ] + ) + + +def test_negative_row_is_not_watermarked(goldens: dict[str, dict]): + candidate = CANDIDATES_BY_ID[NEGATIVE_CANDIDATE_ID] + + assert candidate.detection_key != candidate.key + assert goldens[NEGATIVE_CANDIDATE_ID]["detection"]["is_watermarked"] is False + + +@pytest.mark.parametrize( + "candidate", + [ + candidate + for candidate in WATERMARKING_CANDIDATES + if candidate.scheme == "dual_key_gumbel" + and 0 < candidate.scheme_config.generation_alpha < 1 + ], + ids=lambda candidate: candidate.id, +) +def test_routing_sequence_resolves_alpha(candidate): + alpha = candidate.scheme_config.generation_alpha + uniforms = candidate.fixture.routing_uniforms + boundary = routing_boundary(alpha) + share = sum(1 for uniform in uniforms if boundary <= uniform) / len(uniforms) + + assert abs(share - alpha) <= 0.03 diff --git a/tests/watermarking/test_watermarking.py b/tests/watermarking/test_watermarking.py index 65ea9e93ad..232f19a9a6 100644 --- a/tests/watermarking/test_watermarking.py +++ b/tests/watermarking/test_watermarking.py @@ -276,9 +276,9 @@ def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch): "vllm.v1.watermarking.gpu_sampler.logger.warning_once", messages.append ) - sampler.add_request(0, 1, SamplingParams(temperature=0)) - sampler.add_request(0, 1, SamplingParams(temperature=1)) - sampler.add_request(0, 1, SamplingParams(temperature=0, watermarking=False)) + sampler.add_request(0, SamplingParams(temperature=0)) + sampler.add_request(0, SamplingParams(temperature=1)) + sampler.add_request(0, SamplingParams(temperature=0, watermarking=False)) assert messages == [ ( @@ -945,6 +945,7 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): speculator = object.__new__(StubSpeculator) speculator.model = StubModel() speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None draft_watermarker = object.__new__(DraftWatermarker) draft_watermarker.watermarker = StubWatermarker() draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]]) @@ -978,6 +979,7 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator.seeds = torch.zeros(2, dtype=torch.int64) speculator._step_cols = torch.tensor([0]) speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None watermark_logits: list[torch.Tensor] = [] def sample(logits, sampled, idx_map, temperature): @@ -1012,6 +1014,7 @@ def test_dspark_target_only_watermarking_leaves_drafts_unwatermarked(monkeypatch speculator.seeds = torch.zeros(2, dtype=torch.int64) speculator._step_cols = torch.tensor([0]) speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None speculator.draft_watermarker = None monkeypatch.setattr( "vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample", diff --git a/tests/watermarking/watermarking_goldens.json b/tests/watermarking/watermarking_goldens.json new file mode 100644 index 0000000000..ddcd142729 --- /dev/null +++ b/tests/watermarking/watermarking_goldens.json @@ -0,0 +1,2848 @@ +{ + "candidates": { + "dual-key-gumbel-philox-key0-cw1-alpha0": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.0, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.0, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 55, + "p_value": 2.7410983620034455e-44, + "p_value_ratio": 2.7410983620034455e-42, + "score": 229.87923167230588 + }, + "generation": [ + 81, + 91, + 84, + 122, + 37, + 118, + 11, + 3, + 116, + 56, + 66, + 37, + 44, + 40, + 85, + 31, + 102, + 73, + 126, + 16, + 40, + 93, + 11, + 85, + 81, + 77, + 41, + 12, + 118, + 61, + 57, + 53, + 38, + 111, + 115, + 5, + 1, + 99, + 113, + 28, + 87, + 55, + 83, + 51, + 47, + 57, + 124, + 31, + 116, + 112, + 30, + 43, + 89, + 110, + 60, + 10, + 9, + 58, + 114, + 125, + 68, + 78, + 106, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": "15691511769010696906", + "key_b": "9736202488081271922" + }, + "detector": { + "alpha": 0.0, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": "9736202488081271922", + "p_value_threshold": 0.01, + "prf_key": "15691511769010696906", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.0, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 9, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "dual-key-gumbel-philox-key0-cw4-alpha0.1": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.234132682420224e-51, + "p_value_ratio": 1.234132682420224e-49, + "score": 241.81126265648692 + }, + "generation": [ + 70, + 56, + 91, + 90, + 72, + 68, + 17, + 78, + 21, + 116, + 104, + 62, + 15, + 100, + 82, + 13, + 88, + 70, + 112, + 62, + 79, + 57, + 11, + 113, + 88, + 80, + 23, + 90, + 79, + 42, + 18, + 60, + 10, + 73, + 94, + 64, + 96, + 71, + 35, + 56, + 38, + 93, + 37, + 54, + 82, + 127, + 0, + 45, + 37, + 75, + 19, + 92, + 68, + 127, + 28, + 56, + 45, + 69, + 8, + 1, + 110, + 39, + 127, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": "15691511769010696906", + "key_b": "9736202488081271922" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "9736202488081271922", + "p_value_threshold": 0.01, + "prf_key": "15691511769010696906", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 7, + "key_b_routed": 7, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key2p32-cw5-alpha0.5": { + "configuration": { + "detection_key": "4294967296", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "4294967296", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 5, + "detection_alpha": 0.5, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.5, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.733621085489817e-36, + "p_value_ratio": 6.7336210854898174e-34, + "score": 162.0665949787407 + }, + "generation": [ + 53, + 116, + 48, + 94, + 47, + 118, + 64, + 123, + 106, + 65, + 59, + 48, + 4, + 86, + 64, + 32, + 84, + 116, + 23, + 23, + 65, + 15, + 50, + 67, + 49, + 69, + 34, + 76, + 1, + 95, + 103, + 120, + 102, + 69, + 125, + 121, + 57, + 39, + 127, + 113, + 98, + 87, + 9, + 51, + 100, + 53, + 46, + 42, + 123, + 55, + 108, + 100, + 50, + 103, + 67, + 80, + 94, + 41, + 97, + 117, + 18, + 66, + 74, + 38 + ], + "resolved": { + "derived_keys": { + "key_a": "9864452095710749384", + "key_b": "10834826187781349534" + }, + "detector": { + "alpha": 0.5, + "context_width": 5, + "deduplicate_contexts": true, + "key_b_prf_key": "10834826187781349534", + "p_value_threshold": 0.01, + "prf_key": "9864452095710749384", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.5, + "context_width": 5, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 33, + "key_b_routed": 33, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key42-cw4-alpha-mismatch": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.2, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.6569954931292397e-49, + "p_value_ratio": 1.6569954931292398e-47, + "score": 215.24386344560963 + }, + "generation": [ + 46, + 56, + 6, + 122, + 30, + 100, + 121, + 67, + 63, + 116, + 13, + 65, + 90, + 47, + 25, + 88, + 120, + 31, + 2, + 37, + 40, + 34, + 92, + 28, + 45, + 126, + 94, + 90, + 18, + 36, + 74, + 102, + 10, + 13, + 101, + 51, + 29, + 29, + 46, + 81, + 63, + 2, + 29, + 78, + 121, + 89, + 124, + 109, + 123, + 34, + 30, + 93, + 4, + 110, + 28, + 34, + 105, + 41, + 5, + 48, + 81, + 120, + 60, + 70 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.2, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 7, + "key_b_routed": 7, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key7-cw4-near-threshold": { + "configuration": { + "detection_key": "7", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 8, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "7", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.003 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 8, + "p_value": 0.002130807652336366, + "p_value_ratio": 0.7102692174454553, + "score": 17.2829982488662 + }, + "generation": [ + 7, + 3, + 13, + 2, + 5, + 1, + 4, + 14 + ], + "resolved": { + "derived_keys": { + "key_a": "3490214442843822077", + "key_b": "7720047515392181251" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "7720047515392181251", + "p_value_threshold": 0.003, + "prf_key": "3490214442843822077", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 1, + "key_b_routed": 1, + "partial_context_skips": 0, + "routing_draws": 8 + } + }, + "dual-key-gumbel-philox-keymax-cw16-alpha1": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 16, + "detection_alpha": 1.0, + "detection_deduplicate_contexts": true, + "generation_alpha": 1.0, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 2.5582243247165326e-52, + "p_value_ratio": 2.558224324716533e-50, + "score": 270.9771333793902 + }, + "generation": [ + 45, + 3, + 73, + 41, + 47, + 40, + 21, + 85, + 80, + 70, + 34, + 48, + 58, + 47, + 96, + 28, + 56, + 77, + 101, + 30, + 97, + 103, + 96, + 53, + 87, + 37, + 34, + 113, + 33, + 25, + 103, + 113, + 102, + 6, + 101, + 37, + 29, + 75, + 46, + 67, + 105, + 48, + 115, + 51, + 114, + 4, + 0, + 81, + 98, + 35, + 115, + 111, + 7, + 78, + 21, + 52, + 112, + 48, + 19, + 47, + 68, + 17, + 106, + 73 + ], + "resolved": { + "derived_keys": { + "key_a": "11075965703501209888", + "key_b": "7399018451426338514" + }, + "detector": { + "alpha": 1.0, + "context_width": 16, + "deduplicate_contexts": true, + "key_b_prf_key": "7399018451426338514", + "p_value_threshold": 0.01, + "prf_key": "11075965703501209888", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 1.0, + "context_width": 16, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "dual-key-gumbel-philox-keymax-cw4-alpha0.4": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.4, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.4, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 5.9713149418801004e-43, + "p_value_ratio": 5.9713149418801e-41, + "score": 177.98266424643373 + }, + "generation": [ + 7, + 42, + 13, + 55, + 103, + 125, + 96, + 85, + 119, + 38, + 101, + 108, + 72, + 8, + 96, + 120, + 116, + 109, + 119, + 62, + 26, + 107, + 63, + 113, + 102, + 34, + 83, + 90, + 121, + 53, + 18, + 67, + 20, + 12, + 101, + 40, + 86, + 89, + 59, + 81, + 24, + 20, + 72, + 51, + 22, + 117, + 14, + 88, + 116, + 44, + 100, + 118, + 75, + 7, + 10, + 70, + 91, + 5, + 33, + 22, + 4, + 71, + 31, + 121 + ], + "resolved": { + "derived_keys": { + "key_a": "11075965703501209888", + "key_b": "7399018451426338514" + }, + "detector": { + "alpha": 0.4, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "7399018451426338514", + "p_value_threshold": 0.01, + "prf_key": "11075965703501209888", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.4, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 26, + "key_b_routed": 26, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-repeated-cw1": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 13, + "p_value": 0.002750266035191538, + "p_value_ratio": 0.2750266035191538, + "score": 23.896456918277963 + }, + "generation": [ + 3, + 3, + 6, + 9, + 12, + 3, + 3, + 3, + 3, + 10, + 13, + 2, + 1, + 1, + 3, + 3, + 3, + 6, + 2, + 5, + 12, + 8, + 4, + 14, + 3, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 10, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 6, + 13, + 9, + 5, + 1, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 35, + "key_b_effective": 1, + "key_b_routed": 4, + "partial_context_skips": 0, + "routing_draws": 48 + } + }, + "dual-key-gumbel-philox-repeated-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 47, + "p_value": 8.656644652657153e-07, + "p_value_ratio": 8.656644652657152e-05, + "score": 82.86886159792212 + }, + "generation": [ + 3, + 3, + 6, + 2, + 3, + 1, + 11, + 0, + 14, + 10, + 5, + 15, + 5, + 3, + 7, + 7, + 3, + 15, + 9, + 12, + 11, + 3, + 14, + 3, + 3, + 6, + 2, + 5, + 8, + 11, + 3, + 3, + 10, + 13, + 2, + 12, + 1, + 3, + 3, + 3, + 13, + 13, + 9, + 12, + 3, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": 4, + "key_b_routed": 4, + "partial_context_skips": 4, + "routing_draws": 48 + } + }, + "gumbel-philox-key0-cw1": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 54, + "p_value": 4.532248480235615e-47, + "p_value_ratio": 4.532248480235615e-45, + "score": 236.2768211235414 + }, + "generation": [ + 24, + 42, + 66, + 26, + 51, + 107, + 36, + 25, + 63, + 24, + 6, + 48, + 65, + 124, + 117, + 85, + 45, + 112, + 48, + 101, + 19, + 22, + 89, + 88, + 63, + 77, + 115, + 104, + 65, + 61, + 32, + 20, + 95, + 98, + 30, + 58, + 86, + 50, + 39, + 113, + 122, + 62, + 69, + 40, + 29, + 92, + 113, + 120, + 52, + 112, + 108, + 1, + 25, + 96, + 3, + 95, + 84, + 65, + 76, + 3, + 68, + 71, + 81, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "0", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 10, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key2p32-cw5-no-dedup": { + "configuration": { + "detection_key": "4294967296", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "4294967296", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 5, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": false, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "none", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.1027575381260964e-49, + "p_value_ratio": 1.1027575381260964e-47, + "score": 263.04926919645095 + }, + "generation": [ + 21, + 10, + 73, + 94, + 19, + 61, + 85, + 42, + 95, + 116, + 105, + 47, + 58, + 36, + 74, + 42, + 95, + 70, + 34, + 37, + 125, + 29, + 64, + 85, + 70, + 73, + 44, + 90, + 47, + 107, + 25, + 70, + 98, + 41, + 1, + 51, + 8, + 57, + 99, + 88, + 80, + 119, + 55, + 15, + 121, + 43, + 124, + 56, + 34, + 87, + 115, + 65, + 4, + 110, + 53, + 77, + 58, + 126, + 19, + 72, + 64, + 113, + 109, + 123 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 5, + "deduplicate_contexts": false, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "4294967296", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 5, + "deduplicate_contexts": "none", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 7.128196280874065e-61, + "p_value_ratio": 7.128196280874065e-59, + "score": 296.27295787856264 + }, + "generation": [ + 46, + 24, + 45, + 80, + 14, + 26, + 22, + 124, + 21, + 45, + 16, + 30, + 5, + 89, + 0, + 10, + 27, + 70, + 80, + 62, + 1, + 11, + 117, + 53, + 10, + 45, + 94, + 37, + 43, + 22, + 78, + 49, + 119, + 9, + 22, + 118, + 82, + 75, + 32, + 24, + 84, + 27, + 122, + 66, + 78, + 64, + 124, + 17, + 91, + 12, + 51, + 15, + 75, + 64, + 127, + 45, + 112, + 15, + 83, + 114, + 121, + 109, + 24, + 77 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.0283750962334736e-55, + "p_value_ratio": 6.028375096233473e-53, + "score": 278.8159437407263 + }, + "generation": [ + 46, + 42, + 38, + 34, + 122, + 33, + 43, + 117, + 24, + 52, + 29, + 115, + 12, + 32, + 121, + 124, + 81, + 45, + 112, + 16, + 1, + 100, + 11, + 46, + 95, + 77, + 80, + 5, + 53, + 114, + 64, + 67, + 37, + 52, + 62, + 125, + 107, + 75, + 39, + 28, + 126, + 20, + 23, + 61, + 54, + 3, + 116, + 10, + 59, + 27, + 58, + 104, + 124, + 2, + 28, + 38, + 91, + 101, + 118, + 103, + 46, + 110, + 70, + 45 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 4, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-all-prompt": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [ + 46, + 42, + 38, + 34, + 122 + ], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 4.330336079473992e-56, + "p_value_ratio": 4.3303360794739913e-54, + "score": 282.2076506349452 + }, + "generation": [ + 46, + 42, + 38, + 34, + 30, + 43, + 121, + 123, + 91, + 17, + 66, + 58, + 54, + 103, + 28, + 39, + 3, + 34, + 58, + 23, + 21, + 29, + 57, + 9, + 66, + 91, + 101, + 58, + 85, + 67, + 18, + 102, + 123, + 112, + 2, + 37, + 100, + 32, + 35, + 67, + 116, + 115, + 108, + 5, + 100, + 14, + 67, + 56, + 122, + 23, + 12, + 125, + 103, + 20, + 70, + 8, + 45, + 1, + 90, + 45, + 114, + 10, + 30, + 66 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 1.6692708568346358e-08, + "p_value_ratio": 1.6692708568346358e-06, + "score": 110.3077257312174 + }, + "generation": [ + 7, + 3, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 2, + 5, + 4, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 4, + 3, + 3, + 6, + 1, + 5, + 1, + 3, + 3, + 3, + 6, + 6, + 2, + 3, + 1, + 4, + 7, + 3, + 6, + 2, + 5, + 5, + 0, + 0, + 7, + 3, + 6, + 2, + 5, + 1, + 4, + 2, + 3, + 3, + 2, + 5, + 4, + 1, + 4, + 7, + 3, + 6 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 10, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-history-none": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 1.3019898400616044e-08, + "p_value_ratio": 1.3019898400616043e-06, + "score": 110.81134114351678 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 1, + 4, + 7, + 7, + 3, + 6, + 2, + 5, + 1, + 4, + 0, + 3, + 6 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 6, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-history8": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 2.772032243308163e-09, + "p_value_ratio": 2.772032243308163e-07, + "score": 113.89758610123921 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 7, + 3, + 2, + 2, + 5, + 4, + 4, + 0, + 3, + 2 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-no-dedup": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": false, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "none", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 2.136704912577492e-10, + "p_value_ratio": 2.136704912577492e-08, + "score": 127.21876922102206 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 7, + 3, + 2, + 2, + 5, + 4, + 4, + 0, + 3, + 2 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": false, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "none", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-near-threshold": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 8, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.0015 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 8, + "p_value": 0.0009697880157637389, + "p_value_ratio": 0.6465253438424926, + "score": 19.672172609864035 + }, + "generation": [ + 7, + 2, + 13, + 2, + 5, + 1, + 11, + 14 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.0015, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-wrong-key": { + "configuration": { + "detection_key": "43", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": false, + "num_scored_tokens": 64, + "p_value": 0.5208447936891313, + "p_value_ratio": 52.08447936891313, + "score": 63.25096716585621 + }, + "generation": [ + 46, + 24, + 45, + 80, + 14, + 26, + 22, + 124, + 21, + 45, + 16, + 30, + 5, + 89, + 0, + 10, + 27, + 70, + 80, + 62, + 1, + 11, + 117, + 53, + 10, + 45, + 94, + 37, + 43, + 22, + 78, + 49, + 119, + 9, + 22, + 118, + 82, + 75, + 32, + 24, + 84, + 27, + 122, + 66, + 78, + 64, + 124, + 17, + 91, + 12, + 51, + 15, + 75, + 64, + 127, + 45, + 112, + 15, + 83, + 114, + 121, + 109, + 24, + 77 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "43", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-keymax-cw16-all": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 16, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.166345567165398e-36, + "p_value_ratio": 6.1663455671653984e-34, + "score": 220.2767573421712 + }, + "generation": [ + 46, + 42, + 38, + 34, + 30, + 26, + 22, + 18, + 14, + 10, + 6, + 2, + 44, + 125, + 121, + 117, + 88, + 6, + 97, + 12, + 86, + 121, + 0, + 106, + 98, + 119, + 101, + 36, + 47, + 110, + 25, + 24, + 102, + 6, + 101, + 37, + 114, + 50, + 53, + 127, + 109, + 80, + 69, + 5, + 96, + 60, + 14, + 42, + 38, + 58, + 118, + 79, + 4, + 110, + 42, + 102, + 66, + 80, + 72, + 33, + 78, + 106, + 67, + 52 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 16, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "18446744073709551615", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 16, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 16, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 16, + "routing_draws": 0 + } + }, + "gumbel-philox-repeated-cw1": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 15, + "p_value": 6.97501804959627e-05, + "p_value_ratio": 0.00697501804959627, + "score": 34.40948699058768 + }, + "generation": [ + 14, + 3, + 13, + 2, + 1, + 8, + 3, + 3, + 3, + 10, + 2, + 2, + 5, + 15, + 11, + 3, + 3, + 6, + 2, + 5, + 12, + 11, + 3, + 3, + 3, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 10, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 6, + 13, + 9, + 4, + 11, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 33, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-repeated-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 47, + "p_value": 6.687298332875021e-08, + "p_value_ratio": 6.687298332875021e-06, + "score": 92.47607252164309 + }, + "generation": [ + 3, + 3, + 6, + 2, + 4, + 1, + 3, + 3, + 3, + 10, + 2, + 2, + 3, + 8, + 4, + 3, + 3, + 6, + 12, + 5, + 8, + 4, + 11, + 10, + 3, + 6, + 9, + 5, + 8, + 11, + 3, + 3, + 10, + 6, + 1, + 12, + 3, + 3, + 3, + 3, + 6, + 9, + 5, + 5, + 8, + 0, + 0, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + } + }, + "schema_version": 3 +} diff --git a/scripts/autotune_helion_kernels.py b/tools/autotune_helion_kernels.py similarity index 97% rename from scripts/autotune_helion_kernels.py rename to tools/autotune_helion_kernels.py index e6352a40af..8b63b002c5 100644 --- a/scripts/autotune_helion_kernels.py +++ b/tools/autotune_helion_kernels.py @@ -5,19 +5,19 @@ Usage: # Autotune all registered kernels - python scripts/autotune_helion_kernels.py + python tools/autotune_helion_kernels.py # Autotune specific kernel - python scripts/autotune_helion_kernels.py --kernels silu_mul_fp8 + python tools/autotune_helion_kernels.py --kernels silu_mul_fp8 # Autotune multiple kernels - python scripts/autotune_helion_kernels.py --kernels silu_mul_fp8 rms_norm_fp8 + python tools/autotune_helion_kernels.py --kernels silu_mul_fp8 rms_norm_fp8 # Force re-autotuning - python scripts/autotune_helion_kernels.py --force + python tools/autotune_helion_kernels.py --force # List available kernels - python scripts/autotune_helion_kernels.py --list + python tools/autotune_helion_kernels.py --list """ import argparse diff --git a/scripts/benchmark_helion_kernels.py b/tools/benchmark_helion_kernels.py similarity index 97% rename from scripts/benchmark_helion_kernels.py rename to tools/benchmark_helion_kernels.py index 04350896af..2c0a5e976d 100644 --- a/scripts/benchmark_helion_kernels.py +++ b/tools/benchmark_helion_kernels.py @@ -22,25 +22,25 @@ Two baselines are supported (``--baseline``): Usage: # List available kernels - python scripts/benchmark_helion_kernels.py --list + python tools/benchmark_helion_kernels.py --list # Benchmark a kernel against the autotune baseline (default) - python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant + python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant # Benchmark against the CUDA baseline - python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ + python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ --baseline cuda # Check numerics against the performance baseline instead of eager - python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ + python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ --baseline cuda --numerics-with-perf-baseline # Disable CUDA graph capture and save results - python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ + python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ --no-cudagraph --output results.json # Only verify numerics, skipping the timing runs - python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ + python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ --numerics-only """ diff --git a/build_rust.sh b/tools/build_rust.sh similarity index 91% rename from build_rust.sh rename to tools/build_rust.sh index 1efc1ce39f..eaf4868972 100755 --- a/build_rust.sh +++ b/tools/build_rust.sh @@ -1,13 +1,13 @@ #!/bin/bash # Build vLLM Rust artifacts and install them into the vllm package. -# Usage: ./build_rust.sh [--debug] +# Usage: ./tools/build_rust.sh [--debug] # # By default builds in release mode. Pass --debug for faster compile times # during development. set -euo pipefail -REPO_ROOT="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # Read the required toolchain from rust-toolchain.toml. TOOLCHAIN=$(grep '^channel' "$REPO_ROOT/rust-toolchain.toml" | sed 's/.*= *"\(.*\)"/\1/') diff --git a/.markdownlint.yaml b/tools/pre_commit/.markdownlint.yaml similarity index 100% rename from .markdownlint.yaml rename to tools/pre_commit/.markdownlint.yaml diff --git a/.shellcheckrc b/tools/pre_commit/.shellcheckrc similarity index 100% rename from .shellcheckrc rename to tools/pre_commit/.shellcheckrc diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py index f65b9ec921..126d20094a 100644 --- a/tools/pre_commit/check_forbidden_imports.py +++ b/tools/pre_commit/check_forbidden_imports.py @@ -13,6 +13,16 @@ _HF_NAMES = ( r"|list_repo_refs|repo_exists" ) +# Transformers v4 names kept as aliases in v5. +_TRANSFORMERS_LEGACY_NAMES = ( + r"PretrainedConfig|PreTrainedTokenizer|PreTrainedTokenizerFast" + r"|BaseImageProcessorFast|\w+ImageProcessorFast" +) +_TRANSFORMERS_LEGACY_MODULES = ( + r"tokenization_utils|tokenization_utils_fast|image_processing_utils_fast" + r"|models\.\w+\.\w+_fast" +) + @dataclass class ForbiddenImport: @@ -117,6 +127,26 @@ CHECK_IMPORTS = { allowed_files={"vllm/transformers_utils/repo_utils.py"}, allowed_dirs={"examples/"}, ), + "transformers legacy names": ForbiddenImport( + pattern=( + r"^\s*from\s+transformers(?:\.[\w.]+)?\s+import\s*\([^)]*\b(?:" + + _TRANSFORMERS_LEGACY_NAMES + + r")\b" + r"|" + r"^\s*from\s+transformers(?:\.[\w.]+)?\s+import\b[^\n]*\b(?:" + + _TRANSFORMERS_LEGACY_NAMES + + r")\b" + r"|" + r"^\s*(?:from|import)\s+transformers\.(?:" + + _TRANSFORMERS_LEGACY_MODULES + + r")\b" + ), + tip=( + "Use the Transformers v5 names, e.g. PreTrainedConfig, PythonBackend, " + "TokenizersBackend, TorchvisionBackend, tokenization_utils_tokenizers, " + "tokenization_utils_sentencepiece and image_processing_backends." + ), + ), } @@ -247,6 +277,46 @@ def test_regex(): f"(expected {should_match}, got {result})" ) + transformers_cases = [ + # Should match + ("from transformers import PretrainedConfig", True), + ("from transformers import AutoConfig, PretrainedConfig", True), + ("from transformers import PreTrainedTokenizer", True), + ("from transformers import PreTrainedTokenizerFast", True), + ("from transformers.models.siglip import SiglipImageProcessorFast", True), + ("from transformers import (\n PreTrainedTokenizerFast,\n)", True), + ("from transformers.configuration_utils import PretrainedConfig", True), + (" from transformers import PretrainedConfig", True), + ("from transformers.tokenization_utils import AddedToken", True), + ("from transformers.tokenization_utils_fast import X", True), + ("from transformers.image_processing_utils_fast import X", True), + ( + "from transformers.models.qwen2_vl.image_processing_qwen2_vl_fast import X", + True, + ), + ("import transformers.tokenization_utils_fast", True), + # Should not match + ("from transformers import PreTrainedConfig", False), + ("from transformers import PreTrainedTokenizerBase", False), + ("from transformers import PythonBackend, TokenizersBackend", False), + ("from transformers.tokenization_utils_base import BatchEncoding", False), + ("from transformers.tokenization_utils_tokenizers import X", False), + ( + "from transformers.image_processing_backends import TorchvisionBackend", + False, + ), + ("from transformers.models.qwen2_vl import Qwen2VLImageProcessor", False), + ("from vllm.foo import GLM4VImageProcessorFast", False), + ("from transformers import (\n PreTrainedConfig,\n)", False), + ("# from transformers import PretrainedConfig", False), + ] + for i, (content, should_match) in enumerate(transformers_cases): + result = matches("transformers legacy names", content) + assert result == should_match, ( + f"transformers case {i} failed: {content!r} " + f"(expected {should_match}, got {result})" + ) + print("All regex tests passed.") diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 71dd929d37..f9d064a499 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[qQ]", r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", diff --git a/use_existing_torch.py b/tools/use_existing_torch.py similarity index 100% rename from use_existing_torch.py rename to tools/use_existing_torch.py diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index c0b90e0be6..0e89b39818 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail( _OPS_REGISTERED = False +_AITER_SITUV2_ACT_ENV = { + "a8w4": "AITER_SITUV2_A8W4", + "a4w4": "AITER_SITUV2_A4W4", +} + + +def _resolve_situv2_activation() -> str: + """VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4. + + auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4. + """ + value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower() + if value in ("auto", "1"): + return "a4w4" + if value == "0": + return "a16w4" + return value + + def _sync_aiter_situv2_moe_env() -> None: - """Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env. + """Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2. - AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 / - AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first). - When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4 - (afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override. - - Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4 - dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs - afp4 FlyDSL but falls back to heuristic configs, which is not the - tuned a4w4 path this recipe is meant to use. + AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4, + so exactly one is set for a8w4/a4w4 and both are cleared for a16w4. """ import os - import vllm.envs as envs - - if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2: - os.environ["AITER_SITUV2_A4W4"] = "1" - os.environ.pop("AITER_SITUV2_A8W4", None) - else: - os.environ.pop("AITER_SITUV2_A4W4", None) + selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation()) + for name in _AITER_SITUV2_ACT_ENV.values(): + if name == selected: + os.environ[name] = "1" + else: + os.environ.pop(name, None) class rocm_aiter_ops: @@ -1917,7 +1928,8 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. - VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). + VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation + dtype (a16w4 | a8w4 | a4w4). VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: @@ -1987,7 +1999,7 @@ class rocm_aiter_ops: # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - _MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + _MOE_SITUV2 = _resolve_situv2_activation() # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM # Lazily probed: whether aiter.topk_softmax supports the @@ -2016,9 +2028,10 @@ class rocm_aiter_ops: cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + cls._MOE_SITUV2 = _resolve_situv2_activation() _sync_aiter_situv2_moe_env() cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM + cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY @staticmethod def get_aiter_activation_type(activation_str: str) -> "ActivationType | None": @@ -2131,9 +2144,25 @@ class rocm_aiter_ops: @classmethod @if_aiter_supported def is_fused_moe_situv2_enabled(cls) -> bool: - # _MOE_SITUV2 is a variant of aiter fused moe, so aiter - # fused moe must be enabled as well. - return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 + """True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected.""" + return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4" + + @classmethod + @if_aiter_supported + def get_fused_moe_situv2_activation(cls) -> str: + """SiTUv2 activation dtype name: a16w4, a8w4 or a4w4.""" + return cls._MOE_SITUV2 + + @classmethod + @if_aiter_supported + def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool: + """Whether the SiTUv2 kernels expect gate/up-interleaved w13. + + The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and + a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode + (experts/rocm_aiter_moe.py) must both use this. + """ + return cls._MOE_SITUV2 == "a8w4" @classmethod @if_aiter_supported @@ -3362,6 +3391,115 @@ class rocm_aiter_ops: rotary_dim=kernel_rotary_dim, ) + @staticmethod + def fused_qk_norm_mrope_and_cache( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + is_neox: bool, + mrope_section: list[int], + is_interleaved: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + block_size: int, + x: int, + rotary_dim: int = 0, + ) -> None: + from aiter.ops.fused_qk_norm_mrope_cache_quant import ( + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle, + ) + + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( + qkv, + q_weight, + k_weight, + cos_sin_cache, + positions, + qkv.size(0), + num_heads_q, + num_heads_k, + num_heads_k, + head_dim, + is_neox, + mrope_section, + is_interleaved, + rms_norm_eps, + q_out, + k_cache, + v_cache, + slot_mapping, + k_scale, + v_scale, + None, + None, + False, + False, + block_size, + x, + rotary_dim, + ) + + @staticmethod + def do_qk_norm_mrope_kvcache_update( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + is_neox: bool, + mrope_section: list[int], + is_interleaved: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + kv_cache_dtype: str, + rotary_dim: int = 0, + ) -> None: + if kv_cache_dtype.startswith("fp8"): + key_cache = key_cache.view(current_platform.fp8_dtype()) + value_cache = value_cache.view(current_platform.fp8_dtype()) + + rocm_aiter_ops.fused_qk_norm_mrope_and_cache( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=num_heads_q, + num_heads_k=num_heads_k, + head_dim=head_dim, + is_neox=is_neox, + mrope_section=mrope_section, + is_interleaved=is_interleaved, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + k_cache=key_cache, + v_cache=value_cache, + slot_mapping=slot_mapping, + k_scale=k_scale, + v_scale=v_scale, + block_size=key_cache.shape[1], + x=16 // key_cache.element_size(), + rotary_dim=rotary_dim, + ) + @staticmethod def triton_rope_and_cache( query: torch.Tensor, @@ -3406,7 +3544,7 @@ class rocm_aiter_ops: X: torch.Tensor, W: torch.Tensor, w_scale: torch.Tensor, - Y: torch.Tensor, + Y: torch.Tensor | None = None, transpose_bm: bool | None = False, prequant: bool | None = False, y_scale: torch.Tensor | None = None, diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index dcecce71b3..be9a111a58 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -732,26 +732,6 @@ def moe_gptq_gemm_rdna3( ) -if hasattr(torch.ops._C, "allspark_w8a16_gemm"): - - @register_fake("_C::allspark_w8a16_gemm") - def _allspark_w8a16_gemm_fake( - a: torch.Tensor, - b_qweight: torch.Tensor, - b_scales: torch.Tensor, - b_qzeros: torch.Tensor | None, - n: torch.SymInt, - group_size: torch.SymInt, - sm_count: torch.SymInt, - sm_version: torch.SymInt, - CUBLAS_M_THRESHOLD: torch.SymInt, - has_zp: bool, - n32k16_reorder: bool, - ) -> torch.Tensor: - m = a.size(0) - return torch.empty((m, n), device=a.device, dtype=a.dtype) - - # cutlass def cutlass_scaled_mm_supports_fp4(cuda_device_capability: int) -> bool: return torch.ops._C.cutlass_scaled_mm_supports_fp4(cuda_device_capability) @@ -1911,90 +1891,6 @@ def scaled_fp8_quant( return output, scale -# gptq allspark -def allspark_repack_weight( - qweight: torch.Tensor, - scale: torch.Tensor, - zero_point: torch.Tensor | None = None, - has_zp: bool = False, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Rearrange qweight, scale, and zero_point(if asymmetric) to n32k16 format - for Ampere W8A16 Fused Gemm kernel - - Args: - qweight: uint8 weight tensor, original k x n format. - scale: fp16/bf16 weight scale tensor, 1 x n format. - zero_point: fp16/bf16 weight zero_point tensor, 1 x n format. - Must be provided for asymmetric quantization. - has_zp: if use symmetric quantization, has_zp = False. - if use asymmetric quantization, has_zp = True. - - Returns: - tuple[torch.Tensor, torch.Tensor, torch.Tensor | None] : - rearranged weight, scale, and optionally zero_point. - - """ - K = qweight.shape[0] - N = qweight.shape[1] - N_32align = (N + 32 - 1) // 32 * 32 - - qweight_reorder = torch.empty( - (N_32align, K), device=qweight.device, dtype=qweight.dtype - ) - scale_reorder = torch.empty((1, N_32align), device=scale.device, dtype=scale.dtype) - zero_point_reorder = None - if has_zp: - assert zero_point is not None, ( - "zero_point must be provided for asymmetric quantization." - ) - zero_point_reorder = torch.empty( - (1, N_32align), device=zero_point.device, dtype=zero_point.dtype - ) - - torch.ops._C.rearrange_kn_weight_as_n32k16_order( - qweight, - scale, - zero_point, - has_zp, - qweight_reorder, - scale_reorder, - zero_point_reorder, - K, - N, - N_32align, - ) - - return qweight_reorder, scale_reorder, zero_point_reorder - - -def allspark_w8a16_gemm( - a: torch.Tensor, - b_qweight: torch.Tensor, - b_scales: torch.Tensor, - b_qzeros: torch.Tensor | None, - n: int, - group_size: int, - sm_count: int, - sm_version: int, - CUBLAS_M_THRESHOLD: int, - has_zp: bool, - n32k16_reorder: bool, -) -> torch.Tensor: - return torch.ops._C.allspark_w8a16_gemm( - a, - b_qweight, - b_scales, - b_qzeros, - n, - group_size, - sm_count, - sm_version, - CUBLAS_M_THRESHOLD, - has_zp, - n32k16_reorder, - ) - - # int8 def scaled_int8_quant( input: torch.Tensor, @@ -3263,6 +3159,26 @@ def mnnvl_lamport_reduce_scatter( ) +def mnnvl_multimem_reduce_scatter( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + local_buffer: int, + multicast_buffer: int, + stage_sz_bytes: int, + block_limit: int, +) -> None: + torch.ops._C_custom_ar.mnnvl_multimem_reduce_scatter( + fa, + inp, + out, + local_buffer, + multicast_buffer, + stage_sz_bytes, + block_limit, + ) + + def dispose(fa: int) -> None: torch.ops._C_custom_ar.dispose(fa) diff --git a/vllm/_xpu_ops.py b/vllm/_xpu_ops.py index 5c19b01408..d25ef6b258 100644 --- a/vllm/_xpu_ops.py +++ b/vllm/_xpu_ops.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import torch from vllm_xpu_kernels.flash_attn_interface import flash_attn_varlen_func +from vllm_xpu_kernels.rotary import apply_rotary_emb from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.logger import init_logger @@ -249,6 +250,24 @@ def _xpu_ops_deepseek_scaling_rope_fake( return query, key +def _xpu_apply_rotary_emb_impl( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + return apply_rotary_emb(x, cos, sin, is_neox_style) + + +def _xpu_apply_rotary_emb_fake( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + return torch.empty_like(x) + + def _xpu_fp8_bmm_impl( a: torch.Tensor, b: torch.Tensor, @@ -1257,6 +1276,12 @@ class xpu_ops: dispatch_key=current_platform.dispatch_key, ) + direct_register_custom_op( + op_name="xpu_apply_rotary_emb", + op_func=_xpu_apply_rotary_emb_impl, + fake_impl=_xpu_apply_rotary_emb_fake, + ) + direct_register_custom_op( op_name="xpu_mxfp8_quantize", op_func=_xpu_mxfp8_quantize_impl, diff --git a/vllm/benchmarks/latency.py b/vllm/benchmarks/latency.py index 6209f5f2d8..acb3efef15 100644 --- a/vllm/benchmarks/latency.py +++ b/vllm/benchmarks/latency.py @@ -169,6 +169,7 @@ def main(args: argparse.Namespace): # Output JSON results if specified if args.output_json: results = { + "model_id": args.model, "avg_latency": np.mean(latencies), "latencies": latencies.tolist(), "percentiles": dict(zip(percentages, percentiles.tolist())), diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 14c72386a5..d751f7f22d 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1695,7 +1695,6 @@ def add_cli_args(parser: FlexibleArgumentParser): - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer.\n - "hf" will use the fast tokenizer if available.\n - - "slow" will always use the slow tokenizer.\n - "mistral" will always use the tokenizer from `mistral_common`.\n - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n - Other custom values can be supported via plugins.""", diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py index 94b22b2956..620ce2b08f 100644 --- a/vllm/benchmarks/throughput.py +++ b/vllm/benchmarks/throughput.py @@ -1141,6 +1141,7 @@ def main(args: argparse.Namespace): # Output JSON results if specified if args.output_json: results = { + "model_id": args.model, "elapsed_time": elapsed_time, "num_requests": len(requests), "total_num_tokens": total_num_tokens, diff --git a/vllm/compilation/breakable_cudagraph.py b/vllm/compilation/breakable_cudagraph.py index ca6aa17137..781868804e 100644 --- a/vllm/compilation/breakable_cudagraph.py +++ b/vllm/compilation/breakable_cudagraph.py @@ -25,6 +25,7 @@ from __future__ import annotations import dataclasses import functools import gc +import logging import threading import weakref from collections.abc import Callable @@ -285,7 +286,7 @@ class BreakableCUDAGraphWrapper: self.runtime_mode = runtime_mode self.compilation_config = vllm_config.compilation_config self.graph_pool = current_platform.get_global_graph_pool() - self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG" + self.is_debugging_mode = logger.isEnabledFor(logging.DEBUG) self.entries: dict[BatchDescriptor, _BreakableEntry] = {} BreakableCUDAGraphWrapper._all_instances.add(self) diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index 67aff0fe1e..394799ab87 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -23,6 +23,8 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer logger = init_logger(__name__) +_last_inductor_triton_cache_dir: str | None = None + class CompilerInterface: """The interface for a compiler that can be used by vLLM.""" @@ -453,19 +455,22 @@ class InductorAdaptor(CompilerInterface): def initialize_cache( self, cache_dir: str, disable_cache: bool = False, prefix: str = "" ) -> None: + global _last_inductor_triton_cache_dir self.cache_dir = cache_dir self.prefix = prefix self.base_cache_dir = cache_dir[: -len(prefix)] if prefix else cache_dir if disable_cache: return - # redirect the cache directory to a subdirectory - # set flags so that Inductor and Triton store their cache - # in the cache_dir, then users only need to copy the cache_dir - # to another machine to reuse the cache. + # Keep artifacts together unless the caller supplies a Triton cache. inductor_cache = os.path.join(self.base_cache_dir, "inductor_cache") os.makedirs(inductor_cache, exist_ok=True) os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache - triton_cache = os.path.join(self.base_cache_dir, "triton_cache") + triton_cache = os.environ.get("TRITON_CACHE_DIR") + # A directory assigned by an earlier adaptor is still a model-local + # default, rather than a caller override for subsequent models. + if triton_cache is None or triton_cache == _last_inductor_triton_cache_dir: + triton_cache = os.path.join(self.base_cache_dir, "triton_cache") + _last_inductor_triton_cache_dir = triton_cache os.makedirs(triton_cache, exist_ok=True) os.environ["TRITON_CACHE_DIR"] = triton_cache diff --git a/vllm/compilation/cuda_graph.py b/vllm/compilation/cuda_graph.py index 1306cd612f..dc8a8a9c84 100644 --- a/vllm/compilation/cuda_graph.py +++ b/vllm/compilation/cuda_graph.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import dataclasses +import logging import weakref from collections import Counter from collections.abc import Callable @@ -11,7 +12,6 @@ from unittest.mock import patch import torch -import vllm.envs as envs from vllm.compilation.counter import compilation_counter from vllm.compilation.monitor import validate_cudagraph_capturing_enabled from vllm.config import CUDAGraphMode, VllmConfig @@ -164,7 +164,7 @@ class CUDAGraphWrapper: assumption on the dynamic shape (batch size) of the runtime inputs, as a trade-off for staying orthogonal to compilation logic. Nevertheless, tracing and checking the input addresses to be consistent during replay is - guaranteed when VLLM_LOGGING_LEVEL == "DEBUG". + guaranteed when vLLM debug logging is enabled. """ _all_instances: ClassVar[weakref.WeakSet["CUDAGraphWrapper"]] = weakref.WeakSet() @@ -188,7 +188,7 @@ class CUDAGraphWrapper: self.compilation_config = vllm_config.compilation_config self.first_run_finished = False - self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG" + self.is_debugging_mode = logger.isEnabledFor(logging.DEBUG) self._runnable_str = str(runnable) if self.is_debugging_mode else None # assert runtime_mode is not NONE(no cudagraph), otherwise, we don't diff --git a/vllm/compilation/passes/fusion/attn_quant_fusion.py b/vllm/compilation/passes/fusion/attn_quant_fusion.py index 432d59e5ab..bdbca79e5d 100644 --- a/vllm/compilation/passes/fusion/attn_quant_fusion.py +++ b/vllm/compilation/passes/fusion/attn_quant_fusion.py @@ -7,6 +7,7 @@ from collections.abc import Callable import torch from torch._higher_order_ops.auto_functionalize import auto_functionalized +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -35,7 +36,9 @@ RESHAPE_OP = torch.ops.aten.reshape.default _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=True) -class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): +class AttnFp8StaticQuantPattern( + VllmPatternReplacement[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]] +): """Fusion for Attention+Fp8StaticQuant. Only triggers when the attention implementation returns True in @@ -51,8 +54,20 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): self._dtype = dtype self._quant_matcher = MatcherQuantFP8(_FP8_QUANT_KEY) + def _quantize( + self, value: torch.Tensor, scale: torch.Tensor + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return self._quant_matcher(value, scale)[0] + + def _output( + self, value: torch.Tensor, scale: torch.Tensor + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return value + @property - def pattern(self) -> Callable[..., torch.Tensor]: + def pattern( + self, + ) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]: # When _USE_LAYERNAME is enabled (torch >= 2.11), layer_name is # passed as an explicit pattern input so the pattern matcher # treats it as a wildcard matching hoisted LayerName placeholders. @@ -78,7 +93,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): attn_out_view = RESHAPE_OP( at1[1], [q.shape[0], self._num_heads * self._head_size] ) - return self._quant_matcher(attn_out_view, scale)[0] + return self._quantize(attn_out_view, scale) return _pattern_with_ln @@ -97,12 +112,14 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): attn_out_view = RESHAPE_OP( at1[1], [q.shape[0], self._num_heads * self._head_size] ) - return self._quant_matcher(attn_out_view, scale)[0] + return self._quantize(attn_out_view, scale) return _pattern @property - def replacement(self) -> Callable[..., torch.Tensor]: + def replacement( + self, + ) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]: _ln = _encode_layer_name(self._layer_name) if _USE_LAYERNAME: @@ -126,7 +143,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]) + return self._output( + RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale + ) return _replacement_with_ln @@ -147,7 +166,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]) + return self._output( + RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale + ) return _replacement @@ -168,6 +189,30 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): return inputs +class RocmAttnFp8StaticQuantPattern(AttnFp8StaticQuantPattern): + """Fuse AITER static quantization while preserving its scale output.""" + + def _quantize( + self, value: torch.Tensor, scale: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + out = torch.empty_like(value, dtype=FP8_DTYPE) + _, out, scale = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=out, + x=value, + scale=scale, + is_dynamic=False, + ) + # The shared AITER op declares scale mutable for its dynamic variant. + # Static quant preserves it; keep this graph output when replacing it. + return out, scale + + def _output( + self, value: torch.Tensor, scale: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + return value, scale + + class AttnNvfp4QuantPattern( VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]] ): @@ -387,6 +432,8 @@ class AttnQuantFusionPass(VllmFusionPatternMatcherPass): for layer in layers: if layer.impl.fused_output_quant_supported(_FP8_QUANT_KEY): self.register(AttnFp8StaticQuantPattern(layer, dtype)) + if current_platform.is_rocm() and rocm_aiter_ops.is_enabled(): + self.register(RocmAttnFp8StaticQuantPattern(layer, dtype)) if _USE_LAYERNAME: break diff --git a/vllm/compilation/passes/fusion/collective_fusion.py b/vllm/compilation/passes/fusion/collective_fusion.py index 7a413665ff..f2aa6d7d2f 100644 --- a/vllm/compilation/passes/fusion/collective_fusion.py +++ b/vllm/compilation/passes/fusion/collective_fusion.py @@ -339,6 +339,8 @@ class BasePattern: class GEMMReduceScatterPattern(BasePattern): + rocm_gemm = False + def get_inputs(self) -> list[torch.Tensor]: mul = torch.empty([16, 4], device=self.device, dtype=self.dtype) mm_weight = torch.empty([4, 4], device=self.device, dtype=self.dtype) @@ -346,7 +348,10 @@ class GEMMReduceScatterPattern(BasePattern): def register(self, pm_pass: PatternMatcherPass) -> None: def pattern(mul: torch.Tensor, mm_weight: torch.Tensor) -> torch.Tensor: - mm = torch.ops.aten.mm.default(mul, mm_weight) + if self.rocm_gemm: + mm = torch.ops.vllm.rocm_unquantized_gemm(mul, mm_weight, None) + else: + mm = torch.ops.aten.mm.default(mul, mm_weight) reduce_scatter = torch.ops.vllm.reduce_scatter.default( mm, dim=0, @@ -358,7 +363,7 @@ class GEMMReduceScatterPattern(BasePattern): def replacement(mul: torch.Tensor, mm_weight: torch.Tensor) -> torch.Tensor: gemm_rs = torch.ops.symm_mem.fused_matmul_reduce_scatter( mul, - mm_weight, + mm_weight.t() if self.rocm_gemm else mm_weight, "sum", scatter_dim=0, group_name=self.tp.device_group.group_name, @@ -371,7 +376,13 @@ class GEMMReduceScatterPattern(BasePattern): ) +class RocmGEMMReduceScatterPattern(GEMMReduceScatterPattern): + rocm_gemm = True + + class AllGatherGEMMPattern(BasePattern): + rocm_gemm = False + def get_inputs(self) -> list[torch.Tensor]: x = torch.empty([4, 4], device=self.device, dtype=self.dtype) weight = torch.empty([4, 4], device=self.device, dtype=self.dtype) @@ -390,12 +401,14 @@ class AllGatherGEMMPattern(BasePattern): group_name=self.tp.unique_name, ) + if self.rocm_gemm: + return torch.ops.vllm.rocm_unquantized_gemm(all_gather, weight, None) return torch.ops.aten.mm.default(all_gather, weight) def replacement(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: ag_output, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul( x, - [weight], + [weight.t() if self.rocm_gemm else weight], gather_dim=0, group_name=self.tp.device_group.group_name, ) @@ -406,6 +419,10 @@ class AllGatherGEMMPattern(BasePattern): ) +class RocmAllGatherGEMMPattern(AllGatherGEMMPattern): + rocm_gemm = True + + class ScaledMMReduceScatterPattern(BasePattern): def get_inputs(self) -> list[torch.Tensor]: input = torch.empty([16, 16], device=self.device, dtype=FP8_DTYPE) @@ -906,6 +923,13 @@ class AsyncTPPass(VllmFusionPatternMatcherPass): GEMMReduceScatterPattern(self.model_dtype, self.device).register(self.pm_pass) AllGatherGEMMPattern(self.model_dtype, self.device).register(self.pm_pass) + if current_platform.is_rocm() and self.model_dtype == torch.bfloat16: + RocmGEMMReduceScatterPattern(self.model_dtype, self.device).register( + self.pm_pass + ) + RocmAllGatherGEMMPattern(self.model_dtype, self.device).register( + self.pm_pass + ) # These fusions are enabled only for bfloat16 models because # `scaled_mm` or `cutlass_scaled_mm` with per-token (row-wise) scaling diff --git a/vllm/compilation/passes/fusion/matcher_utils.py b/vllm/compilation/passes/fusion/matcher_utils.py index 7fd44dd005..6d959c3067 100644 --- a/vllm/compilation/passes/fusion/matcher_utils.py +++ b/vllm/compilation/passes/fusion/matcher_utils.py @@ -31,6 +31,7 @@ from vllm.platforms import current_platform ROTARY_OP = torch.ops._C.rotary_embedding.default FLASHINFER_ROTARY_OP = torch.ops.vllm.flashinfer_rotary_embedding.default +MROPE_OP = torch.ops.vllm.mrope.default QUANT_OPS: dict[QuantKey, OpOverload] = { kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501 @@ -162,6 +163,74 @@ class MatcherRotaryEmbedding(MatcherCustomOp): return result +class MatcherMRotaryEmbedding(MatcherCustomOp): + def __init__( + self, + is_neox: bool, + head_size: int, + rotary_dim: int, + num_heads: int, + num_kv_heads: int, + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + enabled: bool | None = None, + ) -> None: + if enabled is None: + enabled = RotaryEmbedding.enabled() + + super().__init__(enabled) + self.is_neox = is_neox + self.head_size = head_size + self.rotary_dim = rotary_dim + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.q_size = self.num_heads * self.head_size + self.kv_size = self.num_kv_heads * self.head_size + self.mrope_section = mrope_section + self.mrope_interleaved = mrope_interleaved + + def inputs(self) -> list[torch.Tensor]: + positions = self.empty_int64(3, 5) + query = self.empty(5, self.q_size) + key = self.empty(5, self.kv_size) + cos_sin_cache = self.empty(4096, self.rotary_dim) + return [positions, query, key, cos_sin_cache] + + def forward_custom( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + result = auto_functionalized( + MROPE_OP, + positions=positions, + query=query, + key=key, + cos_sin_cache=cos_sin_cache, + head_size=self.head_size, + rotary_dim=self.rotary_dim, + mrope_section_t=self.mrope_section[0], + mrope_section_h=self.mrope_section[1], + mrope_section_w=self.mrope_section[2], + mrope_interleaved=self.mrope_interleaved, + is_neox_style=self.is_neox, + ) + return result[1], result[2] + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError( + "MRoPE fusion requires the rotary_embedding custom op" + ) + + class MatcherRMSNormGated(MatcherCustomOp): """Matches RMSNormGated with norm_before_gate=True and group_size=None.""" diff --git a/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py index 80eeb648f4..f418e8a7ef 100644 --- a/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py +++ b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import inspect -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import ParamSpec import torch @@ -19,12 +19,19 @@ from vllm.model_executor.layers.attention.attention import ( Attention, get_attention_context, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.platforms import current_platform -from vllm.utils.torch_utils import direct_register_custom_op +from vllm.utils.torch_utils import ( + _USE_LAYERNAME, + LayerNameType, + _encode_layer_name, + _resolve_layer_name, + direct_register_custom_op, +) from ..inductor_pass import enable_fake_mode from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass -from .matcher_utils import MatcherRotaryEmbedding +from .matcher_utils import MROPE_OP, MatcherMRotaryEmbedding, MatcherRotaryEmbedding from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64 logger = init_logger(__name__) @@ -51,8 +58,9 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_impl( rms_norm_eps: float, cos_sin_cache: torch.Tensor, is_neox: bool, - layer_name: str = "", + layer_name: LayerNameType, ) -> torch.Tensor: + layer_name = _resolve_layer_name(layer_name) _, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name) if layer_slot_mapping is not None: attn_layer.impl.do_qk_norm_rope_kvcache_update( @@ -87,7 +95,7 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_fake( rms_norm_eps: float, cos_sin_cache: torch.Tensor, is_neox: bool, - layer_name: str = "", + layer_name: LayerNameType, ) -> torch.Tensor: return torch.empty(0, device=qkv.device, dtype=qkv.dtype) @@ -100,6 +108,76 @@ direct_register_custom_op( ) +def fused_qk_norm_mrope_and_unified_kv_cache_update_impl( + q_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + is_interleaved: bool, + rotary_dim: int, + layer_name: LayerNameType, +) -> torch.Tensor: + layer_name = _resolve_layer_name(layer_name) + _, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name) + if layer_slot_mapping is not None: + if positions.ndim == 1: + positions = positions.unsqueeze(0).expand(3, -1).contiguous() + attn_layer.impl.do_qk_norm_mrope_kvcache_update( + attn_layer, + qkv, + q_out, + positions, + q_weight, + k_weight, + rms_norm_eps, + cos_sin_cache, + is_neox, + (mrope_section_t, mrope_section_h, mrope_section_w), + is_interleaved, + rotary_dim, + kv_cache, + layer_slot_mapping, + ) + else: + q_out.zero_() + + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +def fused_qk_norm_mrope_and_unified_kv_cache_update_fake( + q_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + is_interleaved: bool, + rotary_dim: int, + layer_name: LayerNameType, +) -> torch.Tensor: + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +direct_register_custom_op( + op_name="fused_qk_norm_mrope_and_unified_kv_cache_update", + op_func=fused_qk_norm_mrope_and_unified_kv_cache_update_impl, + mutates_args=["q_out"], + fake_impl=fused_qk_norm_mrope_and_unified_kv_cache_update_fake, +) + + # --------------------------------------------------------------------------- # Pattern: QK-norm + RoPE + unified_kv_cache_update # --------------------------------------------------------------------------- @@ -142,19 +220,27 @@ class QkNormRopeKvCachePattern: self.eps = eps self.is_neox = is_neox self.quant_query = quant_query + self.encoded_layer_name = _encode_layer_name(self.layer_name) + self.query_quant_group_shape = ( + layer.query_quant.group_shape + if layer.query_quant is not None + else GroupShape.PER_TENSOR + ) self.q_size = self.num_heads * self.head_size self.k_size = self.num_kv_heads * self.head_size self.v_size = self.num_kv_heads * self.head_size_v - self.rope_matcher = MatcherRotaryEmbedding( - is_neox=is_neox, - head_size=self.head_size, - num_heads=self.num_heads, - num_kv_heads=self.num_kv_heads, + self.rope_matcher: MatcherRotaryEmbedding | MatcherMRotaryEmbedding = ( + MatcherRotaryEmbedding( + is_neox=is_neox, + head_size=self.head_size, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + ) ) - def get_inputs(self) -> list[torch.Tensor]: + def get_inputs(self) -> list: T = 5 L = 4096 qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) @@ -166,8 +252,13 @@ class QkNormRopeKvCachePattern: if self.quant_query: q_scale = empty_fp32(1) inputs += [q_scale] + if _USE_LAYERNAME: + inputs.append(self.encoded_layer_name) return inputs + def _get_layer_name(self, layer_name: LayerNameType | None) -> LayerNameType: + return self.encoded_layer_name if layer_name is None else layer_name + def pattern_non_fp8_quant_query( self, qkv: torch.Tensor, @@ -175,6 +266,7 @@ class QkNormRopeKvCachePattern: q_weight: torch.Tensor, k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) @@ -190,7 +282,9 @@ class QkNormRopeKvCachePattern: q_rope = q_rope.view(-1, self.num_heads, self.head_size) k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size_v) - dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + dummy = torch.ops.vllm.unified_kv_cache_update( + k_rope, v, self._get_layer_name(layer_name) + ) return dummy, q_rope, k_rope, v def replacement_non_fp8_quant_query( @@ -200,6 +294,7 @@ class QkNormRopeKvCachePattern: q_weight: torch.Tensor, k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q_out = torch.empty( qkv.shape[0], @@ -228,10 +323,49 @@ class QkNormRopeKvCachePattern: rms_norm_eps=self.eps, cos_sin_cache=cos_sin_cache, is_neox=self.is_neox, - layer_name=self.layer_name, + layer_name=self._get_layer_name(layer_name), ) return results[0], results[1], results[2], v + def _quantize_query( + self, + query: torch.Tensor, + q_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if self.query_quant_group_shape.is_per_tensor(): + q_out = torch.empty_like(query, dtype=current_platform.fp8_dtype()) + q_quant = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=q_out, + x=query, + scale=q_scale, + is_dynamic=False, + ) + # The AITER op's schema marks scale mutable, so preserve its + # functionalized write-back as an output of the matched region. + return q_quant[1], q_quant[2] + + # QuantFP8.forward_hip falls back to the generic static op for the + # historical per-head shape (-1, block_size). This also covers the + # degenerate one-local-KV-head case where a scalar scale is represented + # as (-1, q_size), rather than changing Attention's classification. + q_out = torch.empty( + query.shape, + device=query.device, + dtype=current_platform.fp8_dtype(), + ) + q_quant = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_out, + input=query, + scale=q_scale, + group_shape=[ + self.query_quant_group_shape.row, + self.query_quant_group_shape.col, + ], + ) + return q_quant[1], None + def pattern_fp8_quant_query( self, qkv: torch.Tensor, @@ -240,7 +374,8 @@ class QkNormRopeKvCachePattern: k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, q_scale: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps) @@ -251,24 +386,15 @@ class QkNormRopeKvCachePattern: k_flat = k_normed.view(-1, self.k_size) q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache) - # Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED). - # Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern. - q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype()) - q_quant = auto_functionalized( - torch.ops.vllm.rocm_aiter_per_tensor_quant.default, - out=q_out, - x=q_rope, - scale=q_scale, - is_dynamic=False, - ) - # `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation - # region, so keep q flat (a reshape lands past the barrier and won't match). - q_rope_fp8 = q_quant[1] - q_scale_out = q_quant[2] + q_rope_fp8, q_scale_out = self._quantize_query(q_rope, q_scale) k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size_v) - dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + dummy = torch.ops.vllm.unified_kv_cache_update( + k_rope, v, self._get_layer_name(layer_name) + ) + if q_scale_out is None: + return dummy, q_rope_fp8, k_rope, v return dummy, q_rope_fp8, k_rope, v, q_scale_out def replacement_fp8_quant_query( @@ -279,7 +405,8 @@ class QkNormRopeKvCachePattern: k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, q_scale: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: q_out = torch.empty( qkv.shape[0], self.num_heads, @@ -307,22 +434,14 @@ class QkNormRopeKvCachePattern: rms_norm_eps=self.eps, cos_sin_cache=cos_sin_cache, is_neox=self.is_neox, - layer_name=self.layer_name, + layer_name=self._get_layer_name(layer_name), ) - # Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q. - # Same explicit auto_functionalized form as the pattern: [1] = quantized - # q, [2] = scale (returned so the buffer-writeback use is preserved). + # Re-apply the same quant form on the kernel's bf16 q_out; the fused + # operation does not quantize Q. q_fp8_flat = results[1].view(-1, self.q_size) - q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype()) - q_requant = auto_functionalized( - torch.ops.vllm.rocm_aiter_per_tensor_quant.default, - out=q_fp8_out, - x=q_fp8_flat, - scale=q_scale, - is_dynamic=False, - ) - q_fp8 = q_requant[1] # flat to mirror the pattern (see note above) - q_scale_out = q_requant[2] + q_fp8, q_scale_out = self._quantize_query(q_fp8_flat, q_scale) + if q_scale_out is None: + return results[0], q_fp8, results[2], v return results[0], q_fp8, results[2], v, q_scale_out @staticmethod @@ -345,6 +464,9 @@ class QkNormRopeKvCachePattern: view_to_reshape(gm) + def _extra_check(self, match: pm.Match) -> bool: + return True + def _register(self, pattern, replacement, pm_pass) -> None: trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn( pm.fwd_only, @@ -370,40 +492,338 @@ class QkNormRopeKvCachePattern: inputs, trace_fn, pm_pass, + extra_check=self._extra_check, search_fn_pattern=search_fn_pattern, ) def register(self, pm_pass: PatternMatcherPass) -> None: # make_fx counts `self` in bound-method code params; wrap as plain fns. - # Distinct names per branch so mypy doesn't see one name, two signatures. if self.quant_query: + if _USE_LAYERNAME: - def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale): - return self.pattern_fp8_quant_query( + def pattern_q_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ): + return self.pattern_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ) + + def replacement_q_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ): + return self.replacement_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ) + + self._register(pattern_q_with_layer, replacement_q_with_layer, pm_pass) + else: + + def pattern_q_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ) + ): + return self.pattern_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) - def replacement_q( - qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ): - return self.replacement_fp8_quant_query( + def replacement_q_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ) + ): + return self.replacement_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) - self._register(pattern_q, replacement_q, pm_pass) + self._register( + pattern_q_without_layer, replacement_q_without_layer, pm_pass + ) else: + if _USE_LAYERNAME: - def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): - return self.pattern_non_fp8_quant_query( + def pattern_noq_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ): + return self.pattern_non_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ) + + def replacement_noq_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ): + return self.replacement_non_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ) + + self._register( + pattern_noq_with_layer, replacement_noq_with_layer, pm_pass + ) + else: + + def pattern_noq_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache + ): + return self.pattern_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + def replacement_noq_without_layer( + qkv, positions, q_weight, k_weight, cos_sin_cache + ): + return self.replacement_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + self._register( + pattern_noq_without_layer, + replacement_noq_without_layer, + pm_pass, ) - def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): - return self.replacement_non_fp8_quant_query( - qkv, positions, q_weight, k_weight, cos_sin_cache - ) - self._register(pattern_noq, replacement_noq, pm_pass) +class QkNormMRopeKvCachePattern(QkNormRopeKvCachePattern): + FUSED_OP = torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default + + def __init__( + self, + layer: Attention, + eps: float, + is_neox: bool, + quant_query: bool, + rotary_dim: int, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + ) -> None: + super().__init__( + layer=layer, + eps=eps, + is_neox=is_neox, + quant_query=quant_query, + ) + self.rotary_dim = rotary_dim + self.mrope_section = mrope_section + self.is_interleaved = is_interleaved + self.rope_matcher = MatcherMRotaryEmbedding( + is_neox=is_neox, + head_size=self.head_size, + rotary_dim=rotary_dim, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + mrope_section=mrope_section, + mrope_interleaved=is_interleaved, + ) + + def get_inputs(self) -> list[torch.Tensor]: + T = 5 + L = 4096 + qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) + positions = empty_i64(3, T) + q_weight = empty_bf16(1, self.head_size) + k_weight = empty_bf16(1, self.head_size) + cos_sin_cache = empty_bf16(L, self.rotary_dim) + inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache] + if self.quant_query: + inputs.append(empty_fp32(1)) + if _USE_LAYERNAME: + inputs.append(self.encoded_layer_name) + return inputs + + def _unfused_kv_views(self, qkv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + # The supported backend reads K/V from the cache after the fused write. + # Keep view-compatible placeholders for the attention custom-op ABI. + _, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + k = k.view(qkv.shape[0], self.num_kv_heads, self.head_size) + v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v) + return k, v + + def _fused_results( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None, + ) -> tuple[torch.Tensor, ...]: + q_out = torch.empty( + qkv.shape[0], + self.num_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + return auto_functionalized( + self.FUSED_OP, + q_out=q_out, + qkv=qkv, + positions=positions, + q_weight=q_weight, + k_weight=k_weight, + rms_norm_eps=self.eps, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + mrope_section_t=self.mrope_section[0], + mrope_section_h=self.mrope_section[1], + mrope_section_w=self.mrope_section[2], + is_interleaved=self.is_interleaved, + rotary_dim=self.rotary_dim, + layer_name=self._get_layer_name(layer_name), + ) + + def replacement_non_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + k, v = self._unfused_kv_views(qkv) + results = self._fused_results( + qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name + ) + return results[0], results[1], k, v + + def replacement_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_scale: torch.Tensor, + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: + k, v = self._unfused_kv_views(qkv) + results = self._fused_results( + qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name + ) + q_flat = results[1].view(-1, self.q_size) + q_fp8, q_scale_out = self._quantize_query(q_flat, q_scale) + if q_scale_out is None: + return results[0], q_fp8, k, v + return results[0], q_fp8, k, v, q_scale_out + + def _extra_check(self, match: pm.Match) -> bool: + expected = { + "head_size": self.head_size, + "rotary_dim": self.rotary_dim, + "mrope_section_t": self.mrope_section[0], + "mrope_section_h": self.mrope_section[1], + "mrope_section_w": self.mrope_section[2], + "mrope_interleaved": self.is_interleaved, + "is_neox_style": self.is_neox, + } + mrope_node = next( + (node for node in match.nodes if node.args and node.args[0] == MROPE_OP), + None, + ) + if mrope_node is None or any( + mrope_node.kwargs.get(name) != value for name, value in expected.items() + ): + return False + + # The fused AITER MRoPE entrypoint writes rotated K directly to the + # cache and does not expose a BF16 K result. Returning the raw K view is + # valid only for decoder unified attention, whose backend consumes K + # from that cache. Reject graphs that observe K in any other way. + output_nodes = match.output_nodes() + if len(output_nodes) < 3 or output_nodes[2] is None: + return False + matched_nodes = set(match.nodes) + attention_op = torch.ops.vllm.unified_attention_with_output.default + + def is_unified_attention_user(user: fx.Node) -> bool: + return user.op == "call_function" and ( + user.target == attention_op + or (user.args and user.args[0] == attention_op) + ) + + return all( + user in matched_nodes or is_unified_attention_user(user) + for user in output_nodes[2].users + ) + + +def _discover_mrope_configs( + config: VllmConfig, +) -> tuple[tuple[tuple[int, int, int], bool], ...]: + model_config = config.model_config + hf_configs = ( + getattr(model_config, "hf_text_config", None), + getattr(model_config, "hf_config", None), + ) + configs: set[tuple[tuple[int, int, int], bool]] = set() + for hf_config in hf_configs: + if hf_config is None: + continue + rope_parameters = getattr(hf_config, "rope_parameters", None) + if rope_parameters is None: + rope_parameters = getattr(hf_config, "rope_scaling", None) + if not isinstance(rope_parameters, Mapping): + continue + section = rope_parameters.get("mrope_section") + if not isinstance(section, (list, tuple)) or len(section) != 3: + continue + try: + parsed_section = ( + int(section[0]), + int(section[1]), + int(section[2]), + ) + except (TypeError, ValueError): + continue + if any(value <= 0 for value in parsed_section): + continue + configs.add( + ( + parsed_section, + bool(rope_parameters.get("mrope_interleaved", False)), + ) + ) + return tuple(sorted(configs)) # --------------------------------------------------------------------------- @@ -412,7 +832,7 @@ class QkNormRopeKvCachePattern: class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): - """Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel. + """Fuse QK-norm + RoPE/MRoPE + KV cache update into an AITER HIP kernel. Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for attention layers that support the combined operation, eliminating two @@ -429,22 +849,33 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): cc = config.compilation_config self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num + self.mrope_configs = ( + _discover_mrope_configs(config) + if cc.is_custom_op_enabled("rotary_embedding") + else () + ) dtype = config.model_config.dtype if dtype not in (torch.bfloat16, torch.float16): logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled: unsupported dtype %s", + dtype, ) return attn_layers = get_layers_from_vllm_config(config, Attention) for _, layer in attn_layers.items(): - if not layer.impl.fused_qk_norm_rope_kvcache_supported(): + supports_rope = layer.impl.fused_qk_norm_rope_kvcache_supported() + supports_mrope = ( + bool(self.mrope_configs) + and layer.impl.fused_qk_norm_mrope_kvcache_supported() + ) + if not supports_rope and not supports_mrope: continue if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: " "head_size=%d is not supported by the " "fused_qk_norm_rope_cache_pts_quant_shuffle kernel " "(supported: %s). Falling back to the unfused path.", @@ -455,22 +886,71 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): if layer.head_size_v != layer.head_size: # The fused kernel uses a single head_dim for q/k/v. logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: " "head_size_v=%d differs from head_size=%d, which the fused " "kernel does not support. Falling back to the unfused path.", layer.head_size_v, layer.head_size, ) continue - for epsilon in [1e-5, 1e-6]: - for neox in [True, False]: - for quant_q in [False, True]: - QkNormRopeKvCachePattern( - layer=layer, - eps=epsilon, - is_neox=neox, - quant_query=quant_q, - ).register(self.patterns) + if supports_rope: + for epsilon in [1e-5, 1e-6]: + for neox in [True, False]: + for quant_q in [False, True]: + QkNormRopeKvCachePattern( + layer=layer, + eps=epsilon, + is_neox=neox, + quant_query=quant_q, + ).register(self.patterns) + + if supports_mrope: + for section, is_interleaved in self.mrope_configs: + rotary_dim = 2 * sum(section) + if rotary_dim > layer.head_size: + logger.warning_once( + "QK Norm+MRoPE+KVCache fusion not enabled for a " + "layer: rotary_dim=%d exceeds head_size=%d.", + rotary_dim, + layer.head_size, + ) + continue + vector_width = layer.head_size // 32 + for epsilon in [1e-5, 1e-6]: + for neox in [True, False]: + # Each warp lane processes `vector_width` adjacent + # values. NeoX additionally exchanges vectors + # across the half-rotary boundary. + rotary_alignment = vector_width * (2 if neox else 1) + if rotary_dim % rotary_alignment != 0: + logger.warning_once( + "QK Norm+MRoPE+KVCache fusion not enabled " + "for a layer/style: rotary_dim=%d must be " + "divisible by %d for head_size=%d and " + "is_neox=%s.", + rotary_dim, + rotary_alignment, + layer.head_size, + neox, + ) + continue + for quant_q in [False, True]: + QkNormMRopeKvCachePattern( + layer=layer, + eps=epsilon, + is_neox=neox, + quant_query=quant_q, + rotary_dim=rotary_dim, + mrope_section=section, + is_interleaved=is_interleaved, + ).register(self.patterns) + + # Opaque LayerName is a pattern wildcard on torch >= 2.11, so + # homogeneous attention layers produce duplicate search patterns. + # As in the other LayerName-aware fusion passes, one supported + # layer registers all shape/style variants for the model. + if _USE_LAYERNAME: + break self.dump_patterns(config, self.patterns) @@ -478,8 +958,8 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): def __call__(self, graph: fx.Graph) -> None: self.matched_count = self.patterns.apply(graph) logger.info( - "QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) " - "with AITER fused_qk_norm_rope_cache_pts_quant_shuffle", + "QK-Norm+RoPE/MRoPE+KVCache fusion: replaced %s pattern(s) " + "with an AITER fused attention-prologue kernel", self.matched_count, ) @@ -487,4 +967,10 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): return compile_range.end <= self.max_token_num def uuid(self) -> str: - return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern) + return VllmInductorPass.hash_source( + self, + QkNormRopeKvCachePattern, + QkNormMRopeKvCachePattern, + _discover_mrope_configs, + repr(self.mrope_configs), + ) diff --git a/vllm/compilation/passes/pass_manager.py b/vllm/compilation/passes/pass_manager.py index 8ee8f0354e..df0e58b210 100644 --- a/vllm/compilation/passes/pass_manager.py +++ b/vllm/compilation/passes/pass_manager.py @@ -41,6 +41,7 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu(): if current_platform.is_cuda_alike(): from .fusion.act_quant_fusion import ActivationQuantFusionPass from .fusion.attn_quant_fusion import AttnQuantFusionPass + from .fusion.collective_fusion import AsyncTPPass from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass @@ -50,7 +51,6 @@ if current_platform.is_cuda_alike(): if current_platform.is_cuda(): from .fusion.allreduce_rms_fusion import AllReduceFusionPass - from .fusion.collective_fusion import AsyncTPPass if current_platform.is_xpu(): from .fusion.act_quant_fusion import ActivationQuantFusionPass diff --git a/vllm/config/__init__.py b/vllm/config/__init__.py index 940aea7552..88f75a8fb1 100644 --- a/vllm/config/__init__.py +++ b/vllm/config/__init__.py @@ -20,6 +20,7 @@ from vllm.config.kernel import KernelConfig from vllm.config.kv_events import KVEventsConfig from vllm.config.kv_transfer import KVTransferConfig from vllm.config.load import LoadConfig +from vllm.config.logging import LoggingConfig from vllm.config.lora import LoRAConfig from vllm.config.mamba import MambaConfig from vllm.config.model import ( @@ -97,6 +98,8 @@ __all__ = [ "KVTransferConfig", # From vllm.config.load "LoadConfig", + # From vllm.config.logging + "LoggingConfig", # From vllm.config.lora "LoRAConfig", # From vllm.config.mamba diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 8ccc7332bd..eefcf1f49e 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -154,15 +154,15 @@ class PassConfig: fuse_rope_kvcache: bool = None # type: ignore[assignment] """Fuse the QK rope + KV cache ops.""" fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment] - """Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP + """Fuse QK RMSNorm + RoPE/MRoPE + KV cache update into an AITER HIP kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache - for layers that support it. Auto-enabled at O1+ on ROCm for models - with QK-norm (e.g. Qwen3-MoE).""" + for layers that support it. Auto-enabled at O2+ on ROCm for models + with QK-norm (e.g. Qwen3-MoE and Qwen3-VL-class architectures).""" rope_kvcache_fusion_max_token_num: int = 256 """The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode. Larger batch sizes e.g. during prefill will use the unfused kernels. - Also applies to the fused QK-Norm+RoPE+KVCache pass. + Also applies to the fused QK-Norm+RoPE/MRoPE+KVCache pass. """ fi_allreduce_fusion_max_size_mb: float | None = None diff --git a/vllm/config/engram.py b/vllm/config/engram.py index 0f5ec31b33..fa28d803b5 100644 --- a/vllm/config/engram.py +++ b/vllm/config/engram.py @@ -50,10 +50,19 @@ class EngramConfig: the other settings allow it, falling back to per-replica tables when DP replicas are not co-located on one node or /dev/shm cannot hold them.""" + use_thp: bool = False + """Back private CPU-offloaded tables with transparent huge pages (best + effort, falls back to ordinary pinned pages). Prefaulting the tables at + startup takes longer. Requires cpu_offload without dp_shared_memory.""" + @model_validator(mode="after") def _validate_shared_memory(self) -> Self: if self.dp_shared_memory and not self.cpu_offload: raise ValueError("dp_shared_memory requires cpu_offload=True") + if self.use_thp and (not self.cpu_offload or self.dp_shared_memory): + raise ValueError( + "use_thp requires cpu_offload=True and dp_shared_memory=False" + ) return self def verify_model_config(self, model_config: "ModelConfig | None") -> None: @@ -73,7 +82,8 @@ class EngramConfig: ): raise ValueError( "EngramConfig requires a model with supported Engram " - "embeddings, non-empty n-gram layer ids, and CUDA." + "embeddings, non-empty n-gram layer ids, and a CUDA-alike " + "device (CUDA or ROCm)." ) def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None: diff --git a/vllm/config/load.py b/vllm/config/load.py index ee13e2fccd..d82997be17 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -39,9 +39,8 @@ class LoadConfig: InstantTensor, which enables distributed loading with pipelined prefetching and fast direct I/O. - "ipc_cache" will map post-quantized weights from a local weight cache - daemon via CUDA IPC for fast engine restarts. See - `vllm/model_executor/model_loader/weight_cache/daemon.py` for how to - launch the daemon. + daemon via CUDA IPC for fast engine restarts. Launch the daemon with + `vllm preload` first. - "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is mainly diff --git a/vllm/config/logging.py b/vllm/config/logging.py new file mode 100644 index 0000000000..bb807d617d --- /dev/null +++ b/vllm/config/logging.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Literal, cast + +from pydantic import Field + +import vllm.envs as envs +from vllm.config.utils import config + +LogLevel = Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"] + + +@config +class LoggingConfig: + """vLLM logging. Default ``dictConfig`` adds a formatted stream handler. + + Supply a JSON object with ``--logging-config`` or individual fields with + dotted arguments such as ``--logging-config.log_level DEBUG`` and + ``--logging-config.pylogging_config_file logging.json``. ``--log-level`` + and legacy ``--log-config-file`` override their matching fields. Set + ``configure_logging`` to false to skip applying a ``dictConfig``. + """ + + log_level: LogLevel = Field( + default_factory=lambda: cast(LogLevel, envs.VLLM_LOGGING_LEVEL) + ) + """Log level used when no custom logging configuration is provided.""" + + configure_logging: bool = Field(default_factory=lambda: envs.VLLM_CONFIGURE_LOGGING) + """Whether to apply a Python logging configuration. + + With no ``pylogging_config_file``, the default config adds a formatted + stream handler to the ``vllm`` logger. A custom Python logging configuration + file also requires this to be enabled. + """ + + pylogging_config_file: str | None = Field( + default_factory=lambda: envs.VLLM_LOGGING_CONFIG_PATH + ) + """Path to a Python logging JSON file using + [``dictConfig`` schema](https://docs.python.org/3/library/logging.config.html#configuration-file-format). + + A custom ``dictConfig`` is authoritative over ``log_level`` for logger, + handler, and formatter settings. + """ diff --git a/vllm/config/model.py b/vllm/config/model.py index 7cdba9f78f..15d046de76 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -55,7 +55,7 @@ from vllm.utils.import_utils import LazyLoader from vllm.v1.attention.backends.registry import AttentionBackendEnum if TYPE_CHECKING: - from transformers import PretrainedConfig + from transformers import PreTrainedConfig import vllm.model_executor.layers.quantization as me_quant import vllm.model_executor.models as me_models @@ -64,7 +64,7 @@ if TYPE_CHECKING: from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.v1.sample.logits_processor import LogitsProcessor else: - PretrainedConfig = Any + PreTrainedConfig = Any me_quant = LazyLoader( "model_executor", globals(), "vllm.model_executor.layers.quantization" @@ -89,7 +89,6 @@ ConvertOption = Literal["auto", ConvertType] TokenizerMode = Literal[ "auto", "hf", - "slow", "mistral", "deepseek_v32", "deepseek_v4", @@ -105,7 +104,7 @@ PROCESSED_LOGPROBS_MODES: tuple[LogprobsMode, ...] = ( "processed_logits", "processed_logprobs", ) -HfOverrides = dict[str, Any] | Callable[[PretrainedConfig], PretrainedConfig] +HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig] ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"] LayerBlockType = Literal["attention", "linear_attention", "mamba"] @@ -148,7 +147,6 @@ class ModelConfig: - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer. - "hf" will use the fast tokenizer if available. - - "slow" will always use the slow tokenizer. - "mistral" will always use the tokenizer from `mistral_common`. - "deepseek_v32" will always use the tokenizer from `deepseek_v32`. - "deepseek_v4" will always use the tokenizer from `deepseek_v4`. @@ -183,10 +181,14 @@ class ModelConfig: We must set the global seed because otherwise, different tensor parallel workers would sample different tokens, leading to inconsistent results.""" - hf_config: PretrainedConfig = field(init=False) + hf_config: PreTrainedConfig = field(init=False) """The Hugging Face config of the model.""" - hf_text_config: PretrainedConfig = field(init=False) + hf_text_config: PreTrainedConfig = field(init=False) """The Hugging Face config of the text model (same as hf_config for text models).""" + is_submodel_config: bool = field(default=False, init=False) + """Whether this is a submodule view derived by `VllmConfig.with_hf_config` + (e.g. a multimodal model's text stack). Its architecture list is empty, so + deployment-level validation must not run against it.""" word_embeddings_untied_by_checkpoint: bool = field(default=False, init=False) """Whether `tie_word_embeddings` was overridden to `False` because the checkpoint contains an `lm_head` of its own. The two may still turn out to be identical, in @@ -479,7 +481,7 @@ class ModelConfig: def _update_nested( self, - target: PretrainedConfig | dict[str, Any], + target: PreTrainedConfig | dict[str, Any], updates: dict[str, Any], ) -> None: """Recursively updates a config or dict with nested updates.""" @@ -507,15 +509,15 @@ class ModelConfig: def _apply_dict_overrides( self, - config: PretrainedConfig, + config: PreTrainedConfig, overrides: dict[str, Any], ) -> None: """Apply dict overrides, handling both nested configs and dict values.""" - from transformers import PretrainedConfig + from transformers import PreTrainedConfig for key, value in overrides.items(): attr = getattr(config, key, None) - if attr is not None and isinstance(attr, PretrainedConfig): + if attr is not None and isinstance(attr, PreTrainedConfig): # It's a nested config - recursively update it self._update_nested(attr, value) else: @@ -585,22 +587,27 @@ class ModelConfig: self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer) # If loading model/tokenizer from HF Hub, resolve the revision once - # to prevent resolving it multiple times downstream. - # If the weights come from a different repo, we cannot eagerly resolve revision - weights_from_model = not self.model_weights or self.model_weights == self.model - # If the config comes from a different repo, we cannot eagerly resolve revision - config_from_model = not self.hf_config_path or self.hf_config_path == self.model - can_resolve_model_revision = config_from_model and weights_from_model - if can_resolve_model_revision: - self.revision = resolve_revision( - self.model, + # to prevent resolving it multiple times downstream. A resolved revision + # only pins the repo it was resolved for, so each repo needs its own call. + self.revision = resolve_revision( + self.model, + self.revision, + self.hf_token, + ) + + # The config can live in another repo, which `self.revision` does not pin. + # It stays `None` if the config comes from `self.model`, so that call sites + # fall back to `self.revision` the same way they fall back to `self.model`. + self._hf_config_revision = None + if self.hf_config_path and self.hf_config_path != self.model: + self._hf_config_revision = resolve_revision( + self.hf_config_path, self.revision, self.hf_token, ) if ( - can_resolve_model_revision - and self.tokenizer == self.model + self.tokenizer == self.model and self.tokenizer_revision == requested_revision ): self.tokenizer_revision = self.revision @@ -628,7 +635,7 @@ class ModelConfig: hf_config = get_config( self.hf_config_path or self.model, self.trust_remote_code, - self.revision, + self._hf_config_revision or self.revision, self.code_revision, self.config_format, hf_overrides_kw=hf_overrides_kw, @@ -1082,7 +1089,10 @@ class ModelConfig: # Check if the architecture we're wrapping has defaults runner = None task = None - if defaults := try_match_architecture_defaults(self.architectures[0]): + # architectures is empty for with_hf_config() submodel views. + if self.architectures and ( + defaults := try_match_architecture_defaults(self.architectures[0]) + ): _, (runner, task) = defaults # User specified value take precedence if self.runner != "auto": @@ -1772,7 +1782,7 @@ class ModelConfig: config = try_get_generation_config( self.hf_config_path or self.model, trust_remote_code=self.trust_remote_code, - revision=self.revision, + revision=self._hf_config_revision or self.revision, code_revision=self.code_revision, config_format=self.config_format, hf_token=self.hf_token, @@ -1815,6 +1825,8 @@ class ModelConfig: available_params = [ "repetition_penalty", + "presence_penalty", + "frequency_penalty", "temperature", "top_k", "top_p", @@ -2386,7 +2398,7 @@ def _resolve_auto_dtype( def _get_and_verify_dtype( model_id: str, - config: PretrainedConfig, + config: PreTrainedConfig, dtype: str | torch.dtype, *, is_pooling_model: bool, @@ -2433,7 +2445,7 @@ def _get_and_verify_dtype( def _get_head_dtype( - config: PretrainedConfig, dtype: torch.dtype, runner_type: str + config: PreTrainedConfig, dtype: torch.dtype, runner_type: str ) -> torch.dtype: head_dtype: str | torch.dtype | None = getattr(config, "head_dtype", None) @@ -2457,7 +2469,7 @@ def _get_head_dtype( def _get_and_verify_max_len( - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, model_arch_config: ModelArchitectureConfig, tokenizer_config: dict | None, max_model_len: int | None, diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 4eca42430c..5ce309456b 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -534,11 +534,19 @@ class MultiModalConfig: """Get the keyword arguments to pass to the multi-modal processor according to the extra arguments passed during inference. """ - kwargs = self.mm_processor_kwargs or {} + merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs) if self.mm_device_do_normalize: - kwargs["do_normalize"] = False - kwargs["do_rescale"] = False - return kwargs | dict(inference_kwargs) + # The model expects raw pixels and normalises on device; + # overriding these per request would silently double-normalise. + for key in ("do_normalize", "do_rescale"): + if key in inference_kwargs: + logger.warning_once( + "mm_device_do_normalize is enabled; ignoring " + "per-request mm_processor_kwargs[%r].", + key, + ) + merged[key] = False + return merged def use_gpu_video_backend(self) -> bool: """Return whether the configured video loader or codec uses the GPU.""" diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index 9185c26c4d..1fb94d3efd 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -84,6 +84,12 @@ class SchedulerConfig: The cap is not applied when the request is the only one in the batch, since there is no other request for it to starve.""" + long_prefill_token_threshold_adaptive: bool = Field(default=False) + """Floor the effective long prefill token threshold at a fair share of + the token budget: max_num_batched_tokens divided by the number of + queued and running requests. Only applies when + long_prefill_token_threshold is nonzero.""" + max_num_queued_reqs: int | None = Field(default=None, ge=0) """Maximum number of requests that can be in-flight (waiting or running) at the same time, or None for no limit. When the limit is reached, new @@ -232,13 +238,17 @@ class SchedulerConfig: # The first half of this warning can be removed once the Scheduler interface is # finalized and we can maintain support for scheduler classes that implement it - logger.warning_once( - "Using custom scheduler class %s. This scheduler interface is not public " - "and compatibility may not be maintained. If you have subclassed Scheduler " - "instead of AsyncScheduler, you will see degraded performance due to async " - "scheduling being disabled.", - self.scheduler_cls, # type: ignore[arg-type] - ) + if not ( + isinstance(self.scheduler_cls, str) + and self.scheduler_cls.startswith("vllm.") + ): + logger.warning_once( + "Using custom scheduler class %s. This scheduler interface is not " + "public and compatibility may not be maintained. If you have " + "subclassed Scheduler instead of AsyncScheduler, you will see " + "degraded performance due to async scheduling being disabled.", + self.scheduler_cls, # type: ignore[arg-type] + ) if not isinstance(self.scheduler_cls, str): return cast(type["SchedulerInterface"], self.scheduler_cls) return resolve_obj_by_qualname(self.scheduler_cls) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index eb9d548137..dbec9fd743 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -22,12 +22,12 @@ from vllm.utils.import_utils import LazyLoader, has_arctic_inference from vllm.v1.attention.backends.registry import AttentionBackendEnum if TYPE_CHECKING: - from transformers import PretrainedConfig + from transformers import PreTrainedConfig import vllm.model_executor.layers.quantization as me_quant from vllm.config.vllm import VllmConfig else: - PretrainedConfig = Any + PreTrainedConfig = Any VllmConfig = Any me_quant = LazyLoader( @@ -631,6 +631,8 @@ class SpeculativeConfig: "dspark", ) factors.append(uses_aux_hidden_states) + if self.method == "dspark": + factors.append(self.enable_adaptive_verification) if self.draft_model_config is not None: factors.append(self.draft_model_config.compute_hash()) @@ -657,7 +659,7 @@ class SpeculativeConfig: return hash_str @staticmethod - def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: + def hf_config_override(hf_config: PreTrainedConfig) -> PreTrainedConfig: initial_architecture = hf_config.architectures[0] use_v32_mtp = hf_config.model_type in ("deepseek_v32", "glm_moe_dsa") if hf_config.model_type == "dots3_note": @@ -694,8 +696,8 @@ class SpeculativeConfig: if hf_config.model_type in ("deepseek_v4", "deepseek_v41"): # V4.1 has no classic-MTP draft: its checkpoints ship DSpark stages # under ``mtp.*``, so only V4 gets an MTP architecture here. The - # DSpark path rewrites ``architectures`` itself and needs only - # ``n_predict``; ``method="mtp"`` on V4.1 is rejected below. + # DSpark path rewrites ``architectures`` itself; + # ``method="mtp"`` on V4.1 is rejected below. is_v41 = hf_config.model_type == "deepseek_v41" hf_config.model_type = "deepseek_mtp" n_predict = getattr(hf_config, "num_nextn_predict_layers", None) @@ -1066,16 +1068,16 @@ class SpeculativeConfig: @staticmethod def _apply_composed_hf_override( - target_hf_overrides: Callable[[PretrainedConfig], PretrainedConfig], - hf_config: PretrainedConfig, - ) -> PretrainedConfig: + target_hf_overrides: Callable[[PreTrainedConfig], PreTrainedConfig], + hf_config: PreTrainedConfig, + ) -> PreTrainedConfig: hf_config = SpeculativeConfig.hf_config_override(hf_config) return target_hf_overrides(hf_config) @staticmethod def compose_draft_hf_overrides( target_hf_overrides: HfOverrides | None, - ) -> Callable[[PretrainedConfig], PretrainedConfig]: + ) -> Callable[[PreTrainedConfig], PreTrainedConfig]: """Build the ``hf_overrides`` for the draft ``ModelConfig``. Callable overrides on the target are config-to-config transforms @@ -1440,14 +1442,6 @@ class SpeculativeConfig: draft_hf_config.architectures = [ "DSparkV41DraftModel" if is_v41 else "DSparkDraftModel" ] - if is_v41: - # hf_config_override set n_predict to the number of - # MTP stages (3), but one DSpark round drafts - # dspark_block_size tokens; num_speculative_tokens - # divisibility is checked against n_predict below. - draft_hf_config.n_predict = getattr( - draft_hf_config, "dspark_block_size", None - ) or getattr(draft_hf_config, "n_predict", None) self.draft_model_config.quantization = ( self.target_model_config.quantization ) @@ -1464,11 +1458,6 @@ class SpeculativeConfig: and getattr(hf, "target_layer_ids", None) is not None ): hf.dspark_target_layer_ids = hf.target_layer_ids - if ( - getattr(hf, "n_predict", None) is None - and getattr(hf, "block_size", None) is not None - ): - hf.n_predict = hf.block_size if self.method in ("dflash", "dspark"): self.parallel_drafting = True @@ -1483,7 +1472,15 @@ class SpeculativeConfig: n_predict = getattr( self.draft_model_config.hf_config, "n_predict", None ) - if n_predict is not None: + if self.use_dspark(): + if self.num_speculative_tokens is None: + # DSpark's parallel width is independent of MTP stages. + hf_config = self.draft_model_config.hf_config + block_size = getattr(hf_config, "block_size", None) + if block_size is None: + block_size = getattr(hf_config, "dspark_block_size", None) + self.num_speculative_tokens = block_size + elif n_predict is not None: if self.num_speculative_tokens is None: # Default to max value defined in draft model config. self.num_speculative_tokens = n_predict @@ -1672,7 +1669,7 @@ class SpeculativeConfig: @staticmethod def _maybe_override_draft_max_position_embeddings( - draft_hf_config: PretrainedConfig, + draft_hf_config: PreTrainedConfig, target_max_model_len: int, ) -> None: """Raise an EAGLE draft's max_position_embeddings up to the target's. @@ -1709,7 +1706,7 @@ class SpeculativeConfig: def _verify_and_get_draft_tp( target_parallel_config: ParallelConfig, speculative_draft_tensor_parallel_size: int | None, - draft_hf_config: PretrainedConfig, + draft_hf_config: PreTrainedConfig, ) -> int: """Verifies and adjusts the tensor parallel size for a draft model specified using speculative_draft_tensor_parallel_size. diff --git a/vllm/config/utils.py b/vllm/config/utils.py index ed72474619..c3d7bb1056 100644 --- a/vllm/config/utils.py +++ b/vllm/config/utils.py @@ -138,7 +138,7 @@ def getattr_iter( ) -> Any: """A helper function that retrieves an attribute from an object which may have multiple possible names. This is useful when fetching attributes from - arbitrary `transformers.PretrainedConfig` instances. + arbitrary `transformers.PreTrainedConfig` instances. In the case where the first name in `names` is the preferred name, and any other names are deprecated aliases, setting `warn=True` will log a @@ -314,7 +314,7 @@ def normalize_value(x): except Exception: return str(x) - # PretrainedConfig (must be before dataclass branch as these are now dataclasses) + # PreTrainedConfig (must be before dataclass branch as these are now dataclasses) if hasattr(x, "to_json_string") and callable(x.to_json_string): try: return x.to_json_string() diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 66c537d853..220da0c72b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -40,6 +40,7 @@ from .kernel import KernelConfig from .kv_events import KVEventsConfig from .kv_transfer import KVTransferConfig from .load import LoadConfig +from .logging import LoggingConfig from .lora import LoRAConfig from .mamba import MambaBackendEnum, MambaConfig from .model import ModelConfig @@ -56,12 +57,12 @@ from .watermarking import WatermarkConfig from .weight_transfer import WeightTransferConfig if TYPE_CHECKING: - from transformers import PretrainedConfig + from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.v1.kv_cache_interface import KVCacheConfig else: - PretrainedConfig = Any + PreTrainedConfig = Any QuantizationConfig = Any @@ -242,7 +243,7 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool: def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool: - """Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER.""" + """Enable fused QK-norm + RoPE/MRoPE + KV cache update with AITER.""" from vllm._aiter_ops import rocm_aiter_ops if not rocm_aiter_ops.is_enabled(): @@ -402,6 +403,8 @@ class VllmConfig: default_factory=ObservabilityConfig ) """Observability configuration.""" + logging_config: LoggingConfig = Field(default_factory=LoggingConfig) + """Logging configuration.""" quant_config: QuantizationConfig | None = None """Quantization configuration.""" compilation_config: CompilationConfig = Field(default_factory=CompilationConfig) @@ -722,11 +725,23 @@ class VllmConfig: if model_config is not None and current_platform.is_rocm(): architectures = getattr(model_config, "architectures", ()) if any(arch in ROCM_DEFAULT_MRV1_ARCHITECTURES for arch in architectures): + # This default is a speed preference, not a claim that V1 can + # serve the config, so it yields where V1 cannot. It yields by + # falling through to the checks below, not by selecting V2. + v1_unsupported = self._get_v1_model_runner_unsupported_features() + if not v1_unsupported: + logger.warning_once( + "Defaulting to V1 model runner on ROCm for model " + "architectures: %s", + ", ".join(architectures), + ) + return False logger.warning_once( - "Defaulting to V1 model runner on ROCm for model architectures: %s", + "Skipping the ROCm V1 model runner default for %s: V1 does " + "not support %s.", ", ".join(architectures), + ", ".join(v1_unsupported), ) - return False if not HAS_TRITON: logger.warning_once( @@ -745,8 +760,8 @@ class VllmConfig: return True - def _is_dflash2_draft(self) -> bool: - """Whether the DFlash draft is a DFlash2 one, by the architecture the + def _is_dflash_candidate_draft(self) -> bool: + """Whether the DFlash draft has a candidate head, by the architecture the speculator selects on (v1/worker/gpu/spec_decode/__init__.py).""" spec = self.speculative_config if spec is None or spec.method != "dflash": @@ -754,7 +769,11 @@ class VllmConfig: draft_config = getattr(spec, "draft_model_config", None) if draft_config is None: return False - return "DFlash2DraftModel" in (draft_config.architectures or []) + return bool( + {"DFlash2DraftModel", "LiLiCorrDraftModel"}.intersection( + draft_config.architectures or [] + ) + ) def _dflash_needs_multi_kv_group(self) -> bool: """Whether a DFlash draft mixes sliding-window and full attention.""" @@ -776,6 +795,45 @@ class VllmConfig: architectures = set(model_config.architectures) return bool(architectures & default_breakable_cudagraph_architectures()) + def _uses_breakable_cudagraph_for_batch_invariance(self) -> bool: + """Avoid freezing runtime-M tile lookup in compiled forward (#54243). + Breakable graphs look up tuned bf16, unquantized qkv/o/gate_up/down tiles + at capture; lm_head runs outside compiled forward and does not benefit.""" + from vllm.model_executor.determinism import batch_invariant_configs as bi + from vllm.platforms import current_platform + + model = self.model_config + if ( + not envs.VLLM_BATCH_INVARIANT + or model is None + or model.enforce_eager + or model.dtype != torch.bfloat16 + or model.quantization is not None + or not current_platform.is_cuda() + # Sequence parallelism / async TP are torch.compile passes. + or self.compilation_config.pass_config.enable_sp + or self.compilation_config.pass_config.fuse_gemm_comms + ): + return False + family = bi._get_tuned_matmul_arch_family( + current_platform.get_device_capability() + ) + if family is None or family not in bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: + return False + table = bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS[family] + parallel = self.parallel_config + tp = parallel.tensor_parallel_size + hidden = model.get_hidden_size() + head = model.get_head_size() + heads = model.get_num_attention_heads(parallel) + kv_heads = model.get_num_kv_heads(parallel) + shapes = [((heads + 2 * kv_heads) * head, hidden), (hidden, heads * head)] + intermediate = getattr(model.hf_text_config, "intermediate_size", None) + # Per-layer sizes (e.g. Gemma3n) are not modeled. + if isinstance(intermediate, int): + shapes += [(2 * intermediate // tp, hidden), (hidden, intermediate // tp)] + return any(shape in table for shape in shapes) + def _maybe_enable_breakable_cudagraph(self) -> bool: if ( "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ @@ -786,6 +844,16 @@ class VllmConfig: "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." ) + elif ( + "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ + and envs.VLLM_BATCH_INVARIANT + and self._uses_breakable_cudagraph_for_batch_invariance() + ): + os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1" + logger.info_once( + "VLLM_BATCH_INVARIANT=1: auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " + "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." + ) from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, @@ -890,7 +958,7 @@ class VllmConfig: def with_hf_config( self, - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, architectures: list[str] | None = None, ) -> "VllmConfig": if architectures is not None: @@ -943,6 +1011,7 @@ class VllmConfig: model_config.hf_config = hf_config model_config.model_arch_config = model_config.get_model_arch_config() + model_config.is_submodel_config = True return replace(self, model_config=model_config) @@ -1092,7 +1161,14 @@ class VllmConfig: """Reject configurations unsupported by enabled auxiliary outputs.""" if not self.aux_output_config.enabled: return - if not self.use_v2_model_runner: + from vllm.platforms import current_platform + + # In-tree platforms only wire AuxOutput to MRV2. TPU and out-of-tree + # platforms bring their own model runners and validate AuxOutput + # support themselves. + if not self.use_v2_model_runner and not ( + current_platform.is_tpu() or current_platform.is_out_of_tree() + ): raise ValueError( "AuxOutput Connector requires Model Runner V2; set " "VLLM_USE_V2_MODEL_RUNNER=1." @@ -1331,8 +1407,27 @@ class VllmConfig: # To give each torch profile run a unique instance name. self.instance_id = f"{time.time_ns()}" + if self.model_config is not None and self.model_config.is_submodel_config: + # with_hf_config() view: the parent config was already validated, + # and this view's empty architecture list makes the model-dependent checks + # below unsafe (e.g. use_mla resolves the architecture registry). + return + self._resolve_mm_encoder_only() + if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching: + # Such an instance publishes encoder embeddings and runs no language + # model, so it holds no KV cache for prefix caching to reuse and its + # coordinator would have no group to manage. Disable before + # `try_verify_and_update_config` so model config hooks (e.g. the + # hybrid mamba hook setting `mamba_block_size`) already see prefix + # caching as disabled. + logger.info( + "Disabling prefix caching: this instance runs the " + "multi-modal encoder only." + ) + self.cache_config.enable_prefix_caching = False + if self.performance_mode != "balanced": logger.info_once("Performance mode set to '%s'.", self.performance_mode) @@ -1597,12 +1692,24 @@ class VllmConfig: ) if self.model_config is not None and self.model_config.enforce_eager: - logger.warning_once( - "Enforce eager set, disabling torch.compile and CUDAGraphs. " - "This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none" - ) self.compilation_config.mode = CompilationMode.NONE self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + if self.parallel_config.enable_fault_tolerance: + # Keep JIT warmup: in-inference Triton compilation latency + # spikes can delay peer-fault detection past its deadline. + logger.warning_once( + "Enforce eager set, disabling torch.compile and CUDAGraphs. " + "This is equivalent to setting -cc.mode=none " + "-cc.cudagraph_mode=none" + ) + else: + logger.warning_once( + "Enforce eager set, disabling torch.compile, CUDAGraphs, and " + "JIT kernel warmup. This is equivalent to setting " + "-cc.mode=none -cc.cudagraph_mode=none and " + "--kernel_config.enable_jit_warmup=False" + ) + self.kernel_config.enable_jit_warmup = False if os.environ.get("TORCH_COMPILE_DISABLE") == "1": logger.warning_once( @@ -1611,6 +1718,26 @@ class VllmConfig: ) self.compilation_config.mode = CompilationMode.NONE + # TODO: This is a stopgap that turns sequence parallelism / async TP + # off under batch invariance. Make the sequence-parallel reduce-scatter + # path batch-invariant instead so they can stay enabled (#56370). + # Sequence parallelism / async TP rewrite all_reduce + rms_norm into a + # reduce-scatter whose reduction order depends on the batch, so they + # are not batch-invariant. Decide this before the breakable-CUDA-graph + # default below, which declines to auto-enable when they are on. + pass_config = self.compilation_config.pass_config + if envs.VLLM_BATCH_INVARIANT and ( + pass_config.enable_sp or pass_config.fuse_gemm_comms + ): + logger.warning_once( + "Disabling sequence parallelism and async TP " + "(pass_config.enable_sp / fuse_gemm_comms) when " + "VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path " + "is not batch-invariant (see vllm-project/vllm#56370)." + ) + pass_config.enable_sp = False + pass_config.fuse_gemm_comms = False + breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph() if not breakable_cudagraph_enabled and ( @@ -1894,6 +2021,16 @@ class VllmConfig: ) current_platform.check_and_update_config(self) + # After the platform hook, which has the last word on async scheduling. + if ( + self.diffusion_config is not None + and self.scheduler_config.async_scheduling + and self.scheduler_config.scheduler_cls is None + ): + self.scheduler_config.scheduler_cls = ( + "vllm.v1.core.sched.diffusion_scheduler.DiffusionAsyncScheduler" + ) + self._normalize_piecewise_cudagraph_mode( breakable_cudagraph_enabled=breakable_cudagraph_enabled ) @@ -2069,16 +2206,6 @@ class VllmConfig: self._post_init_kv_transfer_config() self._verify_aux_output_compatibility() - if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching: - # Such an instance publishes encoder embeddings and runs no language - # model, so it holds no KV cache for prefix caching to reuse and its - # coordinator would have no group to manage. - logger.info( - "Disabling prefix caching: this instance runs the " - "multi-modal encoder only." - ) - self.cache_config.enable_prefix_caching = False - # Hybrid KV cache manager (HMA) runtime rules: # - Explicit enable (--no-disable-kv-cache-manager): error if runtime # disables it @@ -3001,11 +3128,11 @@ class VllmConfig: if self._dflash_needs_multi_kv_group(): unsupported.append("mixed sliding/full dflash drafts") - # The DFlash2 candidate selector exists only in the V2 speculator. On + # DFlash candidate heads exist only in the V2 speculator. On # V1 the same checkpoint drafts through DFlashProposer, which never # calls it, so the draft would degrade to DFlash1 silently. - if self._is_dflash2_draft(): - unsupported.append("dflash2 drafts") + if self._is_dflash_candidate_draft(): + unsupported.append("DFlash candidate-head drafts") if self.model_config is not None and self.model_config.is_diffusion: unsupported.append("diffusion models") diff --git a/vllm/config/weight_transfer.py b/vllm/config/weight_transfer.py index 3e849753c2..99676526af 100644 --- a/vllm/config/weight_transfer.py +++ b/vllm/config/weight_transfer.py @@ -9,7 +9,9 @@ from vllm.config.utils import config class WeightTransferConfig: """Configuration for weight transfer during RL training.""" - backend: Literal["nccl", "ipc", "sparse_nccl", "sharded_rdt"] | str = "nccl" + backend: Literal["nccl", "ipc", "sparse_nccl", "sharded_rdt", "nccl_m2n"] | str = ( + "nccl" + ) """The backend to use for weight transfer. Validated against the `WeightTransferEngineFactory` registry at engine creation time. """ diff --git a/vllm/device_allocator/alloc_conf.py b/vllm/device_allocator/alloc_conf.py new file mode 100644 index 0000000000..405a4ecac3 --- /dev/null +++ b/vllm/device_allocator/alloc_conf.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Helpers for reading and editing PyTorch's allocator configuration. + +Torch's allocator config is parsed as a whole: ``parseArgs`` resets every +option that is not explicitly present in the string it is handed. Writing a +single field therefore silently drops the rest of the user's configuration, +for the remainder of the process: + + >>> # PYTORCH_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 + >>> torch._C._accelerator_setAllocatorSettings("expandable_segments:False") + >>> # max_split_size is now SIZE_MAX, not 512 MiB -- and writing + >>> # "expandable_segments:True" back does not bring it home either. + +So anything that wants to toggle one option has to read the current config, +flip just that field, and write the whole string back. +""" + +import os + +import torch + +# Checked in torch's own precedence order: the accelerator-agnostic variable +# wins, then the CUDA one, then the HIP one (which is what ROCm users set). +ALLOC_CONF_ENV_VARS = ( + "PYTORCH_ALLOC_CONF", + "PYTORCH_CUDA_ALLOC_CONF", + "PYTORCH_HIP_ALLOC_CONF", +) + +EXPANDABLE_SEGMENTS = "expandable_segments" + + +def alloc_conf_from_env( + environ: "os._Environ[str] | dict[str, str] | None" = None, +) -> str: + """The allocator config string the process was started with, if any.""" + env = os.environ if environ is None else environ + for name in ALLOC_CONF_ENV_VARS: + conf = env.get(name) + if conf: + return conf + return "" + + +def live_alloc_conf() -> str | None: + """The allocator config string currently in effect, or ``None``. + + Read from the live allocator rather than the environment, so that a + *runtime* write by another component is visible. + """ + try: + settings = torch.cuda.memory._snapshot().get("allocator_settings") + if settings is not None: + for name in ALLOC_CONF_ENV_VARS: + conf = settings.get(name) + if conf is not None: + return conf + except Exception: # noqa: BLE001 + pass + return None + + +def current_alloc_conf() -> str: + """The live allocator config, falling back to the environment.""" + live = live_alloc_conf() + return live if live is not None else alloc_conf_from_env() + + +def conf_flag_enabled(conf: str, key: str) -> bool: + """Parse ``:`` out of an allocator config string.""" + for field in conf.split(","): + name, _, value = field.partition(":") + if name.strip() == key: + return value.strip().lower() in ("true", "1") + return False + + +def with_conf_flag(conf: str, key: str, enabled: bool) -> str: + """Return ``conf`` with only ``key`` flipped, every other field verbatim.""" + want = f"{key}:{'True' if enabled else 'False'}" + fields = [f for f in (f.strip() for f in conf.split(",")) if f] + for i, field in enumerate(fields): + if field.partition(":")[0].strip() == key: + fields[i] = want + return ",".join(fields) + fields.append(want) + return ",".join(fields) + + +def expandable_segments_enabled_from_env( + environ: "os._Environ[str] | dict[str, str] | None" = None, +) -> bool: + return conf_flag_enabled(alloc_conf_from_env(environ), EXPANDABLE_SEGMENTS) + + +def expandable_segments_enabled() -> bool | None: + """Live ``expandable_segments`` state, or ``None`` if it cannot be read. + + ``None`` is deliberately distinct from ``False``: the environment can never + reflect a runtime write, so a caller verifying a write it just issued must + not read "cannot tell" as "the write did not take". + """ + try: + settings = torch.cuda.memory._snapshot().get("allocator_settings") + if settings is not None and EXPANDABLE_SEGMENTS in settings: + return bool(settings[EXPANDABLE_SEGMENTS]) + except Exception: # noqa: BLE001 + pass + return None + + +def set_alloc_conf(conf: str) -> None: + """Write a complete allocator config string.""" + setter = getattr(torch._C, "_accelerator_setAllocatorSettings", None) + if setter is not None: + setter(conf) + return + torch.cuda.memory._set_allocator_settings(conf) diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index 63d4a1f4e0..2e5612e5fa 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -10,7 +10,6 @@ # the only successful approach is to call cuda driver API in C. import atexit import gc -import os from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any @@ -18,6 +17,13 @@ from typing import Any import torch from vllm.device_allocator import AllocationData, HandleType +from vllm.device_allocator.alloc_conf import ( + EXPANDABLE_SEGMENTS, + conf_flag_enabled, + current_alloc_conf, + set_alloc_conf, + with_conf_flag, +) from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.system_utils import find_loaded_library @@ -371,13 +377,17 @@ class CuMemAllocator: # Expandable segments are incompatible with the memory pool used for # sleep mode (see https://github.com/pytorch/pytorch/issues/147851). - # If the user has enabled expandable segments via - # PYTORCH_CUDA_ALLOC_CONF, temporarily disable them for the duration - # of the memory pool context and restore on exit. - conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") - expandable_was_enabled = "expandable_segments:True" in conf + # If the user has enabled expandable segments, temporarily disable + # them for the duration of the memory pool context and restore on + # exit. The whole config string is rewritten, not just this one + # field: torch resets every option that is absent from the string it + # is handed, so a bare "expandable_segments:False" would also drop + # max_split_size_mb, garbage_collection_threshold and + # roundup_power2_divisions for the rest of the process. + prev_conf = current_alloc_conf() + expandable_was_enabled = conf_flag_enabled(prev_conf, EXPANDABLE_SEGMENTS) if expandable_was_enabled: - torch.cuda.memory._set_allocator_settings("expandable_segments:False") + set_alloc_conf(with_conf_flag(prev_conf, EXPANDABLE_SEGMENTS, False)) old_tag = self.current_tag self.current_tag = tag @@ -411,7 +421,7 @@ class CuMemAllocator: finally: self.current_tag = old_tag if expandable_was_enabled: - torch.cuda.memory._set_allocator_settings("expandable_segments:True") + set_alloc_conf(prev_conf) def get_current_usage(self) -> int: """Get the total number of bytes allocated in the memory pool.""" diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 9e53c12e24..21a1f00da4 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -56,11 +56,17 @@ class AgRsAll2AllManager(All2AllManagerBase): def __init__(self, cpu_group, tcp_store_group=None): super().__init__(cpu_group, tcp_store_group) + self.use_ep = get_current_vllm_config().parallel_config.enable_expert_parallel def _get_comm_group(self, is_sequence_parallel: bool) -> Any: if is_sequence_parallel: return get_ep_group() if self.dp_world_size > 1: + if self.use_ep and get_pcp_group().world_size > 1: + assert self.tp_group.world_size == 1, ( + "DP+PCP with TP>1 requires sequence-parallel MoE inputs" + ) + return get_ep_group() return get_dp_group() return get_pcp_group() @@ -72,6 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase): assert dp_metadata is not None sizes = dp_metadata.get_chunk_sizes_across_dp_rank() assert sizes is not None + assert len(sizes) == comm_group.world_size return sizes def dispatch_router_logits( diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py index e18f9194f5..b6fb540c7d 100644 --- a/vllm/distributed/device_communicators/custom_all_reduce.py +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from contextlib import contextmanager -from typing import cast +from typing import Literal, cast import torch import torch.distributed as dist @@ -32,6 +32,9 @@ except ImportError: logger = init_logger(__name__) +_ReduceScatterBackend = Literal["legacy", "mnnvl_lamport", "mnnvl_multimem"] +_MNNVL_MULTIMEM_REDUCE_SCATTER_WORLD_SIZES = (2, 4, 8) + def _has_local_multicast_support(device: torch.device) -> bool: """Return whether this CUDA device can allocate multicast memory.""" @@ -53,6 +56,12 @@ def _has_local_multicast_support(device: torch.device) -> bool: return False +def _all_ranks_true(group: ProcessGroup, local_value: bool) -> bool: + group_value = torch.tensor(int(local_value), dtype=torch.int32, device="cpu") + dist.all_reduce(group_value, op=dist.ReduceOp.MIN, group=group) + return bool(group_value.item()) + + def _group_can_attempt_mnnvl( group: ProcessGroup, device: torch.device, @@ -71,17 +80,7 @@ def _group_can_attempt_mnnvl( and current_platform.has_device_capability(100, device_index) and _has_local_multicast_support(device) ) - group_support = torch.tensor( - int(local_support), - dtype=torch.int32, - device="cpu", - ) - dist.all_reduce( - group_support, - op=dist.ReduceOp.MIN, - group=group, - ) - return bool(group_support.item()) + return _all_ranks_true(group, local_support) def _can_p2p(rank: int, world_size: int) -> bool: @@ -101,6 +100,20 @@ def _can_p2p(rank: int, world_size: int) -> bool: return True +def _supports_mnnvl_multimem_reduce_scatter( + device: torch.device, world_size: int +) -> bool: + return ( + world_size in _MNNVL_MULTIMEM_REDUCE_SCATTER_WORLD_SIZES + and device.index is not None + and ( + current_platform.is_device_capability((10, 0), device.index) + or current_platform.is_device_capability((10, 3), device.index) + ) + and _has_local_multicast_support(device) + ) + + from vllm.distributed.utils import is_weak_contiguous # noqa: E402 @@ -116,6 +129,8 @@ class CustomAllreduce: } _DEFAULT_REDUCE_SCATTER_MAX_SIZE = 16 * 1024 * 1024 _DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE = 16 * 1024 * 1024 + _DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE = 64 * 1024 * 1024 + _MNNVL_MULTIMEM_REDUCE_SCATTER_BLOCKS = 8 # max_size: max supported allreduce size def __init__( @@ -127,6 +142,9 @@ class CustomAllreduce: max_mnnvl_all_gather_size=None, max_reduce_scatter_size=_DEFAULT_REDUCE_SCATTER_MAX_SIZE, max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE, + max_mnnvl_multimem_reduce_scatter_size=( + _DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE + ), symm_mem_enabled=False, ) -> None: """Args: @@ -153,6 +171,13 @@ class CustomAllreduce: self.mnnvl_lamport_epochs = None self.mnnvl_lamport_ag_epoch_ptr = 0 self.mnnvl_lamport_rs_epoch_ptr = 0 + self.mnnvl_multimem_rs_supported = False + self.mnnvl_multimem_rs_initialized = False + self.mnnvl_multimem_rs_buffer = None + self.mnnvl_multimem_rs_handle = None + self.mnnvl_multimem_rs_buffer_size = 0 + self.mnnvl_multimem_rs_local_ptr = 0 + self.mnnvl_multimem_rs_multicast_ptr = 0 self.mnnvl_only = False if not custom_ar: @@ -197,6 +222,9 @@ class CustomAllreduce: # now `device` is a `torch.device` object assert isinstance(device, torch.device) self.device = device + mnnvl_multimem_rs_supported = _supports_mnnvl_multimem_reduce_scatter( + device, world_size + ) if not same_node and not _group_can_attempt_mnnvl(group, device): logger.warning( "Custom collectives are disabled because this multi-node " @@ -294,6 +322,9 @@ class CustomAllreduce: self.max_mnnvl_all_gather_size = max_mnnvl_all_gather_size self.max_reduce_scatter_size = max_reduce_scatter_size self.max_mnnvl_reduce_scatter_size = max_mnnvl_reduce_scatter_size + self.max_mnnvl_multimem_reduce_scatter_size = ( + max_mnnvl_multimem_reduce_scatter_size + ) self.rank = rank self.world_size = world_size self.fully_connected = fully_connected @@ -307,6 +338,11 @@ class CustomAllreduce: max_mnnvl_reduce_scatter_size, ) ) + if world_size in _MNNVL_MULTIMEM_REDUCE_SCATTER_WORLD_SIZES: + self.mnnvl_multimem_rs_supported = _all_ranks_true( + self.group, + mnnvl_multimem_rs_supported and bool(self.mnnvl_multicast_ptr), + ) if not same_node and not self.mnnvl_multicast_ptr: logger.warning( "Custom collectives are disabled because this multi-node " @@ -368,6 +404,59 @@ class CustomAllreduce: except RuntimeError as error: logger.debug("MNNVL AG/RS initialization failed: %s", error) + def _init_mnnvl_multimem_reduce_scatter_buffer(self) -> None: + if self.mnnvl_multimem_rs_initialized: + return + self.mnnvl_multimem_rs_initialized = True + if not self.mnnvl_multimem_rs_supported: + return + + assert torch_symm_mem is not None + buffer = None + signal_size = ops.meta_size() + try: + buffer = torch_symm_mem.empty( + signal_size + self.max_mnnvl_multimem_reduce_scatter_size, + dtype=torch.uint8, + device=self.device, + ) + except RuntimeError as error: + logger.debug("MNNVL multimem RS allocation failed: %s", error) + if not _all_ranks_true(self.group, buffer is not None): + logger.warning_once( + "MNNVL multimem reduce-scatter symmetric-memory allocation " + "failed on at least one rank; falling back to NCCL.", + scope="global", + ) + return + assert buffer is not None + + handle = None + try: + handle = torch_symm_mem.rendezvous(buffer, self.group.group_name) + if handle is not None: + buffer[:signal_size].zero_() + torch.accelerator.synchronize() + except RuntimeError as error: + logger.debug("MNNVL multimem RS rendezvous failed: %s", error) + if not _all_ranks_true( + self.group, + handle is not None and bool(handle.multicast_ptr), + ): + logger.warning_once( + "MNNVL multimem reduce-scatter symmetric-memory rendezvous " + "failed on at least one rank; falling back to NCCL.", + scope="global", + ) + return + assert handle is not None + + self.mnnvl_multimem_rs_buffer = buffer + self.mnnvl_multimem_rs_handle = handle + self.mnnvl_multimem_rs_buffer_size = self.max_mnnvl_multimem_reduce_scatter_size + self.mnnvl_multimem_rs_local_ptr = buffer.data_ptr() + signal_size + self.mnnvl_multimem_rs_multicast_ptr = handle.multicast_ptr + signal_size + @contextmanager def capture(self): """The main responsibility of this context manager is the @@ -509,35 +598,77 @@ class CustomAllreduce: ) return out - def should_custom_reduce_scatter(self, inp: torch.Tensor) -> bool: + def _select_reduce_scatter_backend( + self, inp: torch.Tensor + ) -> _ReduceScatterBackend | None: if self.disabled or not current_platform.is_cuda(): - return False + return None if self.world_size == 16 and not self.mnnvl_only: - return False + return None inp_size = inp.nbytes if inp.dtype not in (torch.float32, torch.float16, torch.bfloat16): - return False + return None if inp.shape[0] % self.world_size != 0: - return False + return None output_size = inp_size // self.world_size - max_size = ( - self.max_mnnvl_reduce_scatter_size - if self.mnnvl_multicast_ptr - else self.max_reduce_scatter_size - ) - return ( - 0 < inp_size <= max_size - and output_size % 16 == 0 - and is_weak_contiguous(inp) - and (self.fully_connected or bool(self.mnnvl_multicast_ptr)) - ) + if inp_size <= 0 or output_size % 16 != 0 or not is_weak_contiguous(inp): + return None + + if self.mnnvl_multicast_ptr: + if inp_size <= self.max_mnnvl_reduce_scatter_size: + return "mnnvl_lamport" + if ( + ( + self.mnnvl_multimem_rs_multicast_ptr + or ( + self.mnnvl_multimem_rs_supported + and not self.mnnvl_multimem_rs_initialized + ) + ) + and not envs.VLLM_BATCH_INVARIANT + and inp_size <= self.max_mnnvl_multimem_reduce_scatter_size + ): + return "mnnvl_multimem" + return None + + if self.fully_connected and inp_size <= self.max_reduce_scatter_size: + return "legacy" + return None + + def should_custom_reduce_scatter(self, inp: torch.Tensor) -> bool: + return self._select_reduce_scatter_backend(inp) is not None + + def should_mnnvl_multimem_reduce_scatter(self, inp: torch.Tensor) -> bool: + return self._select_reduce_scatter_backend(inp) == "mnnvl_multimem" def custom_reduce_scatter(self, inp: torch.Tensor) -> torch.Tensor | None: - if not self.should_custom_reduce_scatter(inp): + backend = self._select_reduce_scatter_backend(inp) + if backend is None: return None + if backend == "mnnvl_multimem" and not self.mnnvl_multimem_rs_multicast_ptr: + if self._IS_CAPTURING: + return None + self._init_mnnvl_multimem_reduce_scatter_buffer() + if not self.mnnvl_multimem_rs_multicast_ptr: + return None + out_shape = (inp.shape[0] // self.world_size,) + inp.shape[1:] out = torch.empty(out_shape, dtype=inp.dtype, device=inp.device) - if self.mnnvl_multicast_ptr: + if backend == "mnnvl_multimem": + logger.info_once( + "Using the low-SM MNNVL multimem reduce-scatter kernel.", + scope="global", + ) + ops.mnnvl_multimem_reduce_scatter( + self._ptr, + inp, + out, + self.mnnvl_multimem_rs_local_ptr, + self.mnnvl_multimem_rs_multicast_ptr, + self.mnnvl_multimem_rs_buffer_size, + self._MNNVL_MULTIMEM_REDUCE_SCATTER_BLOCKS, + ) + elif backend == "mnnvl_lamport": logger.info_once( "Using the MNNVL Lamport reduce-scatter kernel.", scope="global", @@ -571,6 +702,8 @@ class CustomAllreduce: self.mnnvl_handle = None self.mnnvl_buffer = None self.mnnvl_lamport_epochs = None + self.mnnvl_multimem_rs_handle = None + self.mnnvl_multimem_rs_buffer = None def __del__(self): self.close() diff --git a/vllm/distributed/device_communicators/pynccl_allocator.py b/vllm/distributed/device_communicators/pynccl_allocator.py index 4898c6212f..1f48ff5ee4 100644 --- a/vllm/distributed/device_communicators/pynccl_allocator.py +++ b/vllm/distributed/device_communicators/pynccl_allocator.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import atexit import contextlib +import logging import tempfile from typing import Any @@ -83,7 +84,7 @@ def compile_nccl_allocator(): cpp_sources=nccl_allocator_source, with_cuda=True, extra_ldflags=ldflags, - verbose=envs.VLLM_LOGGING_LEVEL == "DEBUG", + verbose=logger.isEnabledFor(logging.DEBUG), is_python_module=False, build_directory=out_dir, extra_include_paths=nccl_include_paths, diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index 35d26bb44b..081d05068e 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -60,24 +60,26 @@ def collect_ec_item_metadata( Keyed by mm_hash, each entry carries a `metadata` dict with whatever placeholder fields `resolver` says this model needs published for its - modality, so a consumer can skip the image transform. `item_indices` + modality, so a consumer can skip media preprocessing. `item_indices` identifies every occurrence in `mm_features`, including repeated hashes. - `data` is None for items served from the processor cache, so the metadata is - unavailable here and the consumer has to fall back to processing the - media itself. A connector that also has transfer coordinates to report + Audio token counts come from the placeholder; other metadata requires + `data` and falls back to raw media when it is unavailable on cache hits. + A connector that also has transfer coordinates to report (e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside `metadata`, not into it. """ items: dict[str, dict[str, Any]] = {} for index, feature in enumerate(mm_features): metadata: dict[str, Any] = {} + wanted = resolver.fields_for(feature.modality) + if "audio_num_tokens" in wanted: + metadata["audio_num_tokens"] = [feature.mm_position.get_num_embeds()] if feature.data is not None: - wanted = resolver.fields_for(feature.modality) for key, value in feature.data.get_data().items(): if key not in wanted: continue if isinstance(value, torch.Tensor): - metadata[key] = value.tolist() + metadata[key] = torch.atleast_1d(value).tolist() elif is_list_of(value, (int, float)): # Some metadata (e.g. Qwen3-VL video timestamps) is # produced as a plain list rather than a tensor. diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index 19fc15b275..2602edf2df 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -550,10 +550,7 @@ class ElasticEPScalingExecutor: is_async_enabled = eplb_state.is_async run_async = async_op and is_async_enabled eplb_state.is_async = run_async - if rank_mapping is None: - eplb_state.rearrange() - else: - eplb_state.rearrange(rank_mapping=rank_mapping) + eplb_state.rearrange(rank_mapping=rank_mapping, use_last_expert_load=True) if not run_async: # Wait for non-blocking expert resharding copies before continuing # the Elastic EP reconfiguration. diff --git a/vllm/distributed/elastic_ep/standby_state.py b/vllm/distributed/elastic_ep/standby_state.py index c9b12f447d..084ed51f8c 100644 --- a/vllm/distributed/elastic_ep/standby_state.py +++ b/vllm/distributed/elastic_ep/standby_state.py @@ -6,6 +6,7 @@ from vllm.distributed.device_communicators.pynccl import defer_comm_warmup_on_ro from vllm.distributed.parallel_state import ( _init_stateless_group, _node_count, + get_pcp_group, get_pp_group, get_tp_group, get_world_group, @@ -72,12 +73,13 @@ def create_standby_groups( _STANDBY_WORLD_NODE_COUNT = _node_count(_STANDBY_WORLD.tcp_store_group) tp_size = get_tp_group().world_size + pcp_size = get_pcp_group().world_size pp_size = get_pp_group().world_size all_ranks = torch.arange(new_world_size_across_dp).reshape( - -1, new_dp_size, pp_size, tp_size + -1, new_dp_size, pp_size, pcp_size, tp_size ) - standby_dp_ranks = all_ranks.transpose(1, 3).reshape(-1, new_dp_size).unbind(0) + standby_dp_ranks = all_ranks.transpose(1, 4).reshape(-1, new_dp_size).unbind(0) standby_dp_ranks = [x.tolist() for x in standby_dp_ranks] # Deferred to commit so the warm-up runs while the engine is paused. @@ -87,7 +89,9 @@ def create_standby_groups( ) standby_ep_ranks = ( - all_ranks.transpose(1, 2).reshape(-1, new_dp_size * tp_size).unbind(0) + all_ranks.transpose(1, 2) + .reshape(-1, new_dp_size * pcp_size * tp_size) + .unbind(0) ) standby_ep_ranks = [x.tolist() for x in standby_ep_ranks] _STANDBY_EP = _init_stateless_group( diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index e4d805b9fe..205226cde4 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -183,6 +183,8 @@ class EplbModelState: See: https://github.com/vllm-project/vllm/pull/22167#pullrequestreview-3086143856 """ + last_expert_load: torch.Tensor + """Global logical-expert totals from the last reshuffle; equal demand initially.""" model_name: str model: MixtureOfExperts expert_buffer: list[torch.Tensor] @@ -499,6 +501,11 @@ class EplbState: expert_load_pass=expert_load_pass, expert_load_pass_buffer=expert_load_pass_buffer, expert_load_window=expert_load_window, + last_expert_load=torch.ones( + (model.num_moe_layers, model.num_logical_experts), + dtype=torch.int64, + device=self.device, + ), model_name=model_config.model, model=model, expert_buffer=expert_buffer, @@ -744,6 +751,7 @@ class EplbState: self, is_profile: bool = False, rank_mapping: dict[int, int] | None = None, + use_last_expert_load: bool = False, ) -> torch.Tensor | None: """Rearrange the experts according to the current load. @@ -753,6 +761,7 @@ class EplbState: no memory movement will be performed. Default is False. rank_mapping (dict[int, int] | None): The rank mapping when scaling is done in EEP. + use_last_expert_load: Use the last reshuffle's logical-expert totals. """ ep_group = get_ep_group().device_group @@ -775,6 +784,13 @@ class EplbState: # Map the physical expert load to global logical experts global_expert_load_windows = [] for eplb_model_state in self.model_states.values(): + if use_last_expert_load: + # Distribute rank 0's saved counts through the existing all-reduce. + saved_load = eplb_model_state.last_expert_load + global_expert_load_windows.append( + saved_load.clone() if is_main_rank else torch.zeros_like(saved_load) + ) + continue expert_load_window = eplb_model_state.expert_load_window physical_to_logical = eplb_model_state.physical_to_logical_map invalid_idx = eplb_model_state.model.num_logical_experts @@ -835,6 +851,8 @@ class EplbState: for eplb_model_state, global_expert_load_window in zip( self.model_states.values(), global_expert_load_windows ): + if not is_profile: + eplb_model_state.last_expert_load = global_expert_load_window if not self.is_async or is_profile: # Get new expert mappings for the model. The policy runs on the # host, so the load window and current map have to come back. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 072018c039..6df9d1cfbe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -31,7 +31,7 @@ The class provides the following primitives: wait_for_layer_load() - blocks until layer i load is done save_kv_layer() - starts saving KV for layer i (maybe async) - wait_for_save() - blocks until all saves are done + wait_for_save() - submits saves and performs required synchronization get_transfer_results() - returns async send/receive completions and receive failures in one snapshot. @@ -42,6 +42,7 @@ The class provides the following primitives: import enum from abc import ABC, abstractmethod from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal @@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC): # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a custom KV cache allocation context, if configured. + + Returning None uses the engine's default memory pool. + """ + return None + def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: """Set the connector metadata from the scheduler. @@ -356,11 +364,14 @@ class KVConnectorBase_V1(ABC): @abstractmethod def wait_for_save(self): - """Block until all the save operations is done. This is called - as the forward context exits to ensure that the async saving - from save_kv_layer is complete before finishing the forward. + """Submit saves and perform required synchronization. - This prevents overwrites of paged KV buffer before saving done. + Called once per step, including empty steps, after any target and draft + forwards. Metadata remains bound during this call. Do nothing if there + is no save work. + + Async saves must retain their source blocks until completion is reported + to the scheduler. Otherwise, wait for saves to finish before returning. """ pass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index cff2367966..cbb7339c91 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -28,6 +28,16 @@ Example: - fill_mean (float): Mean value for random normal fill (default: 0.015) - fill_std (float): Standard deviation for random fill (default: 0.0) Set to 0 for constant values, >0 for random sampling + - startup_fill (bool): Fill the whole KV cache once at startup instead + of filling each new request's blocks in the step that schedules it + (default: False). Circular-buffer caches are still zeroed per + request, and attention caches whose new blocks the engine zeroes + before use (hybrid Mamba or mixed-precision KV caches) are still + filled per request. The fill is not redone after sleep mode + discards the KV cache. + + The dummy KV cache is only meant for performance measurement; outputs are + not meaningful for accuracy evaluation. """ @@ -45,13 +55,20 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( SupportsHMA, ) from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.kv_cache_utils import ( - dcp_world_size_for_kv_cache_spec, resolve_dcp_kv_block_size, ) -from vllm.v1.kv_cache_interface import CircularBufferSpec, iter_layer_specs +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + CircularBufferSpec, + KVCacheSpec, + KVQuantMode, + UniformTypeKVCacheSpecs, + iter_layer_specs, +) if TYPE_CHECKING: from vllm.config import VllmConfig @@ -64,6 +81,45 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def _get_fp8_dtype(spec: KVCacheSpec, cache_dtype: str) -> torch.dtype | None: + """The fp8 dtype of a KV cache that stores plain fp8 values as uint8. + + Returns None for other layouts, including packed ones that embed scales + or other metadata in the uint8 page. + """ + if ( + not isinstance(spec, AttentionSpec) + or spec.dtype != torch.uint8 + or spec.kv_quant_mode != KVQuantMode.FP8_PER_TENSOR + or spec.state_content_bytes is not None + ): + return None + cache_dtype = getattr(spec, "cache_dtype_str", None) or cache_dtype + if cache_dtype in ("fp8", "fp8_e4m3"): + return current_platform.fp8_dtype() + if cache_dtype == "fp8_e5m2": + return torch.float8_e5m2 + return None + + +def _get_startup_fill_group_ids(kv_cache_config: "KVCacheConfig") -> set[int]: + """KV cache groups that startup fill covers. + + Circular buffers are zero-filled per request. Attention groups whose new + blocks the engine zeroes before use are filled per request, since the + zeroing would wipe the startup fill. + """ + group_ids = set() + for group_idx, group in enumerate(kv_cache_config.kv_cache_groups): + spec = group.kv_cache_spec + if any(isinstance(s, CircularBufferSpec) for s in iter_layer_specs(spec)): + continue + if kv_cache_config.needs_kv_cache_zeroing and isinstance(spec, AttentionSpec): + continue + group_ids.add(group_idx) + return group_ids + + @dataclass class DecodeBenchConnectorMetadata(KVConnectorMetadata): """Metadata for DecodeBenchConnector. @@ -191,16 +247,18 @@ class DecodeBenchConnectorScheduler: """Scheduler-side implementation for DecodeBenchConnector.""" def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): + kv_transfer_config = vllm_config.kv_transfer_config + assert kv_transfer_config is not None + self.startup_fill = kv_transfer_config.get_from_extra_config( + "startup_fill", False + ) + self._startup_fill_group_ids = ( + _get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set() + ) dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size self.kv_cache_groups = kv_cache_config.kv_cache_groups self.group_block_sizes = tuple( - resolve_dcp_kv_block_size( - group.kv_cache_spec, - dcp_world_size_for_kv_cache_spec( - group.kv_cache_spec, - dcp_world_size, - ), - ) + resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size) for group in self.kv_cache_groups ) @@ -275,6 +333,9 @@ class DecodeBenchConnectorScheduler: strict=True, ) ): + if group_idx in self._startup_fill_group_ids: + block_ids_per_group_list.append([]) + continue is_circular_buffer = all( isinstance(spec, CircularBufferSpec) for spec in iter_layer_specs(group.kv_cache_spec) @@ -310,6 +371,9 @@ class DecodeBenchConnectorScheduler: ) block_ids_per_group_list.append(block_ids) block_ids_per_group = tuple(block_ids_per_group_list) + self._filled_requests.add(req_id) + if self.startup_fill and not any(block_ids_per_group): + return # Store the blocks to fill for all group. _pending_fills doesn't need cleanup # as it's cleared after build_connector_meta @@ -317,7 +381,6 @@ class DecodeBenchConnectorScheduler: block_ids_per_group, num_external_tokens, ) - self._filled_requests.add(req_id) block_counts = tuple(len(group) for group in block_ids_per_group) logger.debug( @@ -356,10 +419,27 @@ class DecodeBenchConnectorWorker: assert kv_transfer_config is not None self.fill_mean = kv_transfer_config.get_from_extra_config("fill_mean", 0.015) self.fill_std = kv_transfer_config.get_from_extra_config("fill_std", 0.0) + self.startup_fill = kv_transfer_config.get_from_extra_config( + "startup_fill", False + ) + self._startup_fill_group_ids = ( + _get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set() + ) # Will be populated via register_kv_caches self.kv_caches: dict[str, torch.Tensor] | None = None + # Layers whose uint8 KV cache stores plain fp8 values + self._fp8_dtypes: dict[str, torch.dtype] = {} + for group in kv_cache_config.kv_cache_groups: + for layer_name in group.layer_names: + spec = group.kv_cache_spec + if isinstance(spec, UniformTypeKVCacheSpecs): + spec = spec.kv_cache_specs[layer_name] + fp8_dtype = _get_fp8_dtype(spec, vllm_config.cache_config.cache_dtype) + if fp8_dtype is not None: + self._fp8_dtypes[layer_name] = fp8_dtype + # Mapping from KV cache group index to list of layer names in that group self.group_to_layers = { group_idx: list(group.layer_names) @@ -373,6 +453,17 @@ class DecodeBenchConnectorWorker: for spec in iter_layer_specs(group.kv_cache_spec) ) } + num_zeroed_groups = len( + self.group_to_layers.keys() + - self._startup_fill_group_ids + - self._zero_fill_group_ids + ) + if self.startup_fill and num_zeroed_groups: + logger.info( + "DecodeBenchConnector: the engine zeroes new blocks of %d KV " + "cache groups before use, so they are still filled per request.", + num_zeroed_groups, + ) def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): """Store references to the KV cache tensors.""" @@ -383,6 +474,48 @@ class DecodeBenchConnectorWorker: len(kv_caches), ) + if self.startup_fill: + self._fill_all_kv_caches() + + def _fill_all_kv_caches(self): + """Fill the KV cache tensors of the startup-filled groups once, in + place, with dummy values, so that no per-request fill is needed. + """ + assert self.kv_caches is not None + + num_layers = 0 + for group_idx in sorted(self._startup_fill_group_ids): + for layer_name in self.group_to_layers[group_idx]: + kv_cache = self.kv_caches.get(layer_name) + if isinstance(kv_cache, torch.Tensor): + tensors = [kv_cache] + elif isinstance(kv_cache, (list, tuple)) and all( + isinstance(t, torch.Tensor) for t in kv_cache + ): + tensors = list(kv_cache) + else: + logger.warning_once( + "DecodeBenchConnector: skipping startup fill for layer " + "%s whose KV cache is %s, not a tensor or a list/tuple " + "of tensors.", + layer_name, + type(kv_cache).__name__, + ) + continue + for tensor in tensors: + fill_dtype = self._fp8_dtypes.get(layer_name, tensor.dtype) + self._fill_tensor(tensor, self.fill_mean, self.fill_std, fill_dtype) + num_layers += 1 + + logger.info( + "DecodeBenchConnector: Filled %d KV cache layers at startup with %s " + "values (mean=%.3f, std=%.3f)", + num_layers, + "random" if self.fill_std > 0 else "constant", + self.fill_mean, + self.fill_std, + ) + def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata): """Fill the allocated KV cache blocks with dummy values. @@ -455,12 +588,17 @@ class DecodeBenchConnectorWorker: # dimension — so fill each tensor in its entirety with the same # dummy values. if isinstance(kv_cache, torch.Tensor): - self._fill_block_tensor(kv_cache, block_ids, fill_mean, fill_std) + fill_dtype = self._fp8_dtypes.get(layer_name, kv_cache.dtype) + self._fill_block_tensor( + kv_cache, block_ids, fill_mean, fill_std, fill_dtype + ) elif isinstance(kv_cache, (list, tuple)) and all( isinstance(t, torch.Tensor) for t in kv_cache ): for state_tensor in kv_cache: - self._fill_state_tensor(state_tensor, fill_mean, fill_std) + self._fill_tensor( + state_tensor, fill_mean, fill_std, state_tensor.dtype + ) else: logger.warning_once( "DecodeBenchConnector: skipping fill for layer %s whose KV " @@ -486,6 +624,7 @@ class DecodeBenchConnectorWorker: block_ids: list[int], fill_mean: float, fill_std: float, + fill_dtype: torch.dtype, ): """Fill the requested block rows of a block-indexed KV cache tensor. @@ -495,6 +634,8 @@ class DecodeBenchConnectorWorker: tensor's first dim are ignored. fill_mean: Mean value for the fill. fill_std: Standard deviation for the fill. + fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype + of a uint8 fp8 cache. """ # Convert block_ids to tensor on device @@ -509,46 +650,85 @@ class DecodeBenchConnectorWorker: if len(valid_block_ids) == 0: return - # Create fill values - either constant or random - block_shape = kv_cache.shape[1:] - if fill_std > 0: - # Random normal sampling - fill_values = torch.normal( - mean=fill_mean, - std=fill_std, - size=(len(valid_block_ids),) + block_shape, - dtype=kv_cache.dtype, - device=kv_cache.device, - ) - else: - # Constant fill value - fill_values = torch.full( - (len(valid_block_ids),) + block_shape, - fill_mean, - dtype=kv_cache.dtype, - device=kv_cache.device, - ) + fill_values = self._make_fill_values( + (len(valid_block_ids),) + kv_cache.shape[1:], + kv_cache.device, + fill_dtype, + fill_mean, + fill_std, + ) # Batch fill operation - kv_cache[valid_block_ids] = fill_values + kv_cache[valid_block_ids] = fill_values.view(kv_cache.dtype) - def _fill_state_tensor( - self, kv_cache: torch.Tensor, fill_mean: float, fill_std: float + def _make_fill_values( + self, + size: tuple[int, ...], + device: torch.device, + dtype: torch.dtype, + fill_mean: float, + fill_std: float, + ) -> torch.Tensor: + """Create constant or random fill values of ``dtype``, clamped to its + finite range. Non-floating dtypes (packed layouts) are filled with + zeros. + """ + if not dtype.is_floating_point: + if fill_mean or fill_std: + logger.warning_once( + "DecodeBenchConnector: %s KV caches do not hold plain " + "floating-point values; filling them with zeros.", + dtype, + ) + return torch.zeros(size, dtype=dtype, device=device) + + finfo = torch.finfo(dtype) + if fill_std > 0: + sample_dtype = torch.float32 + if dtype in (torch.float16, torch.bfloat16, torch.float32, torch.float64): + sample_dtype = dtype + values = torch.normal( + fill_mean, fill_std, size=size, device=device, dtype=sample_dtype + ) + return values.clamp_(finfo.min, finfo.max).to(dtype) + fill_mean = min(max(fill_mean, finfo.min), finfo.max) + return torch.full(size, fill_mean, dtype=dtype, device=device) + + def _fill_tensor( + self, + kv_cache: torch.Tensor, + fill_mean: float, + fill_std: float, + fill_dtype: torch.dtype, ): - """Fill an entire non-block-indexed state tensor with dummy values. + """Fill an entire tensor in place with dummy values. - Hybrid / linear-attention layers (e.g. Mamba, Kimi Delta Attention) - store their per-layer state as tensors with no num_blocks dimension, - so the whole tensor is filled with the same constant or random values - used for block fills, rather than selected block rows. + Used for startup fills, and for hybrid / linear-attention layers (e.g. + Mamba, Kimi Delta Attention) whose per-layer state tensors are filled + in their entirety with the same constant or random values used for + block fills, rather than selected block rows. Args: - kv_cache: A state tensor to fill in its entirety. + kv_cache: A tensor to fill in its entirety. fill_mean: Mean value for the fill. fill_std: Standard deviation for the fill. + fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype + of a uint8 fp8 cache. """ - if fill_std > 0: - kv_cache.normal_(mean=fill_mean, std=fill_std) - else: - kv_cache.fill_(fill_mean) + view = kv_cache.view(fill_dtype) + if fill_std == 0 or not fill_dtype.is_floating_point: + view.fill_( + self._make_fill_values( + (), torch.device("cpu"), fill_dtype, fill_mean, fill_std + ) + ) + return + # Sample in chunks to bound the temporary memory. + rows = max(1, (1 << 24) // max(1, view[0].numel())) + for chunk in view.split(rows): + chunk.copy_( + self._make_fill_values( + chunk.shape, chunk.device, fill_dtype, fill_mean, fill_std + ) + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py index 96191cd8f7..91552b7fe8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py @@ -142,6 +142,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1): engine.current_layer = 0 if hasattr(engine, "layerwise_retrievers"): engine.layerwise_retrievers = [] + engine.layerwise_storers = [] def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None: """Start loading the KV cache from the connector to vLLM's paged diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py index a50e603db9..ed887f296b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py @@ -606,6 +606,7 @@ class LMCacheConnectorV1Impl: self.async_loading = config.enable_async_loading self.layerwise_retrievers: list[Generator[torch.Tensor | None, None, None]] = [] + self.layerwise_storers: list[Generator[Any, None, None]] = [] self._stats_monitor = LMCStatsMonitor.GetOrCreate() if role == KVConnectorRole.SCHEDULER: # Create lookup client using factory @@ -798,6 +799,7 @@ class LMCacheConnectorV1Impl: """ self.current_layer = 0 self.layerwise_retrievers = [] + self.layerwise_storers = [] @_lmcache_nvtx_annotate def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index b22387e98b..beabf157cc 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -1051,15 +1051,9 @@ class MooncakeConnectorWorker: self.finished_recving_reqs: set[ReqId] = set() # Written from the receiver loop, drained from the worker thread. self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups - ) - ) - # Block IDs are only unique within a group; with HMA the scheduler - # tracks a single merged group, so failures are reported per request. + self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1 + # Block IDs are only unique within a group, so with multiple groups + # load failures are reported per request instead. self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() self.xfer_stats = MooncakeKVConnectorStats() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index eb4a7fafe9..dd64dd671e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication. from collections import Counter from collections.abc import Iterable, Sequence +from contextlib import AbstractContextManager from typing import Any import torch @@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): # Worker-side methods # ============================================================ + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None. + + Called by the Worker before ``initialize_kv_cache`` so that KV + cache is allocated from the Mooncake-managed pool when + ``custom_mem_pool`` is set in ``kv_connector_extra_config``. + """ + if self.connector_worker is None: + return None + return self.connector_worker.get_mem_pool_context() + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): assert self.connector_worker is not None self.connector_worker.register_kv_caches(kv_caches) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 6b6bf18ae4..173aa324e7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -19,13 +19,14 @@ import threading import time from collections.abc import Callable, Sequence from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import AbstractContextManager from typing import Any, TypeVar import torch import zmq import vllm.envs as envs -from vllm.config import VllmConfig +from vllm.config import ModelConfig, VllmConfig from vllm.distributed import ( get_dcp_group, get_pcp_group, @@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread): class MooncakeStoreWorker: """Worker-side component for MooncakeStoreConnector.""" + @staticmethod + def _create_mem_pool( + extra_config: dict[str, Any], model_config: ModelConfig + ) -> torch.cuda.MemPool | None: + pool_type = str(extra_config.get("custom_mem_pool") or "").upper() + if not pool_type: + return None + if pool_type not in ("NVLINK", "BAREX"): + raise ValueError( + f"Unsupported custom_mem_pool={pool_type!r}, " + "expected 'NVLINK' or 'BAREX'" + ) + if model_config.enable_sleep_mode or model_config.enable_cumem_allocator: + raise ValueError( + "custom_mem_pool is incompatible with enable_sleep_mode " + "or enable_cumem_allocator; CuMemAllocator cannot manage " + "allocations from the custom pool." + ) + try: + if pool_type == "NVLINK": + from mooncake.allocator import NVLinkAllocator as allocator_cls + else: # pool_type == "BAREX" + from mooncake.allocator import BarexAllocator as allocator_cls + except ImportError as e: + raise ImportError( + f"custom_mem_pool={pool_type!r} requires " + "mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake." + ) from e + + device = torch.device("cuda", torch.accelerator.current_device_index()) + allocator = allocator_cls.get_allocator(device) + mem_pool = torch.cuda.MemPool(allocator.allocator()) + logger.info("Using Mooncake custom memory pool: %s", pool_type) + return mem_pool + def __init__( self, vllm_config: VllmConfig, @@ -1461,13 +1497,7 @@ class MooncakeStoreWorker: and not self.can_put ) self.cache_config = vllm_config.cache_config - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups - ) - ) + self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1 self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config ) @@ -1477,6 +1507,10 @@ class MooncakeStoreWorker: # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_config() + # Validate and create the custom memory pool before initializing the + # store so failures cannot leak a partially initialized store handle. + self._mem_pool = self._create_mem_pool(extra_config, model_config) + self.store = MooncakeDistributedStore() local_ip = get_ip() local_hostname = rdma_utils.get_requester_local_hostname(local_ip) @@ -1747,6 +1781,13 @@ class MooncakeStoreWorker: ) return token_dbs + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None if + no custom pool is configured.""" + if self._mem_pool is None: + return None + return torch.cuda.use_mem_pool(self._mem_pool) + def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int: if self.dcp_size > 1: return 1 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index 5f1068562c..ae1eecf6e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -69,6 +69,8 @@ class WriteTask: event: torch.cuda.Event remote_notify_port: int remote_ip: str + multi_pod_hosts: list[str] = field(default_factory=list) + remote_dp_size_local: int = 0 enqueue_time: float = field(default_factory=time.perf_counter) retried: int = 0 @@ -96,7 +98,7 @@ class RemoteAllocInfo: writes_expected: int | None = None decode_dp_rank: int = 0 completion_request_id: str | None = None - completion_remote_notify_port: int | None = None + completion_notify_port: int | None = None completion_remote_ip: str | None = None completion_notified: bool = False transfer_statuses: list[Any] = field(default_factory=list) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index dce4a98068..0a8965a670 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker: kv_layer: torch.Tensor, remote_notify_port: int, remote_ip: str, + multi_pod_hosts: list[str], + remote_dp_size_local: int, ) -> None: """Schedule a block write operation. @@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker: kv_layer: KV cache tensor remote_notify_port: Port for completion notification remote_ip: IP address of remote node + multi_pod_hosts: List of pod IPs for multi-pod Wide-EP + remote_dp_size_local: Per-pod DP size for multi-pod """ # synchronization to prevent dirty reads between @@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker: event=event, remote_notify_port=remote_notify_port, remote_ip=remote_ip, + multi_pod_hosts=multi_pod_hosts, + remote_dp_size_local=remote_dp_size_local, ) self._writer.schedule_write(task) @@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker: ) def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer): - # Stash multi_pod_hosts + local DP size on the worker so - # MoRIIOEngine._finalize_if_complete (which sees only the WriteTask, - # not ReqMeta) can pick the per-rank pod IP for the completion notify. - # Last-writer-wins is safe: all requests share the same topology. - if meta.multi_pod_hosts: - self.multi_pod_hosts = list(meta.multi_pod_hosts) - else: - self.multi_pod_hosts = [meta.remote_host] - if meta.remote_dp_size_local: - self.remote_dp_size_local = int(meta.remote_dp_size_local) - else: - self.remote_dp_size_local = int(meta.remote_dp_size) + # Compute per-request values to pass through the task (no shared state). + # This avoids race conditions when concurrent requests target different + # decode pods - each WriteTask carries its own routing info. + hosts = ( + list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host] + ) + dp_local = ( + int(meta.remote_dp_size_local) + if meta.remote_dp_size_local + else int(meta.remote_dp_size) + ) # WRITE does not support HMA, so unwrap blocks into flat lists. # remote_block_ids can itself be empty so need to be careful when unwrapping. local_block_ids = meta.local_block_ids[0] @@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker: kv_layer=kv_layer, remote_notify_port=meta.remote_notify_port, remote_ip=meta.remote_host, + multi_pod_hosts=hosts, + remote_dp_size_local=dp_local, ) def merge_contiguous_blocks( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py index a0c6ec1f6d..4ec3f72fb4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py @@ -314,18 +314,27 @@ class MoRIIOWriter: request_info = self._get_remote_alloc_info(task.transfer_id) with self._write_state_lock: request_info.completion_request_id = task.request_id - request_info.completion_remote_notify_port = task.remote_notify_port # Wide-EP multi-pod: task.remote_ip addresses only the first pod, # so resolve the per-rank host from multi_pod_hosts (falls back to # task.remote_ip for single-pod or on any indexing miss). _remote_ip = task.remote_ip - _hosts = list(getattr(self.worker, "multi_pod_hosts", []) or []) - _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) + _hosts = task.multi_pod_hosts + _dp_local = task.remote_dp_size_local if _hosts and _dp_local > 0: _pod_idx = int(request_info.decode_dp_rank) // _dp_local if 0 <= _pod_idx < len(_hosts): _remote_ip = _hosts[_pod_idx] request_info.completion_remote_ip = _remote_ip + # Resolve the final notify port under the lock so finalize only + # reads it. The offset uses the per-pod local rank (% dp_local), + # since each pod binds notify sockets only for its local ranks; + # single-pod is bit-identical (modulus is a no-op). + _decode_dp_rank_for_port = int(request_info.decode_dp_rank) + if _dp_local > 0: + _decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local + request_info.completion_notify_port = task.remote_notify_port + ( + get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank) + ) if task.transfer_id in self._sealed_writes: request_info.writes_expected = self._sealed_writes[task.transfer_id] @@ -461,9 +470,9 @@ class MoRIIOWriter: if request_info.completion_notified: return request_id = request_info.completion_request_id - remote_notify_port = request_info.completion_remote_notify_port + remote_port = request_info.completion_notify_port remote_ip = request_info.completion_remote_ip - if request_id is None or remote_notify_port is None or remote_ip is None: + if request_id is None or remote_port is None or remote_ip is None: return transfer_statuses = list(request_info.transfer_statuses) request_info.transfer_statuses.clear() @@ -472,16 +481,6 @@ class MoRIIOWriter: # Wait for this request's transfers to complete. self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses) - # The notify port offset must use the per-pod local rank - # (% dp_local), since each pod binds notify sockets only for its local - # ranks. Single-pod is bit-identical (modulus is a no-op). - _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) - _decode_dp_rank_for_port = int(request_info.decode_dp_rank) - if _dp_local > 0: - _decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local - remote_port = remote_notify_port + get_port_offset( - _decode_dp_rank_for_port, self.worker.tp_rank - ) # Consider using RDMA immediate data in decode side # to eliminate the need for this notification. # Consider including the first gen token from prefill in the notification diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index a6f0d4cafb..73b6b0ccf6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA): # ============================== # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Forward a custom KV cache memory pool from a child connector. + + KV cache is allocated once and can only live in one pool, so at + most one child may provide a context. + """ + found: list[tuple[str, AbstractContextManager]] = [] + for connector in self._connectors: + if (ctx := connector.get_mem_pool_context()) is not None: + found.append((type(connector).__name__, ctx)) + + if len(found) > 1: + names = [name for name, _ in found] + raise ValueError( + f"Multiple connectors provide a KV cache memory pool {names}; " + "KV cache is allocated once and can only live in one pool. " + "Configure custom_mem_pool on at most one connector." + ) + return found[0][1] if found else None + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: for c in self._connectors: c.start_load_kv(forward_context, **kwargs) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 0f13a265df..f4a643d2b7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -839,6 +839,8 @@ class NixlBaseConnectorWorker: self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( "engine_ttl", 3600.0 ) + # Map of pull peer (host, port) -> engine_id currently serving it. + self._engine_by_address: dict[tuple[str, int], EngineId] = {} self.model_config = vllm_config.model_config @@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker: self._remote_agents[eid] = remote_agents self._engine_clock_offset[eid] = clock_offset self._engine_last_active[eid] = time.perf_counter() + self._track_remote_engine_replacement(eid, host, port) except Exception as e: self._log_failure( failure_type="handshake_setup_failed", @@ -1567,7 +1570,13 @@ class NixlBaseConnectorWorker: block_stride = physical_page_size else: block_stride = cache.stride(0) * cache.element_size() - storage_is_block_major = num_blocks * block_stride == registration_len + # A connector that registers the backing as one memory region + # may need its length rounded up to a page boundary. Such a tail + # is shorter than one block, so it cannot make the storage + # segmented. + storage_is_block_major = ( + 0 <= registration_len - num_blocks * block_stride < block_stride + ) # A view with a contiguous page can own a strided region even # when sibling layers share the same backing allocation. page_contiguous = ( @@ -1589,7 +1598,9 @@ class NixlBaseConnectorWorker: ): # TODO(Lucas): handle TP slicing for packed_storage; for now # restrict to MLA (DSv4) where kv is replicated. - storage_block_len = registration_len // num_blocks + # The complete storage row is one block stride. Dividing the + # registered length instead would fold the padding tail in. + storage_block_len = block_stride region_specs = [ (registration_base, storage_block_len, storage_block_len) ] @@ -2943,25 +2954,9 @@ class NixlBaseConnectorWorker: self._sync_device_after_direct_recv(direct_device_recving) # Handle timeout to avoid stranding blocks on remote. - now = time.perf_counter() - while self._reqs_to_send: - req_id, expires = next(iter(self._reqs_to_send.items())) - # Sorted dict, oldest requests are put first so we can exit early. - if now < expires: - break - count = self.consumer_notification_counts_by_req.pop(req_id, 0) - self.expected_consumer_notifications_by_req.pop(req_id, None) - self.xfer_stats.record_kv_expired_req() - logger.warning( - "Releasing expired KV blocks for request %s which were " - "retrieved by %d remote worker(s) before lease expired.", - req_id, - count, - ) - self._reqs_to_process.remove(req_id) - del self._reqs_to_send[req_id] - done_sending.add(req_id) + self._reap_expired_send_leases(done_sending) + self._cleanup_replaced_remote_engines() return KVConnectorTransferResults( finished_sending=done_sending, finished_recving=done_recving, @@ -2988,6 +2983,34 @@ class NixlBaseConnectorWorker: """ raise NotImplementedError + def _reap_expired_send_leases(self, done_sending: set[str]) -> None: + """Reclaim expired send-side KV leases into ``done_sending``. + + ``_reqs_to_send`` is not ordered by expiry: heartbeats update the + deadline in place, and mixed TTLs share the map, so a live head + entry can sit in front of already-expired ones. Scan every entry + rather than stopping at the first still-live request. + """ + if not self._reqs_to_send: + return + now = time.perf_counter() + expired = [ + req_id for req_id, expires in self._reqs_to_send.items() if now >= expires + ] + for req_id in expired: + count = self.consumer_notification_counts_by_req.pop(req_id, 0) + self.expected_consumer_notifications_by_req.pop(req_id, None) + self.xfer_stats.record_kv_expired_req() + logger.warning( + "Releasing expired KV blocks for request %s which were " + "retrieved by %d remote worker(s) before lease expired.", + req_id, + count, + ) + self._reqs_to_process.remove(req_id) + del self._reqs_to_send[req_id] + done_sending.add(req_id) + def _handle_heartbeat(self, payload: str) -> None: """Extend leases for requests referenced in a heartbeat. @@ -3522,14 +3545,53 @@ class NixlBaseConnectorWorker: and meta.remote is not None } + def _track_remote_engine_replacement( + self, engine_id: EngineId, host: str, port: int + ) -> None: + """Record the engine currently serving a pull peer address.""" + # TODO: Also handle push mode, which handshakes in both directions. + if self._TRANSFER_MODE != "pull": + return + previous = self._engine_by_address.get((host, port)) + self._engine_by_address[(host, port)] = engine_id + if previous is None or previous == engine_id: + return + logger.info( + "Remote engine at %s:%d changed from %s to %s.", + host, + port, + previous, + engine_id, + ) + + def _cleanup_replaced_remote_engines(self) -> None: + """Release replaced pull peers once requests and handshakes have drained.""" + if self._TRANSFER_MODE != "pull": + return + with self._handshake_lock: + if self._handshake_futures: + return + replaced = self._remote_agents.keys() - self._engine_by_address.values() + if not replaced: + return + # _recving_metadata is only accessed from this (main) thread. + busy = { + meta.remote.engine_id + for meta in self._recving_metadata.values() + if meta.remote is not None + } + for engine_id in replaced - busy: + self._cleanup_remote_engine(engine_id, log_eviction=False) + logger.info("Released NIXL state for replaced remote engine %s.", engine_id) + def _cleanup_remote_engine( self, engine_id: EngineId, *, log_eviction: bool = True ) -> None: """Remove all state for a single remote engine. Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by both TTL eviction and - shutdown. + all per-engine data structures. Used by TTL eviction, replaced peer + cleanup and shutdown. """ assert engine_id in self._remote_agents @@ -3539,6 +3601,9 @@ class NixlBaseConnectorWorker: # Pop under the handshake lock; NIXL teardown stays outside it. with self._handshake_lock: agents = self._remote_agents.pop(engine_id) + for address, eid in list(self._engine_by_address.items()): + if eid == engine_id: + del self._engine_by_address[address] for agent_name in agents.values(): self.nixl_wrapper.remove_remote_agent(agent_name) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 6539b7e57e..11b7215950 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # Start transfers for requests whose handshakes have now finished. while not self._ready_requests.empty(): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) + req_id, meta = self._ready_requests.get_nowait() + assert meta.remote is not None + if meta.remote.engine_id not in self._remote_agents: + # The engine was released after its handshake completed, so + # handshake again. This fails if the engine is gone. + self._background_nixl_handshake(req_id, meta.remote.engine_id, meta) + continue + self._read_blocks_for_req(req_id, meta) if self.pcp_rank > 0 and not self.pcp_dcp_sharded: # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index fbe85f46a3..ec77f426c5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -103,6 +103,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # (``_pop_done_transfers``); guarded by # ``_sending_transfers_lock``. self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + self._send_failures: set[ReqId] = set() self._sending_transfers_lock = threading.Lock() # Writer-thread owned matching state. @@ -162,6 +163,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): for handle in handles: self.nixl_wrapper.release_xfer_handle(handle) self._sending_transfers.clear() + self._send_failures.clear() super().shutdown() # --- Engine-main-thread entry point -------------------------------- # @@ -178,6 +180,12 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): ) assert meta.remote is not None remote_engine_id = meta.remote.engine_id + # Update last activity from this remote (same as pull), but only for + # an already-connected engine so we never leave an _engine_last_active + # entry without a _remote_agents entry (breaks _cleanup_remote_engine). + if remote_engine_id in self._remote_agents: + self._engine_last_active[remote_engine_id] = time.perf_counter() + logger.debug( "start_load_kv (push) for request %s from remote engine %s. " "Num local_block_ids: %s. Num remote_block_ids: %s. ", @@ -728,6 +736,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # don't have a ``_recving_metadata`` entry to invalidate, so # we just release the handle and let the engine reschedule # via the lease / watchdog. + with self._sending_transfers_lock: + self._send_failures.add(request_id) if not self._handle_failed_transfer(request_id, handle): return handle return None @@ -811,13 +821,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): done_pushing, failed_pushing = self._pop_done_transfers( self._sending_transfers ) - # A failed send must never be reported as done: its blocks - # are freed via the lease / watchdog instead. - done_pushing = { - req_id - for req_id in done_pushing - failed_pushing - if req_id in self._recving_metadata - } + # Remember failures until the final sibling WRITE completes. + self._send_failures.update(failed_pushing) + successful = { + req_id + for req_id in done_pushing - self._send_failures + if req_id in self._reqs_to_send or req_id in self._reqs_to_process + } + self._send_failures.difference_update(done_pushing | done_sending) + # Expired requests were already reported, even if their WRITEs finish later. + done_pushing = successful for req_id in done_pushing: self._reqs_to_send.pop(req_id, None) self._reqs_to_process.discard(req_id) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 6dcfcb5ec0..68318a16da 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -247,12 +247,22 @@ class OffloadingEventsTracker: the underlying :class:`OffloadingEvent` stream. """ + removed_keys: set[OffloadKey] = set() for event in events: if event.removed: + if self.self_describing_enabled: + removed_keys.update(event.keys) yield from self._take_removed_event(event) else: yield from self._take_stored_event(event) + # A primary removal can precede a queued secondary store in this batch. + # Keep its payload until all stores have registered their residencies. + for key in removed_keys: + meta = self._pending_event_metadata.get(key) + if meta is not None and not meta.active_residencies: + self._pending_event_metadata.pop(key) + def reset(self) -> None: """Drop all tracked state; pending payloads are stale after a manager cache reset.""" @@ -407,8 +417,6 @@ class OffloadingEventsTracker: maybe_convert_block_hash(h) for h in meta.block_hashes ) meta.active_residencies.discard((event.medium, event.ownership)) - if not meta.active_residencies: - self._pending_event_metadata.pop(key) else: if self.self_describing_enabled: logger.warning_once( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py index 7357088175..def6e1e1f3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py @@ -131,19 +131,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA): pass def wait_for_save(self): - # Store deferral is handled in get_finished(), which always runs even - # when wait_for_save() is skipped (e.g. kv_connector_no_forward). - pass + assert self.connector_worker is not None + assert isinstance(self._connector_metadata, OffloadingConnectorMetadata) + # Defer store jobs to the next step's start_kv_transfers. + self.connector_worker.prepare_store_kv(self._connector_metadata) def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]: assert self.connector_worker is not None assert isinstance(self._connector_metadata, OffloadingConnectorMetadata) - - # Defer store jobs to the next step's start_kv_transfers. Done here - # (rather than wait_for_save) so stores are queued even on steps where - # wait_for_save is skipped. - self.connector_worker.prepare_store_kv(self._connector_metadata) - return self.connector_worker.get_finished(finished_req_ids) def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 64bf8146c9..9124d9eac1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput @@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import ( from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + SimpleCPUOffloadPromMetrics, + SimpleCPUOffloadStats, +) from vllm.v1.simple_kv_offload.worker import ( SimpleCPUOffloadWorker, ) @@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): hash_block_size=hash_block_size, lazy_offload=lazy_offload, disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0, + use_page_cache=use_page_cache if disk_mode else False, ) elif role == KVConnectorRole.WORKER: self.worker_handler = SimpleCPUOffloadWorker( @@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.get_boundary_store_stats() return None + def get_kv_connector_stats(self) -> KVConnectorStats | None: + # Worker-side callers may invoke this hook on connectors constructed + # via __new__ (as in test_worker.py), which lack scheduler_manager. + scheduler_manager: SimpleCPUOffloadScheduler | None = getattr( + self, "scheduler_manager", None + ) + if scheduler_manager is not None: + return scheduler_manager.get_stats() + return None + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + SimpleCPUOffloadStats(data=data) + if data is not None + else SimpleCPUOffloadStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return SimpleCPUOffloadPromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def take_events(self) -> Iterable[KVCacheEvent]: if self.scheduler_manager is not None: return self.scheduler_manager.take_events() diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 20118649e1..88bb3d7c5a 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator: return _DCP +def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]: + """Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled + (e.g. a replicated draft cache) or the group is uninitialized (unit tests).""" + if not enabled or _DCP is None: + return 1, 0 + return _DCP.world_size, _DCP.rank_in_group + + _PP: GroupCoordinator | None = None diff --git a/vllm/distributed/weight_transfer/factory.py b/vllm/distributed/weight_transfer/factory.py index 48f9e4f212..82228bef34 100644 --- a/vllm/distributed/weight_transfer/factory.py +++ b/vllm/distributed/weight_transfer/factory.py @@ -243,6 +243,12 @@ WeightTransferEngineFactory.register_engine( "ShardedRDTWeightTransferEngine", ) +WeightTransferEngineFactory.register_engine( + "nccl_m2n", + "vllm.distributed.weight_transfer.m2n_engine", + "M2NWeightTransferEngine", +) + # Trainer-side engines, parallel to the worker registry above. WeightTransferTrainerFactory.register_engine( @@ -268,3 +274,9 @@ WeightTransferTrainerFactory.register_engine( "vllm.distributed.weight_transfer.sharded_rdt_trainer", "ShardedRDTTrainerWeightTransferEngine", ) + +WeightTransferTrainerFactory.register_engine( + "nccl_m2n", + "vllm.distributed.weight_transfer.m2n_trainer", + "M2NTrainerWeightTransferEngine", +) diff --git a/vllm/distributed/weight_transfer/m2n_common.py b/vllm/distributed/weight_transfer/m2n_common.py new file mode 100644 index 0000000000..a2c6b73844 --- /dev/null +++ b/vllm/distributed/weight_transfer/m2n_common.py @@ -0,0 +1,338 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend. + +M2N reshards a tensor between two disjoint meshes of ranks that live in one +communicator: the trainer occupies ranks `[0, T)` and the inference workers +`[T, T + N)`. This module holds everything both sides need — the optional +runtime import, layout descriptors, and the conversion to `nccl.m2n` types — +so the engine module stays about the transfer itself. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, cast + +import torch + +from vllm.distributed.weight_transfer.base import ParamMeta + +if TYPE_CHECKING: + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + +# Placement code for a mesh axis that replicates. Any other (non-negative) code +# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so +# layouts survive the JSON init handshake without a custom encoder. +REPLICATE = -1 + +# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library. +MAX_TENSOR_DIMS = 3 + +MESH_NDIMS = 2 + +# Destination mesh convention. Keeping these roles named avoids scattering +# positional axis assumptions through the worker-side planner. +DESTINATION_REPLICA_AXIS = 0 +DESTINATION_SHARD_AXIS = 1 + +# m2n bounds how many source shards may feed one destination shard, and how many +# destination shards one source shard may feed, with compile-time arrays in +# `reshard_limits.h`. The bindings do not expose them, so they are mirrored +# here; a build with larger arrays makes these conservative, never wrong. +MAX_SOURCE_SHARDS = 16 +MAX_DEST_SHARDS = 64 + +# Wire dtypes `ncclReshard` accepts. Notably excludes fp4 and any packed +# sub-byte type, so quantized checkpoints are out of scope for now. +SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset( + dtype + for dtype in ( + getattr(torch, name, None) + for name in ( + "int8", + "uint8", + "float8_e4m3fn", + "float8_e5m2", + "float16", + "bfloat16", + "int32", + "uint32", + "float32", + "int64", + "uint64", + "float64", + ) + ) + if dtype is not None +) + +_IMPORT_HINT = ( + "The nccl_m2n weight transfer backend requires the `nccl-extensions` " + "package (and its `nccl4py` dependency), which is not installed by vLLM. " + "See https://github.com/NVIDIA/nccl-extensions for build " + "and install instructions. It needs NCCL 2.30.5 or newer, and " + "VLLM_NCCL_SO_PATH must point at the same libnccl.so that libnccl_m2n.so " + "was linked against." +) + + +def import_m2n() -> Any: + """Import `nccl.m2n` lazily, with an actionable error when it is missing. + + Deferred so that importing vLLM — or any other weight transfer backend — + never requires the m2n runtime to be present. + """ + try: + import nccl.m2n as m2n + except ImportError as e: + raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e + return m2n + + +@dataclass(frozen=True) +class M2NMesh: + """One side's rank topology — pure topology, no tensor placement. + + Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval + `[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a + single-axis topology is spelled with a second axis of size 1. + + One mesh describes every tensor on its side, so it is exchanged once at the + init handshake rather than per parameter. + """ + + dims: tuple[int, int] + start_rank: int + + def __post_init__(self) -> None: + if len(self.dims) != MESH_NDIMS or any(d <= 0 for d in self.dims): + raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints") + if self.start_rank < 0: + raise ValueError( + f"mesh start_rank must be non-negative, got {self.start_rank}" + ) + + @property + def size(self) -> int: + return self.dims[0] * self.dims[1] + + +# A tensor's placement over its side's mesh: one code per mesh axis +# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a +# tensor every rank holds in full. +Placements = tuple[int, int] +REPLICATED: Placements | None = None + + +def check_placements(placements: Placements, context: str = "placements") -> None: + """Reject placement pairs m2n cannot express. + + The header requires exactly one SHARD axis and one REPLICATE axis. + `{REPLICATE, REPLICATE}` hits a degenerate prepare branch, which is why + full replication is carried as `REPLICATED` and resolved separately; + sharding both axes is not expressible at all. + """ + if len(placements) != MESH_NDIMS: + raise ValueError(f"{context} must have {MESH_NDIMS} entries") + invalid = [code for code in placements if code < REPLICATE] + if invalid: + raise ValueError( + f"{context} contains invalid placement code {invalid[0]}; valid " + f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions" + ) + num_sharded = sum(code != REPLICATE for code in placements) + if num_sharded == 0: + raise ValueError( + f"{context}: a fully replicated tensor is carried as REPLICATED, " + "not as two " + "REPLICATE axes" + ) + if num_sharded == MESH_NDIMS: + raise ValueError( + f"{context}: nccl_m2n needs one REPLICATE mesh axis, but " + f"{placements} shards both" + ) + + +def resolve_layout( + mesh: M2NMesh, + placements: Placements | None, + context: str = "placements", +) -> tuple[M2NMesh, Placements]: + """Pair one tensor's placement with the mesh m2n should see for it. + + A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since + m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the + *same rank interval* re-factored as `(size, 1)`. That re-factoring is sound + precisely because replication is order-independent — every rank holds the + whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the + interval in a different order. A sharded tensor keeps its side's own + factorization, where rank order decides who owns which shard. + """ + if placements is None: + return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0) + check_placements(placements, context) + return mesh, placements + + +def shard_count(mesh: M2NMesh, placements: Placements) -> int: + """How many pieces this layout splits the tensor into.""" + return next( + (mesh.dims[axis] for axis, code in enumerate(placements) if code != REPLICATE), + 1, + ) + + +def check_plan_limits( + src: tuple[M2NMesh, Placements], + dst: tuple[M2NMesh, Placements], + name: str, +) -> None: + """Reject plans that exceed m2n's static per-shard fan-in / fan-out arrays. + + Only the unambiguous cases are checked: when one side is a single shard it + is fed by (or feeds) every shard on the other side, so the count is exactly + the other side's shard count. In the general sharded-to-sharded case the + overlap depends on the library's chunking, and reproducing that arithmetic + here would duplicate internals that can change under us. + + m2n re-checks authoritatively and, because the plan is derived from the + shared descriptors, fails identically on every rank -- so this is about + reporting at init with a message that names the parameter, not about + avoiding a hang. + """ + src_shards = shard_count(*src) + dst_shards = shard_count(*dst) + if dst_shards == 1 and src_shards > MAX_SOURCE_SHARDS: + raise ValueError( + f"parameter '{name}' would feed {src_shards} source shards into one " + f"replicated destination, over m2n's MAX_SOURCES={MAX_SOURCE_SHARDS}. " + "Shard the destination, replicate the source, or rebuild m2n with " + "larger arrays." + ) + if src_shards == 1 and dst_shards > MAX_DEST_SHARDS: + raise ValueError( + f"parameter '{name}' would fan one source shard out to {dst_shards} " + f"destination shards, over m2n's MAX_TARGETS={MAX_DEST_SHARDS}. " + "Rebuild m2n with larger arrays to raise the bound." + ) + + +def validate_layout( + mesh: M2NMesh, placements: Placements, shape: Sequence[int], side: str +) -> None: + """Check a resolved layout can describe `shape` before any collective runs. + + Called on both sides at init so a bad plan surfaces as an error from the + init RPC rather than as a hang inside the first reshard. + """ + for axis, code in enumerate(placements): + if code == REPLICATE: + continue + if code >= len(shape): + raise ValueError( + f"{side} layout shards tensor dim {code}, but the tensor " + f"has rank {len(shape)}" + ) + factor = mesh.dims[axis] + if shape[code] % factor: + raise ValueError( + f"{side} layout shards dim {code} (size {shape[code]}) over " + f"{factor} ranks, which does not divide evenly" + ) + + +@dataclass(frozen=True) +class M2NParamMeta(ParamMeta): + """`ParamMeta` extended with how the trainer places this tensor. + + The base class carries only name / dtype / full shape, which is not enough + to plan a reshard. `placements` is relative to its side's `M2NMesh`, or + `REPLICATED` when every rank holds the whole tensor. + """ + + placements: Placements | None + + +def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None: + """Reject tensors m2n cannot move at all, with the parameter named.""" + if dtype not in SUPPORTED_DTYPES: + raise ValueError( + f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not " + f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}" + ) + if not 1 <= len(shape) <= MAX_TENSOR_DIMS: + raise ValueError( + f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports " + f"rank 1..{MAX_TENSOR_DIMS}" + ) + + +def to_mesh(m2n: Any, mesh: M2NMesh) -> Any: + return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank) + + +def to_placements(m2n: Any, placements: Placements) -> list[Any]: + return [ + m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements + ] + + +def publish_destination_placements( + comm: "PyNcclCommunicator", + first_worker_rank: int, + placements: "Sequence[Placements | None] | None", + num_parameters: int, +) -> list[Placements | None]: + """Share the worker-side destination plan with every rank in the group. + + The trainer must issue each reshard with the same destination the workers + use, but once destinations are per-parameter they depend on the inference + model, which only the workers can see. The first worker publishes them here, + over the shared NCCL communicator; trainer ranks pass `None` and receive + them, and the other workers pass their own so a disagreement is caught + rather than deadlocking later. + + Only placements travel: the destination *mesh* is still derived from the + rank split, identically on both sides. Use the NCCL communicator directly + because unique-id rendezvous deliberately has no bootstrap process group. + """ + replicated_sentinel = REPLICATE - 1 + encoded = torch.full( + (num_parameters, MESH_NDIMS), + replicated_sentinel, + dtype=torch.int8, + device=comm.device, + ) + if comm.rank == first_worker_rank: + if placements is None or len(placements) != num_parameters: + raise ValueError( + f"publishing rank needs {num_parameters} destination placements" + ) + rows = [ + [replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement + for placement in placements + ] + if rows: + encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=comm.device)) + + comm.broadcast(encoded, src=first_worker_rank) + return [ + REPLICATED + if all(code == replicated_sentinel for code in row) + else cast(Placements, tuple(row)) + for row in encoded.tolist() + ] + + +def comm_ptr(comm: "PyNcclCommunicator") -> int: + """Raw `ncclComm_t` behind vLLM's `PyNcclCommunicator`. + + m2n links its own NCCL, so the handle only means anything if vLLM loaded + the same `libnccl.so` — set `VLLM_NCCL_SO_PATH` accordingly. + """ + handle = comm.comm + ptr = getattr(handle, "value", handle) + if not ptr: + raise RuntimeError("PyNcclCommunicator has no live NCCL communicator") + return int(ptr) diff --git a/vllm/distributed/weight_transfer/m2n_engine.py b/vllm/distributed/weight_transfer/m2n_engine.py new file mode 100644 index 0000000000..c61bac4f2d --- /dev/null +++ b/vllm/distributed/weight_transfer/m2n_engine.py @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`). + +The trainer and the inference workers share one NCCL communicator: trainer ranks +occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single +`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP +/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its +local shards and never all-gathers a full tensor, which is what the broadcast +NCCL backend forces it to do. + +When an incoming checkpoint parameter maps directly to a live vLLM parameter, +M2N reshards into that worker's local model storage. Parameters whose names or +layouts cannot be resolved receive a full tensor and fall back to +`load_weights`, preserving the existing backend's loading behavior. + +Both meshes participate in every reshard, in the same order, so this backend has +the same concurrency shape as the broadcast NCCL backend: the worker must be +inside `receive_weights` while the trainer is sending. Driving that from the +trainer is the trainer engine's job. +""" + +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, cast + +import torch + +from vllm.config.weight_transfer import WeightTransferConfig +from vllm.distributed.weight_transfer.base import ( + WeightTransferEngine, + WeightTransferInitInfo, + WeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.m2n_common import ( + DESTINATION_SHARD_AXIS, + MESH_NDIMS, + REPLICATED, + M2NMesh, + M2NParamMeta, + Placements, + check_plan_limits, + check_transferable, + comm_ptr, + import_m2n, + publish_destination_placements, + resolve_layout, + to_mesh, + to_placements, + validate_layout, +) +from vllm.distributed.weight_transfer.m2n_layout import ( + M2NDestination, + resolve_parameter_destinations, +) +from vllm.distributed.weight_transfer.nccl_common import ( + NCCLWeightTransferInitInfo, + decode_nccl_unique_id, + worker_init_process_group, +) + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + +__all__ = [ + "M2NWeightTransferInitInfo", + "M2NWeightTransferUpdateInfo", + "M2NWeightTransferEngine", +] + + +# --------------------------------------------------------------------------- +# Wire types +# --------------------------------------------------------------------------- + + +@dataclass(kw_only=True) +class M2NWeightTransferInitInfo(WeightTransferInitInfo): + """Worker-side init info: the rendezvous plus the full transfer plan. + + Layouts are static for the whole run, so they ride the one-time init + handshake and the per-round update info stays a list of names. Everything is + plain JSON so the HTTP control plane carries it unchanged. + """ + + master_address: str | None = None + master_port: int | None = None + rank_offset: int + """First worker rank, i.e. the number of trainer ranks.""" + world_size: int + """Trainer ranks + all inference workers.""" + src_mesh_dims: list[int] + """The trainer's mesh, shared by every parameter (`start_rank` is 0: the + trainer occupies the front of the communicator).""" + dst_mesh_dims: list[int] + """The inference mesh, starting at `rank_offset`. Declared rather than + derived so both sides describe the destination identically.""" + names: list[str] + dtype_names: list[str] + shapes: list[list[int]] + src_placements: list[list[int] | None] + """Per parameter, relative to `src_mesh_dims`; `None` means replicated.""" + nccl_unique_id_b64: str | None = field(default=None, repr=False) + max_cta: int | None = None + + def __post_init__(self) -> None: + _ = self.nccl_unique_id_bytes + num_params = len(self.names) + for label, values in ( + ("dtype_names", self.dtype_names), + ("shapes", self.shapes), + ("src_placements", self.src_placements), + ): + if len(values) != num_params: + raise ValueError( + f"`{label}` should be of the same size as `names`: " + f"got {len(values)} and {num_params}" + ) + if self.rank_offset < 1 or self.rank_offset >= self.world_size: + raise ValueError( + f"`rank_offset` ({self.rank_offset}) must leave at least one " + f"trainer rank and one worker in world_size {self.world_size}" + ) + num_workers = self.world_size - self.rank_offset + dst = tuple(self.dst_mesh_dims) + if len(dst) != MESH_NDIMS or dst[0] * dst[1] != num_workers: + raise ValueError( + f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the " + f"{num_workers} inference workers" + ) + + @property + def nccl_unique_id_bytes(self) -> bytes | None: + return decode_nccl_unique_id( + master_address=self.master_address, + master_port=self.master_port, + nccl_unique_id_b64=self.nccl_unique_id_b64, + ctx="M2NWeightTransferInitInfo", + ) + + +@dataclass +class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo): + """Per-round update info: which parameters this chunk carries, in order. + + Shapes, dtypes and layouts were fixed at init, so a round only needs to say + what is coming and in what order — both sides must issue their reshards in + exactly that order. + """ + + names: list[str] + + +# --------------------------------------------------------------------------- +# Worker engine +# --------------------------------------------------------------------------- + + +class M2NWeightTransferEngine( + WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo] +): + """Inference-side engine: receives each parameter with one reshard. + + Resolvable parameters are resharded from the trainer layout directly into + each worker's live local model storage. Unresolvable parameters receive a + full tensor and use `load_weights`. In both cases, the trainer sends its + local shards without materializing a full tensor. + """ + + init_info_cls = M2NWeightTransferInitInfo + update_info_cls = M2NWeightTransferUpdateInfo + # The destination plan is resolved against the model this engine was built + # for, so it cannot be retargeted at a draft model mid-flight. + supports_draft_weight_update = False + + def __init__( + self, + config: WeightTransferConfig, + vllm_config: "VllmConfig", + device: torch.device, + model: torch.nn.Module, + ) -> None: + super().__init__(config, vllm_config, device, model) + self.model_update_group: PyNcclCommunicator | None = None + self._m2n: Any = None + self._handle: Any = None + self._metas: list[M2NParamMeta] = [] + self._src_mesh: M2NMesh | None = None + self._dst_mesh: M2NMesh | None = None + self._parameter_destinations: list[M2NDestination] = [] + self._index: dict[str, int] = {} + self._uses_load_weights = False + + def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None: + """Join the trainer's communicator and rebuild the transfer plan. + + Every precondition (dtype, tensor rank, divisibility) is checked here so + a bad plan fails the init RPC instead of hanging the first collective. + """ + self._m2n = import_m2n() + self._metas = [] + + self._src_mesh = M2NMesh( + cast(tuple[int, int], tuple(init_info.src_mesh_dims)), 0 + ) + if self._src_mesh.size != init_info.rank_offset: + raise ValueError( + f"source mesh covers {self._src_mesh.size} ranks, but there are " + f"{init_info.rank_offset} trainer ranks" + ) + # The inference topology, as declared by the trainer. This is the mesh + # a sharded destination is placed over; a replicated one is described + # by the alternate descriptor `resolve_layout` derives from it, which + # covers the same ranks but is not this mesh. + self._dst_mesh = M2NMesh( + cast(tuple[int, int], tuple(init_info.dst_mesh_dims)), + init_info.rank_offset, + ) + + for name, name_dtype, shape, placements in zip( + init_info.names, + init_info.dtype_names, + init_info.shapes, + init_info.src_placements, + ): + dtype = getattr(torch, name_dtype, None) + if not isinstance(dtype, torch.dtype): + raise ValueError( + f"parameter '{name}' has invalid dtype name " + f"'{name_dtype}'; expected the name of a torch.dtype" + ) + check_transferable(name, dtype, shape) + codes = ( + REPLICATED + if placements is None + else cast(Placements, tuple(placements)) + ) + src_mesh, src_placements = resolve_layout( + self._src_mesh, codes, f"parameter '{name}' source placements" + ) + validate_layout(src_mesh, src_placements, shape, "source") + # Destination checks are deferred until loader-aware resolution; + # assuming replication here can reject a valid sharded plan. + self._metas.append(M2NParamMeta(name, dtype, tuple(shape), codes)) + + parallel_config = self.parallel_config + shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS] + quantization = getattr(self.model_config, "quantization", None) + # A quantized weight loader may dequantize / transpose / requantize, and + # under PP a rank's local shape is not the checkpoint shape split over + # the shard axis. Both break shape-derived resolution, so take the + # fallback. + allow_direct = ( + quantization is None and parallel_config.pipeline_parallel_size == 1 + ) + # Match every incoming checkpoint parameter to its rank-local target. + # Resolvable parameters can be resharded directly into live model + # storage; the rest receive a full tensor for `load_weights`. + self._parameter_destinations = resolve_parameter_destinations( + self.model, + [m.name for m in self._metas], + [m.dtype for m in self._metas], + [m.shape for m in self._metas], + num_workers=self._dst_mesh.size, + shard_axis_size=shard_axis_size, + allow_direct=allow_direct, + ) + # Check the resolved layout rather than a provisional replicated one. + # For example, 32 source shards feeding 4 destination shards may need + # only 8 sources per destination, while a replicated plan needs all 32. + for meta, destination in zip(self._metas, self._parameter_destinations): + src_mesh, src_placements = resolve_layout( + self._src_mesh, + meta.placements, + f"parameter '{meta.name}' source placements", + ) + dst_mesh, dst_placements = resolve_layout( + self._dst_mesh, + destination.placements, + f"parameter '{meta.name}' destination placements", + ) + validate_layout(dst_mesh, dst_placements, meta.shape, "destination") + check_plan_limits( + (src_mesh, src_placements), + (dst_mesh, dst_placements), + meta.name, + ) + + # Update requests carry names only, so cache their plan indices. Track + # whether any fallback entry requires the `load_weights` lifecycle. + self._index = {meta.name: i for i, meta in enumerate(self._metas)} + self._uses_load_weights = any( + not destination.direct for destination in self._parameter_destinations + ) + + self.model_update_group = worker_init_process_group( + NCCLWeightTransferInitInfo( + master_address=init_info.master_address, + master_port=init_info.master_port, + nccl_unique_id_b64=init_info.nccl_unique_id_b64, + rank_offset=init_info.rank_offset, + world_size=init_info.world_size, + ), + parallel_config, + ) + # Destinations are per-parameter and depend on the inference model, so + # the trainer cannot derive them. The first worker publishes the plan + # and every other rank checks its own against it — a silent + # disagreement between workers would mean mismatched collectives. + mine = [destination.placements for destination in self._parameter_destinations] + agreed = publish_destination_placements( + self.model_update_group, init_info.rank_offset, mine, len(self._metas) + ) + if agreed != mine: + mismatched = next( + meta.name for meta, a, b in zip(self._metas, mine, agreed) if a != b + ) + raise RuntimeError( + "inference workers resolved different nccl_m2n destinations " + f"(first mismatch: '{mismatched}'); all workers must run the " + "same model and parallel config" + ) + + self._handle = self._m2n.Handle.create( + self._m2n.Config(max_cta=init_info.max_cta) + ) + + def start_weight_update(self) -> None: + """Set up layerwise reloading, but only if some parameter needs it. + + Directly-resharded parameters are written in place and never go through + `load_weights`, so a plan with no fallback entries has nothing to + reload. + """ + if not self._uses_load_weights: + return + from vllm.model_executor.model_loader.reload import initialize_layerwise_reload + + initialize_layerwise_reload(self.model) + + def finish_weight_update(self) -> None: + """Finalize layerwise reloading when the plan uses fallback entries.""" + if not self._uses_load_weights: + return + from vllm.model_executor.model_loader.reload import finalize_layerwise_reload + + finalize_layerwise_reload(self.model, self.model_config) + + def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None: + """Receive each requested parameter using its initialization-time plan.""" + if self._handle is None or self.model_update_group is None: + raise RuntimeError( + "nccl_m2n weight transfer not initialized. " + "Call init_transfer_engine() first." + ) + + requested = [] + for name in update_info.names: + index = self._index.get(name) + if index is None: + raise ValueError( + f"parameter '{name}' was not declared at init; the " + "trainer must send the same parameter set it announced" + ) + requested.append((name, index)) + + from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, + ) + + comm = comm_ptr(self.model_update_group) + stream = torch.cuda.current_stream() + + # Reshard lazily so one `load_weights` invocation sees the complete + # fallback sequence without staging every full tensor at once. Direct + # destinations still execute in request order as the iterator advances. + def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]: + for name, index in requested: + meta = self._metas[index] + destination = self._parameter_destinations[index] + buffer = destination.tensor + if buffer is None: + buffer = torch.empty( + meta.shape, dtype=meta.dtype, device=self.device + ) + + self._reshard(comm, stream, meta, destination.placements, buffer) + + if destination.tensor is None: + # `load_weights` reads on the host stream, so the transfer + # has to have landed before it runs. + stream.synchronize() + yield name, buffer + + has_fallback = any( + self._parameter_destinations[index].tensor is None for _, index in requested + ) + with disable_mtp_completeness_check(): + received_weights = reshard_requested_weights() + if has_fallback: + # Some model loaders keep invocation-local state to combine + # related checkpoint tensors, so preserve one iterable scope. + self.model.load_weights(received_weights) + else: + # The iterator performs the reshards. With nothing to yield to + # `load_weights`, exhaust it here to execute direct transfers. + for _ in received_weights: + pass + + def _reshard( + self, + comm: int, + stream: torch.cuda.Stream, + src_meta: M2NParamMeta, + dst_placements: Placements | None, + dst_buffer: torch.Tensor, + ) -> None: + """Reshard one parameter from its trainer layout into a worker buffer. + + `dst_placements` describes the buffer's logical placement over the + worker mesh; `dst_buffer` is this rank's physical storage. + """ + assert self._src_mesh is not None + assert self._dst_mesh is not None + m2n = self._m2n + src_mesh, resolved_src_placements = resolve_layout( + self._src_mesh, src_meta.placements + ) + dst_mesh, resolved_dst_placements = resolve_layout( + self._dst_mesh, dst_placements + ) + m2n.reshard( + None, + dst_buffer, + comm, + stream, + src_mesh=to_mesh(m2n, src_mesh), + src_placements=to_placements(m2n, resolved_src_placements), + dst_mesh=to_mesh(m2n, dst_mesh), + dst_placements=to_placements(m2n, resolved_dst_placements), + handle=self._handle, + ) + + def shutdown(self) -> None: + """Finish pending GPU work and release M2N communication state.""" + if self._handle is not None: + # M2N does not synchronize caller streams on finalize. + torch.accelerator.synchronize() + self._handle.destroy() + self._handle = None + self.model_update_group = None + self._parameter_destinations = [] diff --git a/vllm/distributed/weight_transfer/m2n_layout.py b/vllm/distributed/weight_transfer/m2n_layout.py new file mode 100644 index 0000000000..3296ef63fb --- /dev/null +++ b/vllm/distributed/weight_transfer/m2n_layout.py @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Destination-layout resolution for the NCCL M2N weight transfer backend. + +For every incoming checkpoint parameter the worker needs a destination buffer +plus its placement over the inference mesh. Two outcomes are possible: + +* **direct** — when the model is not quantized, the parameter maps 1:1 onto a + live vLLM parameter and its loader is a small, known-safe copy or tensor- + parallel loader. The loader's declared input/output dimension determines the + placement; shapes are validation, not an inference mechanism. The reshard + writes straight into the live parameter, so each rank receives only its own + shard and nothing is copied afterwards. +* **fallback** — anything else. The reshard delivers the whole tensor to every + rank and `load_weights` does the sharding, exactly as the broadcast NCCL + backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`) + take this path: the checkpoint name does not name a vLLM parameter, so there + is nothing to resolve against. + +Correctness never depends on a parameter resolving — the fallback is always +available and is the same path the existing backend uses. +""" + +import inspect +import math +from collections.abc import Sequence +from dataclasses import dataclass +from functools import partial +from typing import cast + +import torch + +from vllm.distributed.weight_transfer.m2n_common import ( + DESTINATION_REPLICA_AXIS, + DESTINATION_SHARD_AXIS, + MESH_NDIMS, + REPLICATE, + REPLICATED, + Placements, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_DIRECT_WEIGHT_LOADING_BLOCKLIST = { + "vllm.model_executor.models.gpt2.GPT2Model", +} + + +def _module_type_name(module: torch.nn.Module) -> str: + cls = type(module) + return f"{cls.__module__}.{cls.__name__}" + + +def _destination_placements(shard_dim: int) -> Placements: + """Place one tensor shard on the configured destination mesh axis.""" + placements = [REPLICATE] * MESH_NDIMS + placements[DESTINATION_REPLICA_AXIS] = REPLICATE + placements[DESTINATION_SHARD_AXIS] = shard_dim + return cast(Placements, tuple(placements)) + + +@dataclass +class M2NDestination: + """Where one checkpoint parameter lands on this worker.""" + + name: str + placements: Placements | None + """Placement over the inference mesh, or `REPLICATED` for the fallback.""" + tensor: torch.Tensor | None + """The live parameter view to reshard into, or None for the fallback path + (the engine allocates a full replica per round and calls `load_weights`).""" + + @property + def direct(self) -> bool: + return self.tensor is not None + + +def _base_loader_shard_dim( + param: torch.nn.Parameter, + shard_axis_size: int, +) -> int | None: + """Return a known-safe loader's declared shard dim, or reject it. + + Loader identity is deliberately fail-closed. A missing loader is not + treated as ``default_weight_loader`` because a model-level ``load_weights`` + implementation may transform the tensor before it reaches that default. + Wrappers and partials are also rejected: recognizing their wrapped callable + would ignore behavior added by the wrapper. + """ + loader = getattr(param, "weight_loader", None) + if loader is None or isinstance(loader, partial): + return None + + owner = loader.__self__ if inspect.ismethod(loader) else None + loader_fn = loader.__func__ if inspect.ismethod(loader) else loader + + # Lazy imports avoid making the weight-transfer registry import all model + # executor layers during normal vLLM startup. + from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, + ) + from vllm.model_executor.model_loader.weight_utils import default_weight_loader + + replicated_loaders = { + default_weight_loader, + ReplicatedLinear.weight_loader, + } + column_loaders = { + ColumnParallelLinear.weight_loader, + ColumnParallelLinear.weight_loader_v2, + } + row_loaders = { + RowParallelLinear.weight_loader, + RowParallelLinear.weight_loader_v2, + } + + # These attributes signal packing, a transform, or pre-sharded checkpoint + # storage. Such semantics cannot be reproduced by a plain M2N placement. + # Check them before accepting replicated loaders too: model-level loading + # may transform a tensor before handing it to an otherwise plain copier. + metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size") + flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed") + if any(getattr(param, attr, None) is not None for attr in metadata_attrs): + return None + if any(bool(getattr(param, attr, False)) for attr in flag_attrs): + return None + + if loader_fn in replicated_loaders: + return REPLICATE + if loader_fn not in column_loaders | row_loaders: + return None + + if owner is None or getattr(owner, "tp_size", None) != shard_axis_size: + return None + + attr = "output_dim" if loader_fn in column_loaders else "input_dim" + dim = getattr(param, attr, None) + return dim if isinstance(dim, int) and dim >= 0 else None + + +def _validated_shard_dim( + global_shape: Sequence[int], + local_shape: Sequence[int], + declared_dim: int, + shard_axis_size: int, +) -> int | None: + """Validate a loader-declared shard dimension against both shapes.""" + if len(global_shape) != len(local_shape): + return None + + if declared_dim == REPLICATE: + return REPLICATE if tuple(global_shape) == tuple(local_shape) else None + if declared_dim >= len(global_shape): + return None + for dim, (whole, local) in enumerate(zip(global_shape, local_shape)): + expected = local * shard_axis_size if dim == declared_dim else local + if whole != expected: + return None + return declared_dim + + +def resolve_parameter_destinations( + model: torch.nn.Module, + names: Sequence[str], + dtypes: Sequence[torch.dtype], + shapes: Sequence[Sequence[int]], + *, + num_workers: int, + shard_axis_size: int, + allow_direct: bool, +) -> list[M2NDestination]: + """Build this worker's destination plan, one entry per checkpoint parameter. + + Placements are relative to the inference mesh: axis 0 replicates and axis 1 + shards. `allow_direct=False` forces every parameter onto the fallback path; + the engine sets it for pipeline-parallel or quantized deployments, where a + parameter's local shape is not simply the checkpoint shape split across the + shard axis. Parameters below a model type with known model-level checkpoint + preprocessing also fall back, since bypassing `load_weights` would skip the + transformation. + """ + params = dict(model.named_parameters()) if allow_direct else {} + blocked_param_ids = { + id(param) + for module in model.modules() + if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST + for param in module.parameters() + } + + destinations: list[M2NDestination] = [] + for name, dtype, shape in zip(names, dtypes, shapes): + param = params.get(name) + dim = None + if ( + param is not None + and id(param) not in blocked_param_ids + and param.dtype == dtype + and param.data.is_contiguous() + and num_workers % shard_axis_size == 0 + ): + declared_dim = _base_loader_shard_dim(param, shard_axis_size) + if declared_dim is not None: + dim = _validated_shard_dim( + shape, param.shape, declared_dim, shard_axis_size + ) + + if dim is None: + # A replicated parameter is identical on every rank, so it needs no + # placement of its own — REPLICATED lets resolve_layout spread it + # over all the workers regardless of how the mesh is factored. + destinations.append(M2NDestination(name, REPLICATED, None)) + elif dim == REPLICATE: + assert param is not None + destinations.append(M2NDestination(name, REPLICATED, param.data)) + else: + assert param is not None + destinations.append( + M2NDestination(name, _destination_placements(dim), param.data) + ) + + # Layerwise reload finalizes a module as a unit. Mixing direct and fallback + # parameters within one module can copy stale storage over the direct load. + fallback_modules = { + name.rpartition(".")[0] + for name, destination in zip(names, destinations) + if not destination.direct and name in params + } + destinations = [ + M2NDestination(destination.name, REPLICATED, None) + if destination.direct + and destination.name.rpartition(".")[0] in fallback_modules + else destination + for destination in destinations + ] + + num_direct = sum(d.direct for d in destinations) + parameter_bytes = [ + math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes) + ] + total_bytes = sum(parameter_bytes) + direct_bytes = sum( + nbytes + for destination, nbytes in zip(destinations, parameter_bytes) + if destination.direct + ) + direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0 + logger.info( + "nccl_m2n destination plan: %d/%d parameters resharded directly into " + "the model, %d via full-tensor fallback; direct byte coverage: " + "%d/%d bytes (%.1f%%)", + num_direct, + len(destinations), + len(destinations) - num_direct, + direct_bytes, + total_bytes, + direct_byte_percentage, + ) + return destinations diff --git a/vllm/distributed/weight_transfer/m2n_source.py b/vllm/distributed/weight_transfer/m2n_source.py new file mode 100644 index 0000000000..dd08d24b56 --- /dev/null +++ b/vllm/distributed/weight_transfer/m2n_source.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Trainer-side weight sources for the NCCL M2N backend. + +m2n plans a transfer from both sides' layouts, so the trainer has to say how it +holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not +express. This module supplies the m2n flavor of both, and the DTensor-backed +implementation that covers the common trainer. + +A source declares its mesh once (`mesh()`) and a placement per parameter, since +one topology describes every tensor on the side. +""" + +from collections.abc import Iterator +from typing import Any + +import torch + +from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource +from vllm.distributed.weight_transfer.m2n_common import ( + REPLICATE, + REPLICATED, + M2NMesh, + M2NParamMeta, + Placements, +) + +__all__ = [ + "M2NWeightSource", + "DTensorModuleSource", + "mesh_from_tensor", + "placements_from_tensor", +] + + +class M2NWeightSource(WeightSource): + """A `WeightSource` that also describes how the trainer holds its weights. + + Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a + materialized full tensor: gathering is exactly the cost m2n removes. + """ + + def mesh(self) -> M2NMesh: + """The trainer's rank topology, shared by every parameter.""" + raise NotImplementedError + + def metadata(self) -> list[ParamMeta]: + """Name, dtype, full shape, and trainer placement for each parameter.""" + raise NotImplementedError + + def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]: + """Yield `(name, local shard)` pairs in the same order as `metadata()`.""" + raise NotImplementedError + + +def _placement_code(placement: Any) -> int: + """Map a `torch.distributed` placement onto an m2n placement code.""" + name = type(placement).__name__ + if name == "Replicate": + return REPLICATE + if name == "Shard": + return int(placement.dim) + raise ValueError( + f"nccl_m2n cannot express the {name} placement; only Replicate and " + "Shard are supported" + ) + + +def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh: + """The mesh a parameter lives on, as an `M2NMesh`. + + A plain tensor is identical on every trainer rank, so it spans all of them. + """ + device_mesh = getattr(tensor, "device_mesh", None) + if device_mesh is None: + return M2NMesh((num_trainer_ranks, 1), 0) + + grid = device_mesh.mesh + ranks = grid.flatten().tolist() + if ranks != list(range(len(ranks))): + raise ValueError( + "nccl_m2n requires the trainer's device mesh to cover the " + f"contiguous rank interval [0, {len(ranks)}); got {ranks}" + ) + if grid.ndim > 2: + raise ValueError( + f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D" + ) + dims = tuple(grid.shape) + return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0) + + +def placements_from_tensor(tensor: torch.Tensor) -> Placements | None: + """How a parameter is placed over its mesh, or `REPLICATED`. + + `REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n + cannot express that, and the size-1-axis encoding it needs instead is + applied later by `resolve_layout`, which owns that workaround. + """ + placements = getattr(tensor, "placements", None) + if placements is None: + return REPLICATED + codes = [_placement_code(p) for p in placements] + if all(code == REPLICATE for code in codes): + return REPLICATED + if len(codes) == 1: + return (codes[0], REPLICATE) + return (codes[0], codes[1]) + + +class DTensorModuleSource(M2NWeightSource): + """`M2NWeightSource` over `module.named_parameters()`. + + Covers both the FSDP/DTensor trainer (placement read off each parameter, + local shard yielded via `to_local()`) and the plain replicated trainer, with + no special casing. Trainers with a custom producer (a Megatron export, MoE + re-fusing) subclass `M2NWeightSource` instead. + """ + + def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None: + """Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors.""" + self._module = module + self._num_trainer_ranks = num_trainer_ranks + + def mesh(self) -> M2NMesh: + """The mesh every sharded parameter agrees on. + + Replicated parameters span the trainer without a device mesh of their + own, so they never decide this; a model whose *sharded* parameters + disagree cannot be described by one side mesh and is rejected. + """ + meshes = { + mesh_from_tensor(p, self._num_trainer_ranks) + for _, p in self._module.named_parameters() + if placements_from_tensor(p) is not REPLICATED + } + if len(meshes) > 1: + raise ValueError( + "nccl_m2n needs one mesh per side, but this module's sharded " + f"parameters span several: {sorted(m.dims for m in meshes)}" + ) + return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0) + + def metadata(self) -> list[ParamMeta]: + """Read global shape/dtype and placements without gathering shards.""" + return [ + M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p)) + for name, p in self._module.named_parameters() + ] + + def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]: + """Yield each parameter's local shard (`to_local()`), or the tensor itself.""" + for name, param in self._module.named_parameters(): + to_local = getattr(param, "to_local", None) + yield name, (to_local() if callable(to_local) else param) diff --git a/vllm/distributed/weight_transfer/m2n_trainer.py b/vllm/distributed/weight_transfer/m2n_trainer.py new file mode 100644 index 0000000000..68c62f72db --- /dev/null +++ b/vllm/distributed/weight_transfer/m2n_trainer.py @@ -0,0 +1,414 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Trainer-side weight transfer engine for the NCCL M2N backend. + +Symmetric to `M2NWeightTransferEngine` but in the training process. Every +trainer rank joins the shared communicator and runs every reshard, sending its +own local shard; only rank 0 touches the inference control plane. + +Workers join the communicator during init and run reshard from inside +`update_weights`, so those two RPCs overlap with work on this side. Both run on +a helper thread and are joined afterwards, which keeps that overlap inside the +engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than +in every caller. +""" + +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, cast + +import torch +from typing_extensions import Self + +from vllm.distributed.weight_transfer.base import ( + TrainerInitInfo, + TrainerWeightTransferEngine, + VLLMWeightSyncClient, + WeightSource, +) +from vllm.distributed.weight_transfer.m2n_common import ( + M2NMesh, + M2NParamMeta, + Placements, + check_transferable, + comm_ptr, + import_m2n, + publish_destination_placements, + resolve_layout, + to_mesh, + to_placements, + validate_layout, +) +from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource +from vllm.distributed.weight_transfer.nccl_common import ( + decode_nccl_unique_id, + trainer_init, + uid_init_process_group, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + +logger = init_logger(__name__) + +__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"] + + +def _dtype_name(dtype: torch.dtype) -> str: + """Short name for a dtype, e.g. `bfloat16`, for the worker-side init dict.""" + return str(dtype).split(".")[-1] + + +@dataclass(kw_only=True) +class M2NTrainerInitInfo(TrainerInitInfo): + """Trainer-side init info for nccl_m2n. + + `rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also + its rank in the shared communicator, since the trainer occupies + `[0, num_trainer_ranks)`. Rank 0 drives the control plane. + """ + + backend: ClassVar[str] = "nccl_m2n" + + master_address: str | None = None + master_port: int | None = None + world_size: int + """Trainer ranks + all inference workers.""" + num_trainer_ranks: int = 1 + dst_mesh_dims: tuple[int, int] | None = None + """How the inference ranks are laid out: axis 0 replicates and axis 1 + shards. The trainer declares it so both sides describe the destination + identically; defaults to a flat `(num_workers, 1)`, which is all a + replicated destination needs.""" + max_cta: int | None = None + nccl_unique_id_b64: str | None = field(default=None, repr=False) + + @property + def destination_mesh_dims(self) -> tuple[int, int]: + """`dst_mesh_dims`, or a flat mesh over every inference worker.""" + if self.dst_mesh_dims is not None: + return self.dst_mesh_dims + return (self.world_size - self.num_trainer_ranks, 1) + + def __post_init__(self) -> None: + """Reject rank counts or destination meshes that cannot form a group.""" + _ = self.nccl_unique_id_bytes + if not 0 < self.num_trainer_ranks < self.world_size: + raise ValueError( + f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at " + f"least one worker in world_size {self.world_size}" + ) + num_workers = self.world_size - self.num_trainer_ranks + if self.dst_mesh_dims is not None: + dims = tuple(self.dst_mesh_dims) + if len(dims) != 2 or dims[0] * dims[1] != num_workers: + raise ValueError( + f"`dst_mesh_dims` {dims} must be two dims covering the " + f"{num_workers} inference workers" + ) + if not 0 <= self.rank < self.num_trainer_ranks: + raise ValueError( + f"trainer `rank` ({self.rank}) must be below " + f"`num_trainer_ranks` ({self.num_trainer_ranks})" + ) + + @property + def nccl_unique_id_bytes(self) -> bytes | None: + return decode_nccl_unique_id( + master_address=self.master_address, + master_port=self.master_port, + nccl_unique_id_b64=self.nccl_unique_id_b64, + ctx="M2NTrainerInitInfo", + ) + + +class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]): + """Trainer-side engine: sends every rank's local shard, once per parameter. + + Called on every trainer rank. All ranks join the communicator and run every + reshard; only rank 0 touches the control plane. + """ + + init_info_cls = M2NTrainerInitInfo + + def __init__( + self, + *, + client: VLLMWeightSyncClient, + source: M2NWeightSource, + is_controller: bool, + num_trainer_ranks: int, + ) -> None: + """Hold the client and source; `trainer_init` fills in the group.""" + super().__init__(client=client, source=source, is_sender=is_controller) + self.source: M2NWeightSource = source + self.is_controller = is_controller + self.num_trainer_ranks = num_trainer_ranks + self.group: PyNcclCommunicator | None = None + self._m2n: Any = None + self._handle: Any = None + self._metas: list[M2NParamMeta] = [] + self._src_mesh: M2NMesh | None = None + self._dst_mesh: M2NMesh | None = None + self._dst_placements: list[Placements | None] = [] + self._executor: ThreadPoolExecutor | None = None + + @classmethod + def trainer_init( + cls, + init_info: M2NTrainerInitInfo, + *, + client: VLLMWeightSyncClient, + source: WeightSource | None = None, + ) -> Self: + """Build the engine and rendezvous with the inference side. + + Runs on *every* trainer rank. Rank 0 additionally ships the transfer + plan to the workers and drives the control plane; the other ranks only + build local state and join the communicator. Every rank participates + in every reshard and sends its local shard. + + The ordering here is not arbitrary -- see the comments inline. In + short: validate before anything blocks; start the init RPC without + waiting (workers join the communicator inside that RPC, so it cannot + return until this side has joined too); then join the NCCL + communicator; then create the M2N handle; and only then wait for the RPC + to finish. + """ + # The base WeightSource carries name/dtype/shape but not layout, which + # is not enough to plan a reshard. Fail here rather than at the first + # missing attribute deep inside the send loop. + if not isinstance(source, M2NWeightSource): + raise TypeError( + "nccl_m2n needs per-parameter layouts, so its source must be an " + f"M2NWeightSource (e.g. DTensorModuleSource); got " + f"{type(source).__name__}" + ) + + # Trainer-side weight transfer engine for this rank. `__init__` only + # holds the client and source; the communicator, meshes, and M2N handle + # are filled in below. + engine = cls( + client=client, + source=source, + is_controller=init_info.rank == 0, + num_trainer_ranks=init_info.num_trainer_ranks, + ) + engine._m2n = import_m2n() + + engine._src_mesh = source.mesh() + if engine._src_mesh.size != init_info.num_trainer_ranks: + raise ValueError( + f"source mesh covers {engine._src_mesh.size} ranks, but there " + f"are {init_info.num_trainer_ranks} trainer ranks" + ) + + # Validate the whole plan before any rendezvous. A layout m2n cannot + # express is a mismatched collective at send time, which hangs rather + # than raises -- so every precondition is checked while an exception can + # still propagate to the caller. + engine._metas = cast(list[M2NParamMeta], source.metadata()) + for meta in engine._metas: + check_transferable(meta.name, meta.dtype, meta.shape) + mesh, placements = resolve_layout(engine._src_mesh, meta.placements) + validate_layout(mesh, placements, meta.shape, "source") + + rendezvous: Future | None = None + if engine.is_controller: + engine._executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="m2n-weight-sync" + ) + # Only trainer rank 0 starts the control-plane RPC. Workers join + # the communicator inside it, so submit it first, join below, and + # wait for it at the end. + rendezvous = engine._executor.submit( + client.init_weight_transfer_engine, + engine._worker_init_info(init_info), + ) + + # Every trainer rank joins -- reshard is collective over the whole + # communicator. Trainer ranks take + # [0, num_trainer_ranks) and the workers follow, which is the + # contiguous-interval layout m2n requires of both meshes. + unique_id_bytes = init_info.nccl_unique_id_bytes + if unique_id_bytes is not None: + engine.group = uid_init_process_group( + unique_id_bytes, + rank=init_info.rank, + world_size=init_info.world_size, + device=torch.accelerator.current_device_index(), + ) + else: + assert init_info.master_address is not None + assert init_info.master_port is not None + engine.group = trainer_init( + { + "master_address": init_info.master_address, + "master_port": init_info.master_port, + "world_size": init_info.world_size, + }, + rank=init_info.rank, + ) + + # The trainer declares the inference topology; the worker uses exactly + # this, so neither side has to infer the other's factorization. + engine._dst_mesh = M2NMesh( + init_info.destination_mesh_dims, init_info.num_trainer_ranks + ) + engine._dst_placements = engine._receive_destination_plan( + init_info.num_trainer_ranks + ) + logger.info( + "nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), " + "inference ranks [%d, %d)", + len(engine._metas), + init_info.num_trainer_ranks, + init_info.num_trainer_ranks, + init_info.world_size, + ) + + # Created after the group so a failed rendezvous does not leave a live + # handle behind; shutdown() releases it once caller streams are synced. + engine._handle = engine._m2n.Handle.create( + engine._m2n.Config(max_cta=init_info.max_cta) + ) + + # Now safe to collect: both sides have joined, so the RPC has returned + # or raised. Doing this last means a worker-side init failure surfaces + # here, as an exception from trainer_init, rather than at the first send. + if rendezvous is not None: + rendezvous.result() + return engine + + def _receive_destination_plan( + self, first_worker_rank: int + ) -> list[Placements | None]: + """Receive and validate the inference-side per-parameter placements.""" + if self.group is None or self._dst_mesh is None: + raise RuntimeError( + "destination plan requires the communicator and destination mesh" + ) + + # The first inference worker publishes one placement per parameter. + # Every trainer rank receives the complete plan in this single, + # initialization-time broadcast. + placements = publish_destination_placements( + self.group, first_worker_rank, None, len(self._metas) + ) + # The send loop matches placements to metadata by position, so reject + # an incomplete or oversized plan before entering any M2N collective. + if len(placements) != len(self._metas): + raise RuntimeError( + f"inference side planned {len(placements)} parameters, but " + f"this trainer announced {len(self._metas)}" + ) + # Check every worker-selected destination against the corresponding + # global parameter shape before the first transfer. + for meta, placement in zip(self._metas, placements): + mesh, resolved = resolve_layout(self._dst_mesh, placement) + validate_layout(mesh, resolved, meta.shape, "destination") + return placements + + def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]: + """Handshake payload: rendezvous, both meshes, and the transfer plan.""" + assert self._src_mesh is not None + payload: dict[str, Any] = { + "master_address": init_info.master_address, + "master_port": init_info.master_port, + "nccl_unique_id_b64": init_info.nccl_unique_id_b64, + "rank_offset": init_info.num_trainer_ranks, + "world_size": init_info.world_size, + "src_mesh_dims": list(self._src_mesh.dims), + "dst_mesh_dims": list(init_info.destination_mesh_dims), + "names": [m.name for m in self._metas], + "dtype_names": [_dtype_name(m.dtype) for m in self._metas], + "shapes": [list(m.shape) for m in self._metas], + "src_placements": [ + None if m.placements is None else list(m.placements) + for m in self._metas + ], + "max_cta": init_info.max_cta, + } + return {key: value for key, value in payload.items() if value is not None} + + def send_weights(self) -> None: + """Drive one update round: start, reshard concurrently, then finish.""" + if self._handle is None or self.group is None: + raise RuntimeError("nccl_m2n trainer engine is not initialized") + + update: Future | None = None + if self.is_controller: + self.client.start_weight_update() + # The workers reshard from inside `update_weights`, concurrently + # with the sends below. + assert self._executor is not None + update = self._executor.submit( + self.client.update_weights, + {"names": [m.name for m in self._metas]}, + ) + + try: + self._send() + finally: + if update is not None: + update.result() + + if self.is_controller: + self.client.finish_weight_update() + + def _send(self) -> None: + """Reshard each local shard into its worker-planned destination.""" + assert self.group is not None + assert self._src_mesh is not None + assert self._dst_mesh is not None + m2n = self._m2n + comm = comm_ptr(self.group) + stream = torch.cuda.current_stream() + + for index, (name, tensor) in enumerate(self.source): + meta = self._metas[index] + if name != meta.name: + raise RuntimeError( + f"weight source yielded '{name}' at position {index}, but " + f"its metadata declared '{meta.name}'; the source must " + "iterate in a stable order" + ) + shard = tensor.detach().contiguous() + src_mesh, src_placements = resolve_layout(self._src_mesh, meta.placements) + dst_mesh, dst_placements = resolve_layout( + self._dst_mesh, self._dst_placements[index] + ) + m2n.reshard( + shard, + None, + comm, + stream, + src_mesh=to_mesh(m2n, src_mesh), + src_placements=to_placements(m2n, src_placements), + dst_mesh=to_mesh(m2n, dst_mesh), + dst_placements=to_placements(m2n, dst_placements), + handle=self._handle, + ) + # A custom source may yield temporary storage allocated on another + # stream. M2N retains only its device pointer for asynchronous work, + # so tell the allocator not to reuse that storage until this stream + # has finished with it. For storage allocated on this stream, + # record_stream is effectively a no-op. + shard.record_stream(stream) + + # As in the other weight-transfer backends, wait before returning + # rather than relying on same-stream ordering. The caller may then + # safely mutate parameters or continue work on another stream. + stream.synchronize() + + def shutdown(self) -> None: + """Destroy the m2n handle, join the helper thread, and drop the group.""" + if self._handle is not None: + torch.accelerator.synchronize() + self._handle.destroy() + self._handle = None + if self._executor is not None: + self._executor.shutdown() + self._executor = None + self.group = None diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py index 57a17b9830..a329e456d5 100644 --- a/vllm/distributed/weight_transfer/nccl_common.py +++ b/vllm/distributed/weight_transfer/nccl_common.py @@ -220,11 +220,11 @@ def worker_init_process_group( def trainer_init( init_info: NCCLRendezvous | dict, + rank: int = 0, ) -> "PyNcclCommunicator": """Initialize NCCL process group for trainer-side weight transfer. - The trainer is always rank 0 in the process group. Uses the current - CUDA device (torch.accelerator.current_device_index()). + Uses the current CUDA device (torch.accelerator.current_device_index()). Args: init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or @@ -232,6 +232,9 @@ def trainer_init( - master_address: str - master_port: int - world_size: int + rank: This trainer process's rank in the group. The broadcast backends + have a single trainer at rank 0; multi-rank trainers (m2n) occupy + ranks `[0, num_trainer_ranks)` and the workers start after them. Returns: PyNcclCommunicator for weight transfer. @@ -246,12 +249,11 @@ def trainer_init( master_port = init_info.master_port world_size = init_info.world_size - # Trainer is always rank 0 device = torch.accelerator.current_device_index() return stateless_init_process_group( master_address, master_port, - 0, + rank, world_size, device, ) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 0a5472bce7..3e84f50997 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -8,6 +8,7 @@ import functools import json import os import sys +import warnings from collections.abc import Callable from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass from types import UnionType @@ -50,6 +51,7 @@ from vllm.config import ( KVEventsConfig, KVTransferConfig, LoadConfig, + LoggingConfig, LoRAConfig, MambaConfig, ModelConfig, @@ -84,6 +86,7 @@ from vllm.config.kernel import ( SparseIndexerTopkBackend, ) from vllm.config.load import SafetensorsLoadStrategy +from vllm.config.logging import LogLevel from vllm.config.lora import MaxLoRARanks from vllm.config.mamba import MambaBackendEnum, MambaSSUAlgorithm from vllm.config.model import ( @@ -113,7 +116,7 @@ from vllm.config.scheduler import SchedulerPolicy from vllm.config.utils import get_field from vllm.config.vllm import OptimizationLevel, PerformanceMode from vllm.config.watermarking import WatermarkConfig -from vllm.logger import init_logger, suppress_logging +from vllm.logger import configure_logging_if_needed, init_logger, suppress_logging from vllm.platforms import CpuArchEnum, current_platform from vllm.plugins import load_general_plugins from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized @@ -443,6 +446,20 @@ def get_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: return copy.deepcopy(_compute_kwargs(cls)) +_LOG_CONFIG_FILE_DEPRECATION_MESSAGE = ( + "--log-config-file is deprecated and will be removed in v0.33.0. " + "Use --logging-config.pylogging_config_file instead." +) + + +class DeprecatedLogConfigFileAction(argparse.Action): + """Warn when the legacy ``--log-config-file`` option is used.""" + + def __call__(self, parser, namespace, values, option_string=None): + warnings.warn(_LOG_CONFIG_FILE_DEPRECATION_MESSAGE, stacklevel=1) + setattr(namespace, self.dest, values) + + @dataclass class EngineArgs: """Arguments for vLLM engine.""" @@ -573,6 +590,9 @@ class EngineArgs: max_num_batched_tokens: int | None = None max_num_scheduled_tokens: int | None = None long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold + long_prefill_token_threshold_adaptive: bool = ( + SchedulerConfig.long_prefill_token_threshold_adaptive + ) max_num_seqs: int | None = None max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs max_num_queued_reqs: int | None = None @@ -730,6 +750,10 @@ class EngineArgs: profiler_config: ProfilerConfig = get_field(VllmConfig, "profiler_config") + logging_config: LoggingConfig | None = None + log_level: LogLevel | None = None + log_config_file: str | None = None + kv_transfer_config: KVTransferConfig | None = None kv_events_config: KVEventsConfig | None = None @@ -1548,6 +1572,34 @@ class EngineArgs: **lora_kwargs["enable_moe_shared_loras"], ) + # Logging arguments + logging_group = parser.add_argument_group( + title="LoggingConfig", + description=LoggingConfig.__doc__, + ) + logging_config_kwargs = get_kwargs(VllmConfig)["logging_config"] + logging_config_kwargs["default"] = argparse.SUPPRESS + logging_group.add_argument("--logging-config", **logging_config_kwargs) + logging_group.add_argument( + "--log-level", + choices=get_args(LogLevel), + default=argparse.SUPPRESS, + help=( + "Shortcut for --logging-config.log_level. " + "Overrides that field if both are specified." + ), + ) + logging_group.add_argument( + "--log-config-file", + dest="log_config_file", + default=argparse.SUPPRESS, + metavar="PYLOGGING_CONFIG_FILE", + action=DeprecatedLogConfigFileAction, + help=_LOG_CONFIG_FILE_DEPRECATION_MESSAGE, + ) + # Retain the warning in v0.31.0 and v0.32.0. Remove this option and its + # compatibility mapping in create_logging_config() in v0.33.0. + # Observability arguments observability_kwargs = get_kwargs(ObservabilityConfig) observability_group = parser.add_argument_group( @@ -1643,6 +1695,10 @@ class EngineArgs: "--long-prefill-token-threshold", **scheduler_kwargs["long_prefill_token_threshold"], ) + scheduler_group.add_argument( + "--long-prefill-token-threshold-adaptive", + **scheduler_kwargs["long_prefill_token_threshold_adaptive"], + ) # multi-step scheduling has been removed; corresponding arguments # are no longer supported. scheduler_group.add_argument( @@ -2074,6 +2130,16 @@ class EngineArgs: jit_monitor_verbose=self.jit_monitor_verbose, ) + def create_logging_config(self) -> LoggingConfig: + config = self.logging_config or LoggingConfig() + if self.log_level is not None: + config = dataclasses.replace(config, log_level=self.log_level) + if self.log_config_file is not None: + config = dataclasses.replace( + config, pylogging_config_file=self.log_config_file + ) + return config + def create_engine_config( self, usage_context: UsageContext | None = None, @@ -2083,6 +2149,9 @@ class EngineArgs: NOTE: If VllmConfig is incompatible, we raise an error. """ + logging_config = self.create_logging_config() + configure_logging_if_needed(logging_config) + current_platform.pre_register_and_update() device_config = DeviceConfig(device=cast(Device, current_platform.device_type)) @@ -2489,6 +2558,9 @@ class EngineArgs: policy=self.scheduling_policy, scheduler_cls=self.scheduler_cls, long_prefill_token_threshold=self.long_prefill_token_threshold, + long_prefill_token_threshold_adaptive=( + self.long_prefill_token_threshold_adaptive + ), scheduler_reserve_full_isl=self.scheduler_reserve_full_isl, watermark=self.watermark, prefill_schedule_interval=self.prefill_schedule_interval, @@ -2711,6 +2783,7 @@ class EngineArgs: diffusion_config=diffusion_config, structured_outputs_config=self.structured_outputs_config, observability_config=observability_config, + logging_config=logging_config, compilation_config=compilation_config, kv_transfer_config=self.kv_transfer_config, kv_events_config=self.kv_events_config, diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index c879866dfb..9bd40a9161 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -111,13 +111,53 @@ class AnthropicJsonOutputFormat(BaseModel): type: Literal["json_schema"] = "json_schema" +AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"] +# reasoning_effort sent to the chat template for thinking.type=disabled. +AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort] +AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort] +AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"] + + class AnthropicOutputConfig(BaseModel): """Configuration options for the model's output, such as the output format.""" - effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None + effort: AnthropicEffort | None = None format: AnthropicJsonOutputFormat | None = None +class AnthropicThinkingConfigEnabled(BaseModel): + """Extended thinking with a fixed token budget. + + ``display`` is accepted but ignored: reasoning is always returned. + """ + + type: Literal["enabled"] + budget_tokens: int = Field(ge=1024) + display: AnthropicThinkingDisplay | None = None + + +class AnthropicThinkingConfigDisabled(BaseModel): + type: Literal["disabled"] + + +class AnthropicThinkingConfigAdaptive(BaseModel): + """Extended thinking whose depth the model chooses. + + ``display`` is accepted but ignored: reasoning is always returned. + """ + + type: Literal["adaptive"] + display: AnthropicThinkingDisplay | None = None + + +AnthropicThinkingConfig = Annotated[ + AnthropicThinkingConfigEnabled + | AnthropicThinkingConfigDisabled + | AnthropicThinkingConfigAdaptive, + Field(discriminator="type"), +] + + class AnthropicMessagesRequest(BaseModel): """Anthropic Messages API request.""" @@ -126,6 +166,7 @@ class AnthropicMessagesRequest(BaseModel): max_tokens: int metadata: dict[str, Any] | None = None output_config: AnthropicOutputConfig | None = None + thinking: AnthropicThinkingConfig | None = None stop_sequences: ( Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None ) = None @@ -190,6 +231,19 @@ class AnthropicMessagesRequest(BaseModel): raise ValueError("max_tokens must be positive") return v + @model_validator(mode="after") + def validate_thinking_budget(self) -> "AnthropicMessagesRequest": + # P/D prefill legs are sent with max_tokens=1 and never decode; the + # decode leg carries the client's max_tokens and is still checked. + if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"): + return self + if ( + isinstance(self.thinking, AnthropicThinkingConfigEnabled) + and self.thinking.budget_tokens >= self.max_tokens + ): + raise ValueError("thinking.budget_tokens must be less than max_tokens") + return self + class AnthropicDelta(BaseModel): """Delta for streaming responses.""" diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 6cc1271add..22484c744d 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -10,7 +10,7 @@ import logging import time import uuid from collections.abc import AsyncGenerator -from typing import Any +from typing import Any, get_args import jinja2 from fastapi import Request @@ -22,11 +22,15 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicCountTokensRequest, AnthropicCountTokensResponse, AnthropicDelta, + AnthropicDisabledThinkingEffort, + AnthropicDisabledThinkingEffortOption, + AnthropicEffort, AnthropicError, AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption @@ -47,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.logger import init_logger +from vllm.renderers.hf import HfRenderer, resolve_chat_template from vllm.renderers.online_renderer import OnlineRenderer -logger = logging.getLogger(__name__) +logger = init_logger(__name__) def _build_anthropic_usage( @@ -116,6 +122,7 @@ class AnthropicServingMessages(OpenAIServingChat): enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, + disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto", ): super().__init__( engine_client=engine_client, @@ -139,7 +146,87 @@ class AnthropicServingMessages(OpenAIServingChat): "length": "max_tokens", "tool_calls": "tool_use", } - self._merge_inline_system = self._detect_merge_inline_system(chat_template) + self._merge_inline_system = self._should_merge_inline_system(online_renderer) + # Resolved lazily from the renderer when "auto". + self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( + None if disabled_thinking_effort == "auto" else disabled_thinking_effort + ) + + async def _get_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: + if self._disabled_thinking_effort is None: + self._disabled_thinking_effort = ( + await self._probe_disabled_thinking_effort() + ) + logger.info( + "Anthropic thinking.type=disabled maps to reasoning_effort=%r", + self._disabled_thinking_effort, + ) + return self._disabled_thinking_effort + + async def _probe_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: + """Use ``low`` if the renderer rejects or ignores ``none``. + + ``none`` is ignored when it renders the same prompt as a thinking + effort, e.g. GLM-5.3 treats unknown efforts as ``max``. + """ + none_prompt = await self._render_probe_prompt("none") + if none_prompt is None: + return "low" + for effort in get_args(AnthropicEffort): + if await self._render_probe_prompt(effort) == none_prompt: + return "low" + return "none" + + async def _render_probe_prompt( + self, effort: AnthropicDisabledThinkingEffort + ) -> tuple[list[int] | None, str | None] | None: + request = ChatCompletionRequest( + messages=[{"role": "user", "content": "Hi"}], + reasoning_effort=effort, + ) + try: + result = await self.online_renderer.render_chat(request) + except Exception: + logger.debug("Rendering reasoning_effort=%r failed", effort, exc_info=True) + return None + if isinstance(result, ErrorResponse): + return None + _, (engine_input,) = result + components = self._extract_prompt_components(engine_input) + return components.token_ids, components.text + + @classmethod + def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool: + """Probe the chat templates the renderer applies, not the CLI override. + + Both the tool and non-tool templates are checked since they may differ. + """ + renderer = online_renderer.renderer + tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, []) + if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None: + merge = cls._detect_merge_inline_system(online_renderer.chat_template) + else: + merge = any( + cls._detect_merge_inline_system( + resolve_chat_template( + renderer.tokenizer, + online_renderer.chat_template, + tools, + model_config=online_renderer.model_config, + ) + ) + for tools in tool_variants + ) + if merge: + logger.warning( + "The chat template requires system-first ordering, so inline " + "system messages in /v1/messages requests (e.g. Claude Code's " + "per-turn reminders) are merged into the leading system prompt. " + "Each new one changes the prompt prefix, so the rest of the " + "conversation misses the prefix cache. Pass a --chat-template " + "that accepts non-leading system messages to avoid this." + ) + return merge @staticmethod def _detect_merge_inline_system(chat_template: str | None) -> bool: @@ -197,6 +284,7 @@ class AnthropicServingMessages(OpenAIServingChat): anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, *, merge_inline_system: bool = False, + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", ) -> ChatCompletionRequest: """Convert Anthropic message format to OpenAI format.""" openai_messages: list[dict[str, Any]] = [] @@ -214,6 +302,7 @@ class AnthropicServingMessages(OpenAIServingChat): req = cls._build_base_request(anthropic_request, openai_messages) cls._handle_streaming_options(req, anthropic_request) cls._handle_output_config(req, anthropic_request) + cls._handle_thinking(req, anthropic_request, disabled_thinking_effort) cls._convert_tool_choice(anthropic_request, req) cls._convert_tools(anthropic_request, req) return req @@ -494,6 +583,33 @@ class AnthropicServingMessages(OpenAIServingChat): chat_template_kwargs=anthropic_request.chat_template_kwargs, ) + @classmethod + def _handle_thinking( + cls, + req: ChatCompletionRequest, + anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", + ) -> None: + """Handle extended-thinking configuration. + + ``display`` is intentionally ignored: suppressing reasoning would mark + it ended for structured outputs and drop it from multi-turn history. + """ + if isinstance(anthropic_request, AnthropicCountTokensRequest): + return + thinking: AnthropicThinkingConfig | None = anthropic_request.thinking + if thinking is None: + return + + if thinking.type == "disabled": + # "none" clears enable_thinking for templates that honor it; models + # that cannot disable thinking are configured with a low effort. + req.reasoning_effort = disabled_thinking_effort + elif thinking.type == "enabled": + req.thinking_token_budget = thinking.budget_tokens + # "adaptive" pins nothing: the model chooses depth beneath the ceiling + # already set from output_config.effort. + @classmethod def _handle_output_config( cls, @@ -602,9 +718,13 @@ class AnthropicServingMessages(OpenAIServingChat): """ if logger.isEnabledFor(logging.DEBUG): logger.debug("Received messages request %s", request.model_dump_json()) + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none" + if request.thinking is not None and request.thinking.type == "disabled": + disabled_thinking_effort = await self._get_disabled_thinking_effort() chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, + disabled_thinking_effort=disabled_thinking_effort, ) if logger.isEnabledFor(logging.DEBUG): logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json()) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index e3cc744ba7..f739495a8a 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1450,6 +1450,7 @@ class ChatTemplateConfig: chat_template: str | None = None chat_template_content_format: ChatTemplateContentFormatOption = "auto" trust_request_chat_template: bool = False + trust_request_mm_kwargs: bool = False def validate_chat_template(chat_template: Path | str | None): diff --git a/vllm/entrypoints/cli/main.py b/vllm/entrypoints/cli/main.py index ab9afcb008..4e1ae68b00 100644 --- a/vllm/entrypoints/cli/main.py +++ b/vllm/entrypoints/cli/main.py @@ -9,7 +9,7 @@ import importlib.metadata import sys from importlib.util import find_spec -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger logger = init_logger(__name__) @@ -44,6 +44,7 @@ def main(): import vllm.entrypoints.cli.collect_env import vllm.entrypoints.cli.launch import vllm.entrypoints.cli.openai + import vllm.entrypoints.cli.preload import vllm.entrypoints.cli.run_batch import vllm.entrypoints.cli.serve import vllm.entrypoints.cli.snapshot @@ -59,6 +60,7 @@ def main(): vllm.entrypoints.cli.launch, vllm.entrypoints.cli.benchmark.main, vllm.entrypoints.cli.collect_env, + vllm.entrypoints.cli.preload, vllm.entrypoints.cli.run_batch, vllm.entrypoints.cli.snapshot, ] @@ -108,7 +110,9 @@ def main(): cmds[cmd.name] = cmd args = parser.parse_args() if args.subparser in cmds: - cmds[args.subparser].validate(args) + cmd = cmds[args.subparser] + configure_logging_from_args(args) + cmd.validate(args) if hasattr(args, "dispatch_function"): args.dispatch_function(args) diff --git a/vllm/entrypoints/cli/preload.py b/vllm/entrypoints/cli/preload.py new file mode 100644 index 0000000000..d312b6c36a --- /dev/null +++ b/vllm/entrypoints/cli/preload.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The `preload` subcommand for the vLLM CLI. + +Launches one weight cache daemon per GPU; each daemon holds the +post-quantized, TP-sharded weights of its rank in GPU memory and serves +CUDA IPC handles to vLLM engines over a Unix domain socket, so restarting +engines can map the weights via zero-copy IPC instead of reloading from disk: + + vllm preload --model /path/to/model --tensor-parallel-size 4 + +Engines then load from the daemons with: + + vllm serve /path/to/model --tensor-parallel-size 4 --load-format ipc_cache + +Tensor, expert and data parallelism are supported, including across nodes; +see vllm/model_executor/model_loader/weight_cache/daemon.py for the rank +layout and rendezvous details. +""" + +import argparse +import multiprocessing +import queue +import signal +import sys +import typing +from itertools import product + +from vllm.engine.arg_utils import EngineArgs +from vllm.entrypoints.cli.types import CLISubcommand +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG +from vllm.logger import init_logger +from vllm.model_executor.model_loader.weight_cache.protocol import ( + check_ipc_platform_support, +) +from vllm.utils.network_utils import get_distributed_init_method, get_open_port + +if typing.TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.utils.argparse_utils import FlexibleArgumentParser +else: + FlexibleArgumentParser = argparse.ArgumentParser + +logger = init_logger(__name__) + + +class PreloadSubcommand(CLISubcommand): + """The `preload` subcommand for the vLLM CLI.""" + + name = "preload" + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser) -> None: + EngineArgs.add_cli_args(parser) + parser.add_argument( + "--weight-cache-socket-dir", + type=str, + default=None, + help="Directory for the daemon Unix sockets (default: tempdir).", + ) + parser.add_argument( + "--weight-cache-master-port", + type=int, + default=None, + help="Rendezvous port for the daemon's own TP group. Must differ " + "from the engine's --master-port (the daemon holds its group open " + "while serving) and match across nodes. Required when --nnodes > " + "1; defaults to a free port for single-node.", + ) + parser.add_argument( + "--weight-cache-draft-master-port", + type=int, + default=None, + help="Rendezvous port for the MTP draft daemon group. Defaults to " + "--weight-cache-master-port + 1 for multi-node, or a free port " + "for single-node.", + ) + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + # Imported here so `vllm --help` and friends stay free of torch. + from vllm.model_executor.model_loader.weight_cache.daemon import ( + _reject_unsupported_parallelism, + _run_daemon, + get_draft_daemon_config, + plan_local_ranks, + ) + from vllm.model_executor.model_loader.weight_cache.utils import ( + format_daemon_role, + ) + + engine_args = EngineArgs.from_cli_args(args) + vllm_config = engine_args.create_engine_config() + if vllm_config.load_config.load_format == "ipc_cache": + raise ValueError( + "The weight cache daemon itself must load from disk; use the " + "default --load-format" + ) + # Config-only so it can fail before any model loading; the quant + # method check needs the created model and runs in + # WeightCacheDaemon.get_model. + check_ipc_platform_support() + parallel_config = vllm_config.parallel_config + _reject_unsupported_parallelism(parallel_config) + tp_size = parallel_config.tensor_parallel_size + dp_size = parallel_config.data_parallel_size + + placements = plan_local_ranks(parallel_config) + local_world_size = len(placements) + if dp_size == 1: + nnodes = parallel_config.nnodes + node_rank = parallel_config.node_rank + master_addr = parallel_config.master_addr + else: + master_addr = parallel_config.data_parallel_master_ip + if parallel_config.nnodes > 1: + nnodes = parallel_config.nnodes + node_rank = parallel_config.node_rank + else: + nnodes = dp_size // parallel_config.data_parallel_size_local + node_rank = parallel_config.data_parallel_rank // ( + parallel_config.data_parallel_size_local + ) + if nnodes > 1 and master_addr in ("127.0.0.1", "localhost"): + raise ValueError( + "Data parallelism across nodes requires a reachable " + "--data-parallel-address for the daemon rendezvous" + ) + + # The daemon forms its own world group and holds it open while + # serving, so it needs a rendezvous port distinct from the engine's. + # All nodes must agree on it; single-node can auto-pick a free port. + # The master address is the engine's --master-addr (TP across nodes) + # or --data-parallel-address. + if nnodes > 1 and args.weight_cache_master_port is None: + raise ValueError( + "--weight-cache-master-port is required when the daemons span nodes" + ) + master_port = args.weight_cache_master_port or get_open_port() + distributed_init_method = get_distributed_init_method(master_addr, master_port) + + # (is_draft, vllm_config, rendezvous) per daemon group. + groups: list[tuple[bool, VllmConfig, str]] = [ + (False, vllm_config, distributed_init_method) + ] + draft_vllm_config = get_draft_daemon_config(vllm_config) + if draft_vllm_config is not None: + draft_master_port = args.weight_cache_draft_master_port or ( + master_port + 1 if nnodes > 1 else get_open_port() + ) + if draft_master_port == master_port: + raise ValueError( + "--weight-cache-draft-master-port must differ from " + "--weight-cache-master-port" + ) + groups.append( + ( + True, + draft_vllm_config, + get_distributed_init_method(master_addr, draft_master_port), + ) + ) + + ctx = multiprocessing.get_context("spawn") + ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue() + # Local index == device index; global rank enumerates DP then TP. + expected_ready = { + (format_daemon_role(is_draft), dp_rank * tp_size + tp_rank) + for (is_draft, _, _), (_, dp_rank, tp_rank) in product(groups, placements) + } + procs = [ + ctx.Process( + target=_run_daemon, + args=( + tp_rank, + local_rank, + config, + init_method, + args.weight_cache_socket_dir, + ready_queue, + is_draft, + dp_rank, + ), + name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-" + f"{dp_rank * tp_size + tp_rank}", + ) + for (is_draft, config, init_method), ( + local_rank, + dp_rank, + tp_rank, + ) in product(groups, placements) + ] + for proc in procs: + proc.start() + + def _shutdown(signum, frame): + for proc in procs: + proc.terminate() + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + + ready: set[tuple[str, int]] = set() + while len(ready) < len(expected_ready): + try: + ready.add(ready_queue.get(timeout=1.0)) + except queue.Empty: + dead = [p for p in procs if p.exitcode is not None] + if dead: + logger.error( + "Weight cache daemon rank(s) exited during startup " + "(exitcodes=%s); shutting down.", + [p.exitcode for p in dead], + ) + for proc in procs: + proc.terminate() + for proc in procs: + proc.join() + sys.exit(max((p.exitcode or 0) for p in procs)) + socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir" + logger.info_once( + "===== Weight cache daemon READY: node %d/%d serving %d local " + "rank(s) x %d role(s) in %s =====", + node_rank, + nnodes, + local_world_size, + len(groups), + socket_dir_msg, + ) + + for proc in procs: + proc.join() + sys.exit(max(proc.exitcode or 0 for proc in procs)) + + def subparser_init( + self, subparsers: argparse._SubParsersAction + ) -> FlexibleArgumentParser: + preload_parser = subparsers.add_parser( + self.name, + help="Launch weight cache daemons (one per GPU) for fast engine restarts.", + description="Launch weight cache daemons (one per GPU) that " + "keep post-quantized weights in GPU memory and serve them to " + "engines over CUDA IPC.", + usage="vllm preload --model [options]", + ) + self.add_cli_args(preload_parser) + preload_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name) + return preload_parser + + +def cmd_init() -> list[CLISubcommand]: + return [PreloadSubcommand()] diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 189c4e6779..c5703515d8 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -185,6 +185,7 @@ async def init_generate_state( enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, default_chat_template_kwargs=default_chat_template_kwargs, + disabled_thinking_effort=args.anthropic_disabled_thinking_effort, ) if "generate" in supported_tasks else None diff --git a/vllm/entrypoints/generate/base/protocol.py b/vllm/entrypoints/generate/base/protocol.py index 76dc5e47a1..c183498143 100644 --- a/vllm/entrypoints/generate/base/protocol.py +++ b/vllm/entrypoints/generate/base/protocol.py @@ -6,10 +6,8 @@ import json from typing import Annotated, Any, Literal, TypeAlias -import regex as re from pydantic import ( BaseModel, - ConfigDict, Field, model_serializer, ) @@ -21,7 +19,6 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.sampling_params import StructuredOutputsParams -from vllm.utils.import_utils import resolve_obj_by_qualname logger = init_logger(__name__) @@ -52,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None: ) +def validate_request_mm_kwargs( + *, + mm_processor_kwargs: dict[str, Any] | None, + media_io_kwargs: dict[str, dict[str, Any]] | None, + trust_request_mm_kwargs: bool, +) -> None: + """Reject untrusted per-request multimodal kwarg overrides.""" + if trust_request_mm_kwargs: + return + + if mm_processor_kwargs: + raise VLLMValidationError( + "Per-request mm_processor_kwargs are disabled by default because " + "they can change multimodal preprocessing resource usage. Start " + "the server with --trust-request-mm-kwargs only when clients " + "are trusted.", + parameter="mm_processor_kwargs", + ) + if media_io_kwargs: + raise VLLMValidationError( + "Per-request media_io_kwargs are disabled by default because " + "they can change multimodal media loading resource usage. Start " + "the server with --trust-request-mm-kwargs only when clients " + "are trusted.", + parameter="media_io_kwargs", + ) + + class SpeculativeDecodingMetrics(OpenAIBaseModel): """Per-request speculative-decoding acceptance metrics. @@ -261,53 +286,6 @@ class FunctionDefinition(OpenAIBaseModel): return data -# extra="forbid" is a workaround to have kwargs as a field, -# see https://github.com/pydantic/pydantic/issues/3125 -class LogitsProcessorConstructor(BaseModel): - qualname: str - args: list[Any] | None = None - kwargs: dict[str, Any] | None = None - - model_config = ConfigDict(extra="forbid") - - -LogitsProcessors = list[str | LogitsProcessorConstructor] - - -def get_logits_processors( - processors: LogitsProcessors | None, pattern: str | None -) -> list[Any] | None: - if processors and pattern: - logits_processors = [] - for processor in processors: - qualname = processor if isinstance(processor, str) else processor.qualname - if not re.match(pattern, qualname): - raise ValueError( - f"Logits processor '{qualname}' is not allowed by this " - "server. See --logits-processor-pattern engine argument " - "for more information." - ) - try: - logits_processor = resolve_obj_by_qualname(qualname) - except Exception as e: - raise ValueError( - f"Logits processor '{qualname}' could not be resolved: {e}" - ) from e - if isinstance(processor, LogitsProcessorConstructor): - logits_processor = logits_processor( - *processor.args or [], **processor.kwargs or {} - ) - logits_processors.append(logits_processor) - return logits_processors - elif processors: - raise ValueError( - "The `logits_processors` argument is not supported by this " - "server. See --logits-processor-pattern engine argument " - "for more information." - ) - return None - - class FunctionCall(OpenAIBaseModel): # Internal field to preserve native tool call ID from tool parser. # Excluded from serialization to maintain OpenAI API compatibility diff --git a/vllm/entrypoints/generate/generative_scoring/api_router.py b/vllm/entrypoints/generate/generative_scoring/api_router.py index d3e2316fcd..0b5f4c8a23 100644 --- a/vllm/entrypoints/generate/generative_scoring/api_router.py +++ b/vllm/entrypoints/generate/generative_scoring/api_router.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, FastAPI, Request from fastapi.responses import JSONResponse from vllm.entrypoints.generate.generative_scoring.serving import ( + GenerativeScoringRequest, GenerativeScoringResponse, ServingGenerativeScoring, ) @@ -31,26 +32,22 @@ def generative_scoring(request: Request) -> ServingGenerativeScoring | None: dependencies=[Depends(validate_json_request)], responses={ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, }, ) @with_cancellation @load_aware_call -async def create_generative_scoring(raw_request: Request): +async def create_generative_scoring( + request: GenerativeScoringRequest, raw_request: Request +): handler = generative_scoring(raw_request) if handler is None: raise NotImplementedError( "The model does not support the Generative Scoring API" ) - raw_body = await raw_request.json() - - from vllm.entrypoints.generate.generative_scoring.serving import ( - GenerativeScoringRequest, - ) - - gen_request = GenerativeScoringRequest(**raw_body) - result = await handler.create_generative_scoring(gen_request, raw_request) + result = await handler.create_generative_scoring(request, raw_request) if isinstance(result, ErrorResponse): return JSONResponse(content=result.model_dump(), status_code=result.error.code) diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index d33be65949..4f30beb1a8 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -92,6 +92,7 @@ async def init_app_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, + trust_request_mm_kwargs=args.trust_request_mm_kwargs, enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, diff --git a/vllm/entrypoints/launchers/api_server/entry.py b/vllm/entrypoints/launchers/api_server/entry.py index f3dcf40362..02837a451d 100644 --- a/vllm/entrypoints/launchers/api_server/entry.py +++ b/vllm/entrypoints/launchers/api_server/entry.py @@ -15,7 +15,7 @@ from typing import Any import vllm.envs as envs from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.protocol import EngineClient -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger from vllm.reasoning import ReasoningParserManager from vllm.tool_parsers import ToolParserManager from vllm.usage.usage_lib import UsageContext @@ -216,6 +216,7 @@ def main(): ) parser = make_arg_parser(parser) args = parser.parse_args() + configure_logging_from_args(args) validate_parsed_serve_args(args) uvloop.run(run_server(args)) diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 488a7f2c43..ce6692565c 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -14,6 +14,9 @@ from typing import Any, Literal import vllm.envs as envs from vllm.config import config from vllm.engine.arg_utils import AsyncEngineArgs, optional_type +from vllm.entrypoints.anthropic.protocol import ( + AnthropicDisabledThinkingEffortOption, +) from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, validate_chat_template, @@ -90,6 +93,11 @@ class BaseFrontendArgs: """Whether to trust the chat template provided in the request. If False, the server will always use the chat template specified by `--chat-template` or the ones from tokenizer.""" + trust_request_mm_kwargs: bool = False + """Whether to trust per-request multimodal kwargs (`mm_processor_kwargs` + and `media_io_kwargs`). If False, the server rejects non-empty values + because they can change multimodal preprocessing resource usage. Only + enable this when API clients are trusted.""" default_chat_template_kwargs: dict[str, Any] | None = None """Default keyword arguments to pass to the chat template renderer. These will be merged with request-level chat_template_kwargs, @@ -131,8 +139,6 @@ class BaseFrontendArgs: The `demo` Python tool executes model-generated code in Docker without network isolation by default. See the security guide for more information.""" - log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH - """Path to logging config JSON file for both vllm and uvicorn""" max_log_len: int | None = None """Max number of prompt characters or prompt ID numbers being printed in log. The default of None means unlimited.""" @@ -181,6 +187,13 @@ class BaseFrontendArgs: ``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any explicit request-level ``chat_template_kwargs.cohere_format`` takes priority.""" + anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto" + """Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for + requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking + off for models that support it; ``low`` suits models that always think + (e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses + ``low`` when the renderer rejects ``none`` or renders it the same as a + thinking effort, and ``none`` otherwise.""" log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE """If set to True, log the stack trace of error responses""" tokens_only: bool = False diff --git a/vllm/entrypoints/launchers/dp_supervisor.py b/vllm/entrypoints/launchers/dp_supervisor.py index 91754b71e5..192bbb1698 100644 --- a/vllm/entrypoints/launchers/dp_supervisor.py +++ b/vllm/entrypoints/launchers/dp_supervisor.py @@ -23,7 +23,7 @@ import uvloop from fastapi import FastAPI, Response import vllm.envs as envs -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger from vllm.utils.system_utils import ( decorate_logs, kill_process_tree, @@ -251,6 +251,8 @@ def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None: def _run_vllm_dp_server(child_args: argparse.Namespace) -> None: """Entrypoint function for the vLLM DP Server.""" + configure_logging_from_args(child_args) + # Create a fresh process group for the vLLM DP Server, # so that CTRL-C is propagated cleanly. os.setpgrp() @@ -558,4 +560,5 @@ class DPSupervisor: def run_dp_supervisor(args: argparse.Namespace) -> None: + configure_logging_from_args(args) uvloop.run(DPSupervisor(args).run()) diff --git a/vllm/entrypoints/launchers/grpc_server.py b/vllm/entrypoints/launchers/grpc_server.py index aab3f57c5e..2f3808526a 100644 --- a/vllm/entrypoints/launchers/grpc_server.py +++ b/vllm/entrypoints/launchers/grpc_server.py @@ -44,7 +44,7 @@ import uvloop from vllm import envs from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.serve.utils.api_utils import log_version_and_model -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger from vllm.usage.usage_lib import UsageContext from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.v1.engine.async_llm import AsyncLLM @@ -89,10 +89,18 @@ async def serve_grpc(args: argparse.Namespace): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), - # Tolerate client keepalive pings every 10s (default 300s is too - # strict for non-streaming requests where no DATA frames flow - # during generation) - ("grpc.http2.min_recv_ping_interval_without_data_ms", 10000), + # A non-streaming Generate writes nothing while the engine + # decodes, so client keepalive and BDP-probe PINGs arriving in + # that window are strikes and eventually earn a + # GOAWAY(ENHANCE_YOUR_CALM, "too_many_pings") that fails every + # in-flight RPC. Disable the strike count to tolerate them. + ("grpc.http2.max_ping_strikes", 0), + # GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS expands to + # a string with no "recv", and gRPC Core silently ignores unknown + # keys, so the misspelt form left the 300s default in force. + ("grpc.http2.min_ping_interval_without_data_ms", 10000), + # Without this, gRPC Core widens the floor to a fixed two hours + # once no streams are open, striking keepalives between requests. ("grpc.keepalive_permit_without_calls", True), ], ) @@ -187,6 +195,7 @@ def main(): parser = AsyncEngineArgs.add_cli_args(parser) args = parser.parse_args() + configure_logging_from_args(args) # Run server try: diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index 1d242977be..bbd9a61275 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -57,6 +57,7 @@ async def init_render_app_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, + trust_request_mm_kwargs=args.trust_request_mm_kwargs, enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, diff --git a/vllm/entrypoints/launchers/render/entry.py b/vllm/entrypoints/launchers/render/entry.py index b36cd64772..b42e1f5911 100644 --- a/vllm/entrypoints/launchers/render/entry.py +++ b/vllm/entrypoints/launchers/render/entry.py @@ -8,7 +8,7 @@ from argparse import Namespace from vllm import AsyncEngineArgs, envs from vllm.config import VllmConfig -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger from ..app import build_app from ..launcher import serve_http, setup_server @@ -87,7 +87,10 @@ async def run_launch_fastapi(args: argparse.Namespace) -> None: # cache space warning from CpuPlatform.check_and_update_config. envs.VLLM_CPU_KVCACHE_SPACE = 0 - vllm_config = VllmConfig(model_config=model_config) + vllm_config = VllmConfig( + model_config=model_config, + logging_config=engine_args.create_logging_config(), + ) shutdown_task = await build_and_serve_renderer( vllm_config, listen_address, sock, args ) @@ -115,6 +118,7 @@ if __name__ == "__main__": ) parser = make_arg_parser(parser) args = parser.parse_args() + configure_logging_from_args(args) validate_parsed_serve_args(args) uvloop.run(run_launch_fastapi(args)) diff --git a/vllm/entrypoints/launchers/run_batch.py b/vllm/entrypoints/launchers/run_batch.py index b5732ba65a..f54e87dafe 100644 --- a/vllm/entrypoints/launchers/run_batch.py +++ b/vllm/entrypoints/launchers/run_batch.py @@ -61,7 +61,7 @@ from vllm.entrypoints.speech_to_text.translation.protocol import ( TranslationResponseVerbose, ) from vllm.exceptions import VLLMValidationError -from vllm.logger import init_logger +from vllm.logger import configure_logging_from_args, init_logger from vllm.reasoning import ReasoningParserManager from vllm.utils import random_uuid from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -289,6 +289,7 @@ def make_arg_parser(parser: FlexibleArgumentParser): def parse_args(): parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.") args = make_arg_parser(parser).parse_args() + configure_logging_from_args(args) # Backward compatibility: If --url is set, use it for host url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv) diff --git a/vllm/entrypoints/launchers/utils/server_utils.py b/vllm/entrypoints/launchers/utils/server_utils.py index 91f5e62089..ada01706db 100644 --- a/vllm/entrypoints/launchers/utils/server_utils.py +++ b/vllm/entrypoints/launchers/utils/server_utils.py @@ -38,7 +38,13 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None: 3. Otherwise, return None (use uvicorn defaults) """ # First, try to load from file if specified - log_config = load_log_config(args.log_config_file) + logging_config = getattr(args, "logging_config", None) + log_config_file = ( + logging_config.pylogging_config_file + if logging_config is not None + else getattr(args, "log_config_file", None) + ) + log_config = load_log_config(log_config_file) if log_config is not None: return log_config diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 999310aa61..37348533b8 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -42,7 +42,7 @@ from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin from vllm.entrypoints.pooling.offline import PoolingOfflineMixin from vllm.entrypoints.serve.utils.api_utils import log_non_default_args from vllm.inputs import PromptType -from vllm.logger import init_logger +from vllm.logger import configure_logging_if_needed, init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.outputs import PoolingRequestOutput, RequestOutput @@ -76,8 +76,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): Args: model: The name or path of a HuggingFace Transformers model. tokenizer: The name or path of a HuggingFace Transformers tokenizer. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. + tokenizer_mode: The tokenizer mode. See + [ModelConfig.tokenizer_mode][vllm.config.ModelConfig.tokenizer_mode]. skip_tokenizer_init: If true, skip initialization of tokenizer and detokenizer. Expect valid prompt_token_ids and None for prompt from the input. @@ -338,6 +338,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): **kwargs, ) + configure_logging_if_needed(engine_args.create_logging_config()) log_non_default_args(engine_args) self.llm_engine = LLMEngine.from_engine_args( diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 225e992441..5c1bc594d7 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -215,7 +215,7 @@ class ChatCompletionRequest(OpenAIBaseModel): # https://platform.openai.com/docs/api-reference/chat/create messages: list[ChatCompletionMessageParam] model: str | None = None - frequency_penalty: float | None = 0.0 + frequency_penalty: float | None = None logit_bias: dict[str, float] | None = None logprobs: bool | None = False top_logprobs: int | None = 0 @@ -226,7 +226,7 @@ class ChatCompletionRequest(OpenAIBaseModel): ) max_completion_tokens: int | None = None n: int | None = 1 - presence_penalty: float | None = 0.0 + presence_penalty: float | None = None response_format: AnyResponseFormat | None = None seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) stop: StopParam = [] @@ -624,6 +624,8 @@ class ChatCompletionRequest(OpenAIBaseModel): # Default sampling parameters for chat completion requests _DEFAULT_SAMPLING_PARAMS: dict = { "repetition_penalty": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, "temperature": 1.0, "top_p": 1.0, "top_k": 0, @@ -661,28 +663,15 @@ class ChatCompletionRequest(OpenAIBaseModel): max_tokens: int, default_sampling_params: dict, ) -> SamplingParams: - # Default parameters - if (repetition_penalty := self.repetition_penalty) is None: - repetition_penalty = default_sampling_params.get( - "repetition_penalty", - self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"], - ) - if (temperature := self.temperature) is None: - temperature = default_sampling_params.get( - "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"] - ) - if (top_p := self.top_p) is None: - top_p = default_sampling_params.get( - "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"] - ) - if (top_k := self.top_k) is None: - top_k = default_sampling_params.get( - "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"] - ) - if (min_p := self.min_p) is None: - min_p = default_sampling_params.get( - "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"] + # Priority: user -> server default -> OpenAI default + sampling_params = { + name: ( + value + if (value := getattr(self, name)) is not None + else default_sampling_params.get(name, default) ) + for name, default in self._DEFAULT_SAMPLING_PARAMS.items() + } # Merge server-default stop_token_ids (e.g., model-specific tokens # like for gpt-oss) with any request-specified ones @@ -709,14 +698,8 @@ class ChatCompletionRequest(OpenAIBaseModel): extra_args["ec_transfer_params"] = self.ec_transfer_params return SamplingParams.from_optional( n=self.n, - presence_penalty=self.presence_penalty, - frequency_penalty=self.frequency_penalty, - repetition_penalty=repetition_penalty, - temperature=temperature, + **sampling_params, watermarking=self.watermarking, - top_p=top_p, - top_k=top_k, - min_p=min_p, seed=self.seed, stop=self.stop, stop_token_ids=stop_token_ids, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 3e2d8f7369..be35415538 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -672,11 +672,14 @@ class OpenAIServingChat(GenerateBaseServing): # set the previous values for the next iteration previous_num_tokens[i] += len(output.token_ids) - if parser is not None: + if parser is not None and self._include_reasoning_tokens_details: generated_token_ids[i].extend(output.token_ids) - previous_reasoning_tokens[i] = parser.count_reasoning_tokens( - tuple(generated_token_ids[i]) - ) + if include_continuous_usage: + previous_reasoning_tokens[i] = ( + parser.count_reasoning_tokens( + tuple(generated_token_ids[i]) + ) + ) # if the message delta is None (e.g. because it was a # "control token" for tool calls or the parser otherwise @@ -693,11 +696,13 @@ class OpenAIServingChat(GenerateBaseServing): logprobs = None if delta_message is None: - # NOTE: If return_token_ids is enabled, we still need to - # send a chunk with token_ids even if delta_message is None + # NOTE: If return_token_ids or logprobs are enabled, we + # still need to send a chunk even if delta_message is None # to ensure all tokens are included in the response - if output.finish_reason is None and ( - not request.return_token_ids or hide_stream_metadata + if ( + output.finish_reason is None + and logprobs is None + and (not request.return_token_ids or hide_stream_metadata) ): continue delta_message = DeltaMessage() @@ -757,7 +762,11 @@ class OpenAIServingChat(GenerateBaseServing): # finish_reason is: # "tool_calls" for "auto" or "required" tool calls, # and "stop" for named tool calls. - if tools_streamed[i] and not tool_choice_function_name: + if ( + tools_streamed[i] + and not tool_choice_function_name + and output.finish_reason == "stop" + ): finish_reason_ = "tool_calls" else: finish_reason_ = ( @@ -814,6 +823,13 @@ class OpenAIServingChat(GenerateBaseServing): data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" + if self._include_reasoning_tokens_details and not include_continuous_usage: + for i, parser in enumerate(parsers): + if parser is not None and generated_token_ids[i]: + previous_reasoning_tokens[i] = parser.count_reasoning_tokens( + tuple(generated_token_ids[i]) + ) + # once the final token is handled, if stream_options.include_usage # is sent, send the usage if include_usage: diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index bf0a94eea7..a0974cb91c 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -55,12 +55,12 @@ class CompletionRequest(OpenAIBaseModel): | None ) = None echo: bool | None = False - frequency_penalty: float | None = 0.0 + frequency_penalty: float | None = None logit_bias: dict[str, float] | None = None logprobs: int | None = None max_tokens: int | None = 16 n: int = 1 - presence_penalty: float | None = 0.0 + presence_penalty: float | None = None seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) stop: StopParam = [] stream: bool | None = False @@ -278,6 +278,8 @@ class CompletionRequest(OpenAIBaseModel): # Default sampling parameters for completion requests _DEFAULT_SAMPLING_PARAMS: dict = { "repetition_penalty": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, "temperature": 1.0, "top_p": 1.0, "top_k": 0, @@ -321,28 +323,15 @@ class CompletionRequest(OpenAIBaseModel): if default_sampling_params is None: default_sampling_params = {} - # Default parameters - if (repetition_penalty := self.repetition_penalty) is None: - repetition_penalty = default_sampling_params.get( - "repetition_penalty", - self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"], - ) - if (temperature := self.temperature) is None: - temperature = default_sampling_params.get( - "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"] - ) - if (top_p := self.top_p) is None: - top_p = default_sampling_params.get( - "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"] - ) - if (top_k := self.top_k) is None: - top_k = default_sampling_params.get( - "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"] - ) - if (min_p := self.min_p) is None: - min_p = default_sampling_params.get( - "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"] + # Priority: user -> server default -> OpenAI default + sampling_params = { + name: ( + value + if (value := getattr(self, name)) is not None + else default_sampling_params.get(name, default) ) + for name, default in self._DEFAULT_SAMPLING_PARAMS.items() + } # Merge server-default stop_token_ids (e.g., model-specific tokens # like for gpt-oss) with any request-specified ones @@ -371,14 +360,8 @@ class CompletionRequest(OpenAIBaseModel): extra_args["ec_transfer_params"] = self.ec_transfer_params return SamplingParams.from_optional( n=self.n, - presence_penalty=self.presence_penalty, - frequency_penalty=self.frequency_penalty, - repetition_penalty=repetition_penalty, - temperature=temperature, + **sampling_params, watermarking=self.watermarking, - top_p=top_p, - top_k=top_k, - min_p=min_p, seed=self.seed, stop=self.stop, stop_token_ids=stop_token_ids, diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 4d892d89d5..d7e235da1b 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing): See https://platform.openai.com/docs/api-reference/completions/create for the API specification. This API mimics the OpenAI Completion API. - NOTE: Currently we do not support the following feature: - - suffix (the language models we currently support do not support - suffix) + NOTE: suffix is only supported by models that implement FIM rendering. """ return await self._with_kv_transfer_rejection_cleanup( self._create_completion(request, raw_request), request, raw_request diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 0db47cf367..59b42f96a3 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -615,12 +615,14 @@ class HarmonyContext(ConversationContext): available_tools: list[str], function_tool_names: frozenset[str], response_parser: Parser | None = None, + request: ResponsesRequest | None = None, ): from vllm.parser.harmony import HarmonyParser, Segment assert isinstance(response_parser, HarmonyParser) self._messages = messages + self.request = request self.response_parser: HarmonyParser = response_parser self.finish_reason: str | None = None self.available_tools = available_tools diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 750d1db84c..7e286aa782 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -4,7 +4,7 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py import time -from typing import Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias from openai.types.responses import ( ResponseCodeInterpreterCallCodeDeltaEvent, @@ -49,6 +49,7 @@ from openai.types.responses.tool import Tool from openai.types.shared import Metadata, Reasoning from openai_harmony import Message as OpenAIHarmonyMessage from pydantic import ( + BeforeValidator, Field, ValidationError, field_serializer, @@ -135,6 +136,31 @@ ResponseInputOutputMessage: TypeAlias = ( ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem +def _default_input_image_details(value: Any) -> Any: + """Set the API default for input images before SDK type validation.""" + if not isinstance(value, dict): + return value + + content = value.get("content") + if not isinstance(content, list): + return value + + new_content = [] + changed = False + for part in content: + new_part = part + if ( + isinstance(part, dict) + and part.get("type") == "input_image" + and "detail" not in part + ): + new_part = {**part, "detail": "auto"} + changed = True + new_content.append(new_part) + + return {**value, "content": new_content} if changed else value + + class ResponsesRequest(OpenAIBaseModel): # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/responses/create @@ -152,7 +178,15 @@ class ResponsesRequest(OpenAIBaseModel): ] | None ) = None - input: str | list[ResponseInputOutputItem] + input: ( + str + | list[ + Annotated[ + ResponseInputOutputItem, + BeforeValidator(_default_input_image_details), + ] + ] + ) instructions: str | None = None max_output_tokens: int | None = None max_tool_calls: int | None = None diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 8108c5f775..9fba86b80c 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -443,6 +443,7 @@ class OpenAIServingResponses(GenerateBaseServing): available_tools, function_tool_names, response_parser=response_parser, + request=request, ) else: if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: @@ -678,8 +679,12 @@ class OpenAIServingResponses(GenerateBaseServing): tok_params=tok_params, ) - sampling_params.max_tokens = max_model_len - self._extract_prompt_len( - engine_input + sampling_params.max_tokens = get_max_tokens( + max_model_len, + context.request.max_output_tokens if context.request else None, + self._extract_prompt_len(engine_input), + self.default_sampling_params, + self.override_max_tokens, ) elif isinstance(context, ParsableContext): (engine_input,) = await self._render_next_turn( diff --git a/vllm/entrypoints/pooling/base/serving.py b/vllm/entrypoints/pooling/base/serving.py index e94d2ce2e3..c0c0c15e9c 100644 --- a/vllm/entrypoints/pooling/base/serving.py +++ b/vllm/entrypoints/pooling/base/serving.py @@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs from vllm.config import VllmConfig from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ChatTemplateConfig +from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.engine.typing import AnyRequest @@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing): self.return_tokens_as_token_ids = return_tokens_as_token_ids self.log_error_stack = log_error_stack self.chat_template_config = chat_template_config + self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs # Shared thread pool executor for preprocessing and postprocessing. self._executor: Executor = self.renderer._executor @@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing): request: AnyPoolingRequest, raw_request: Request | None = None, ): + validate_request_mm_kwargs( + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + media_io_kwargs=getattr(request, "media_io_kwargs", None), + trust_request_mm_kwargs=self.trust_request_mm_kwargs, + ) base_request_id = self._base_request_id( raw_request, getattr(request, "request_id", None) ) diff --git a/vllm/entrypoints/pooling/factories.py b/vllm/entrypoints/pooling/factories.py index c7615054dd..e6d01be5a9 100644 --- a/vllm/entrypoints/pooling/factories.py +++ b/vllm/entrypoints/pooling/factories.py @@ -166,6 +166,7 @@ def init_pooling_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, + trust_request_mm_kwargs=args.trust_request_mm_kwargs, ) state.serving_pooling = ( diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index 5e6104eedd..8ffa2e4ddd 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -621,7 +621,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): model_config, ) - # Apply truncation before defining closures + # Limit the query and document separately before composing them. if max_tokens_per_query > 0 and isinstance(prompt_1, str): prompt_1 = truncate_text_to_tokens( prompt_1, tokenizer, max_tokens_per_query @@ -666,16 +666,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): full_prompt = tokenizer.decode(prompt_inputs["input_ids"]) else: # `llm as reranker` defaults to not using separating token. - if max_tokens_per_doc > 0 and isinstance(prompt_2, str): - query_ids = tokenizer.encode(prompt_1, add_special_tokens=False) - doc_ids = tokenizer.encode(prompt_2, add_special_tokens=False) - doc_ids = doc_ids[:max_tokens_per_doc] - input_ids = query_ids + doc_ids - full_prompt = tokenizer.decode(input_ids) - prompt_inputs = {"input_ids": input_ids} - else: - full_prompt = prompt_1 + prompt_2 - prompt_inputs = tokenizer(text=full_prompt, **local_kwargs) + full_prompt = prompt_1 + prompt_2 + prompt_inputs = tokenizer(text=full_prompt, **local_kwargs) return full_prompt, prompt_inputs # FIXME: For now, we only apply a template when one is explicitly provided. diff --git a/vllm/entrypoints/pooling/scoring/utils.py b/vllm/entrypoints/pooling/scoring/utils.py index cff8ff61d9..d7cc888a15 100644 --- a/vllm/entrypoints/pooling/scoring/utils.py +++ b/vllm/entrypoints/pooling/scoring/utils.py @@ -48,13 +48,27 @@ def truncate_text_to_tokens( Uses offset_mapping to slice the original text at the exact character boundary, avoiding lossy encode→decode round-trips that can shift - the token count by 1-3 tokens due to BPE merge boundary changes. + the token count due to BPE merge boundary changes. """ - encoding = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) - if len(encoding["input_ids"]) <= max_tokens: - return text - char_end = encoding["offset_mapping"][max_tokens - 1][1] - return text[:char_end] + if max_tokens <= 0: + return "" + + while text: + encoding = tokenizer( + text, + add_special_tokens=False, + return_offsets_mapping=True, + ) + offsets = encoding["offset_mapping"] + if len(encoding["input_ids"]) <= max_tokens: + break + + char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0]) + if char_end >= len(text): + char_end = len(text) - 1 + text = text[:char_end] + + return text def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor: diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 7973ac3c3d..117ede0f9b 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -57,9 +57,7 @@ class ServingRender(BaseServing): self.tool_server = tool_server self._merge_inline_system = ( - AnthropicServingMessages._detect_merge_inline_system( - online_renderer.chat_template - ) + AnthropicServingMessages._should_merge_inline_system(online_renderer) ) self._placeholder_metadata_parser: MultiModalDataParser | None = None diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 59fda2dbf8..b480b802ec 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -590,7 +590,10 @@ class ServingTokens(GenerateBaseServing): step_top_logprobs.items() ) if num_output_top_logprobs is not None - and i < max(num_output_top_logprobs, 1) + and ( + num_output_top_logprobs == -1 + or i < max(num_output_top_logprobs, 1) + ) ], ) ) diff --git a/vllm/entrypoints/serve/engine/serving.py b/vllm/entrypoints/serve/engine/serving.py index 7a1befbe4c..86ba524946 100644 --- a/vllm/entrypoints/serve/engine/serving.py +++ b/vllm/entrypoints/serve/engine/serving.py @@ -41,6 +41,9 @@ class BaseServing: self, request: AnyRequest | AnyPoolingRequest, ) -> ErrorResponse | None: + if self.request_logger is not None: + self.request_logger.log_request_body(request) + error_response = None if self._is_model_supported(request.model): diff --git a/vllm/entrypoints/serve/exception_handling/utils.py b/vllm/entrypoints/serve/exception_handling/utils.py index 73d9992237..6c1dd20516 100644 --- a/vllm/entrypoints/serve/exception_handling/utils.py +++ b/vllm/entrypoints/serve/exception_handling/utils.py @@ -6,7 +6,7 @@ import regex as re def sanitize_message(message: str) -> str: """Strip memory addresses, tracebacks, and file paths from error messages.""" - message = re.sub(r" at 0x[0-9a-f]+>", ">", message) + message = re.sub(r" at 0x[0-9a-fA-F]+>", ">", message) message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message) message = re.sub( r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "", message diff --git a/vllm/entrypoints/serve/utils/request_logger.py b/vllm/entrypoints/serve/utils/request_logger.py index ac10feab10..09b27e0673 100644 --- a/vllm/entrypoints/serve/utils/request_logger.py +++ b/vllm/entrypoints/serve/utils/request_logger.py @@ -6,6 +6,8 @@ from collections.abc import Sequence import torch +from vllm.entrypoints.pooling.typing import AnyPoolingRequest +from vllm.entrypoints.serve.engine.typing import AnyRequest from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.pooling_params import PoolingParams @@ -29,7 +31,7 @@ class RequestLogger: "`--enable-log-requests` is set but " "the minimum log level is higher than DEBUG. " "Only limited information will be logged to minimize overhead. " - "To view more details, set `VLLM_LOGGING_LEVEL=DEBUG`." + "To view more details, set `--log-level DEBUG`." ) def log_inputs( @@ -67,6 +69,15 @@ class RequestLogger: lora_request, ) + def log_request_body(self, request: AnyRequest | AnyPoolingRequest) -> None: + if logger.isEnabledFor(logging.DEBUG): + max_log_len = self.max_log_len if self.max_log_len is not None else -1 + logger.debug( + "Request %s JSON body: %s", + getattr(request, "request_id", "N/A"), + request.model_dump_json(exclude_unset=True)[:max_log_len], + ) + def log_outputs( self, request_id: str, diff --git a/vllm/env_override.py b/vllm/env_override.py index 87501eb469..bfcdfa7af2 100644 --- a/vllm/env_override.py +++ b/vllm/env_override.py @@ -916,6 +916,58 @@ def _patch_inductor_fallback_allow_list() -> None: _patch_inductor_fallback_allow_list() + +def _apply_inductor_pattern_matcher_patch() -> None: + """Allow custom ops and functionalization wrappers with unsupported dtypes.""" + from torch._inductor import pattern_matcher + from torch._inductor.lowering import fallback_node_due_to_unsupported_type + + def fallback_for_builtin(node, allow_cpu_inputs=True): + return ( + isinstance(node.target, torch._ops.OpOverload) + and torch._library.utils.is_builtin(node.target) + and fallback_node_due_to_unsupported_type(node, allow_cpu_inputs) + ) + + pattern_matcher.fallback_node_due_to_unsupported_type = fallback_for_builtin + + +def _patch_inductor_pattern_matcher() -> None: + """Apply the backport when Inductor imports its pattern matcher.""" + import sys + + target_name = "torch._inductor.pattern_matcher" + if target_name in sys.modules: + _apply_inductor_pattern_matcher_patch() + return + + import importlib.abc + + class _PatternMatcherPatchFinder(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if fullname != target_name: + return None + sys.meta_path.remove(self) + spec = importlib.util.find_spec(fullname) + if spec is None or spec.loader is None: + return None + original_exec = spec.loader.exec_module + + def _exec_then_patch(module): + original_exec(module) + _apply_inductor_pattern_matcher_patch() + + spec.loader.exec_module = _exec_then_patch # type: ignore[method-assign] + return spec + + sys.meta_path.insert(0, _PatternMatcherPatchFinder()) + + +# Remove once the minimum supported torch includes +# https://github.com/pytorch/pytorch/pull/196013. +if not is_torch_equal_or_newer("2.16.0.dev"): + _patch_inductor_pattern_matcher() + # ============================================================ # Triton Autotuner determinism # ============================================================ diff --git a/vllm/envs.py b/vllm/envs.py index 30b3de8d64..b0d72a71db 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -63,6 +63,7 @@ if TYPE_CHECKING: VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto" VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True + VLLM_XPU_PP_MICROBATCH: bool = True VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False VLLM_DISTRIBUTED_USE_SPLIT_GROUP: bool = False VLLM_XLA_USE_SPMD: bool = False @@ -123,6 +124,7 @@ if TYPE_CHECKING: VLLM_USE_TRITON_AWQ: bool = False VLLM_FASTSAFETENSORS_QUEUE_SIZE: int = 0 VLLM_TRITON_FORCE_FIRST_CONFIG: bool = False + VLLM_TRITON_JIT_WARMUP_NUM_THREADS: int = 4 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False VLLM_SKIP_P2P_CHECK: bool = False VLLM_DISABLED_KERNELS: list[str] = [] @@ -139,7 +141,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 - VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False + VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" @@ -317,7 +319,6 @@ if TYPE_CHECKING: VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True - VLLM_XPU_ENABLE_XPU_GRAPH: bool = False VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False VLLM_XPU_USE_SAMPLER_KERNEL: bool = True VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto" @@ -924,6 +925,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool( int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1")) ), + # Using a flag to control microbatch on XPU device, users on XPU device can + # decide to disable it when they needed. + "VLLM_XPU_PP_MICROBATCH": lambda: bool( + int(os.getenv("VLLM_XPU_PP_MICROBATCH", "1")) + ), # When True and distributed_executor_backend="ray", use RayExecutorV2 # (MQ-based) instead of RayDistributedExecutor (compiled-graph backend). "VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool( @@ -1182,6 +1188,11 @@ environment_variables: dict[str, Callable[[], Any]] = { os.environ.get("VLLM_TRITON_FORCE_FIRST_CONFIG", "0").strip().lower() in ("1", "true") ), + # Maximum compiler threads per worker for registered CUDA Triton JIT warmup. + # Set to 1 for serial warmup. Does not affect runtime JIT or autotuning. + "VLLM_TRITON_JIT_WARMUP_NUM_THREADS": lambda: int( + os.getenv("VLLM_TRITON_JIT_WARMUP_NUM_THREADS", "4") + ), # If set, allow loading or unloading lora adapters in runtime, "VLLM_ALLOW_RUNTIME_LORA_UPDATING": lambda: ( os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() @@ -1269,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), - # Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4 - # activations, separated gate/up layout) instead of default a16w4. vLLM - # sets AITER_SITUV2_A4W4 at init when this flag is on and clears any - # legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first). - # VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing - # recipes; it does not select a8w4 kernels. - # Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag - # and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics. - "VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: ( - os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1") - or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() - in ("true", "1") + # Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2): + # auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4. + "VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices( + "VLLM_ROCM_USE_AITER_MOE_SITUV2", + "auto", + ["auto", "a4w4", "a8w4", "a16w4", "0", "1"], + case_sensitive=False, ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass @@ -2139,10 +2145,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool( int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1")) ), - # Whether enable XPU graph on Intel GPU - "VLLM_XPU_ENABLE_XPU_GRAPH": lambda: bool( - int(os.getenv("VLLM_XPU_ENABLE_XPU_GRAPH", "0")) - ), # Force N-contiguous weight layout for all XPU unquantized linears. "VLLM_XPU_FORCE_N_CONTIG_WEIGHT": lambda: bool( int(os.getenv("VLLM_XPU_FORCE_N_CONTIG_WEIGHT", "0")) @@ -2277,6 +2279,7 @@ def compile_factors() -> dict[str, object]: hash everything else. This keeps the cache key aligned across workers.""" ignored_factors: set[str] = { "MAX_JOBS", + "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", "VLLM_RPC_BASE_PATH", "VLLM_USE_MODELSCOPE", "VLLM_RINGBUFFER_WARNING_INTERVAL", diff --git a/vllm/forward_context.py b/vllm/forward_context.py index 6a31b3faff..9ddc8e9255 100644 --- a/vllm/forward_context.py +++ b/vllm/forward_context.py @@ -58,8 +58,17 @@ class BatchDescriptor: def _compute_sp_num_tokens( - num_tokens_across_dp_cpu: torch.Tensor, sequence_parallel_size: int + num_tokens_across_dp_cpu: torch.Tensor, + sequence_parallel_size: int, + pcp_size: int = 1, + use_ep: bool = True, ) -> list[int]: + if pcp_size > 1: + num_tokens_across_dp_cpu = ( + num_tokens_across_dp_cpu.repeat_interleave(pcp_size) + if use_ep + else num_tokens_across_dp_cpu * pcp_size + ) sp_tokens = ( num_tokens_across_dp_cpu + sequence_parallel_size - 1 ) // sequence_parallel_size @@ -98,12 +107,14 @@ class DPMetadata: return DPMetadata(num_tokens_across_dp_cpu) @contextmanager - def sp_local_sizes(self, sequence_parallel_size: int): + def sp_local_sizes( + self, sequence_parallel_size: int, pcp_size: int = 1, use_ep: bool = False + ): """Context manager for setting self.local_sizes. Same as self.chunked_sizes but without any chunking. """ self.local_sizes = _compute_sp_num_tokens( - self.num_tokens_across_dp_cpu, sequence_parallel_size + self.num_tokens_across_dp_cpu, sequence_parallel_size, pcp_size, use_ep ) try: yield self.local_sizes diff --git a/vllm/kernels/helion/config_manager.py b/vllm/kernels/helion/config_manager.py index 8887cdaea4..ca157d461c 100644 --- a/vllm/kernels/helion/config_manager.py +++ b/vllm/kernels/helion/config_manager.py @@ -74,7 +74,7 @@ class ConfigSet: f"If your GPU is a variant of a supported platform, " f"consider adding a mapping in _GPU_NAME_ALIASES in " f"vllm/kernels/helion/utils.py, or run " - f"scripts/autotune_helion_kernels.py to generate configs " + f"tools/autotune_helion_kernels.py to generate configs " f"for your platform." ) diff --git a/vllm/kernels/helion/ops/__init__.py b/vllm/kernels/helion/ops/__init__.py index 0fb82f6cfe..e184cc93bf 100644 --- a/vllm/kernels/helion/ops/__init__.py +++ b/vllm/kernels/helion/ops/__init__.py @@ -9,7 +9,7 @@ specific op module it needs, e.g.:: which triggers that op's ``@register_kernel`` as an import side effect. -Tools that need the full registry (e.g. scripts/autotune_helion_kernels.py) +Tools that need the full registry (e.g. tools/autotune_helion_kernels.py) call ``import_all_ops()`` to force every op module to register. """ diff --git a/vllm/logger.py b/vllm/logger.py index 05ee5936d4..5e5f63aefc 100644 --- a/vllm/logger.py +++ b/vllm/logger.py @@ -9,22 +9,52 @@ import os import sys from collections.abc import Generator, Hashable from contextlib import contextmanager +from copy import deepcopy +from dataclasses import replace from functools import lru_cache, partial from logging import Logger from logging.config import dictConfig from os import path from types import MethodType -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast import vllm.envs as envs from vllm.logging_utils import ColoredFormatter, NewLineFormatter +if TYPE_CHECKING: + from vllm.config.logging import LoggingConfig + _FORMAT = ( - f"{envs.VLLM_LOGGING_PREFIX}%(levelname)s %(asctime)s " + f"{envs.VLLM_LOGGING_PREFIX}(%(vllm_process_name)s " + "pid=%(process)d) %(levelname)s %(asctime)s " "[%(fileinfo)s:%(lineno)d] %(message)s" ) _DATE_FORMAT = "%m-%d %H:%M:%S" +_base_log_record_factory = logging.getLogRecordFactory() +_vllm_process_info: tuple[str, int] | None = None + + +def _vllm_log_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: + """Add vLLM process metadata to each log record.""" + record = _base_log_record_factory(*args, **kwargs) + process_info = _vllm_process_info + pid = os.getpid() + if process_info is None or process_info[1] != pid: + record.vllm_process_name = record.processName + else: + record.vllm_process_name = process_info[0] + return record + + +def set_vllm_process_name(process_name: str, *, skip_if_set: bool = False) -> None: + """Set the vLLM process name added to subsequent log records.""" + global _vllm_process_info + pid = os.getpid() + if skip_if_set and _vllm_process_info is not None and _vllm_process_info[1] == pid: + return + _vllm_process_info = (process_name, pid) + def _use_color() -> bool: if envs.NO_COLOR or envs.VLLM_LOGGING_COLOR == "0": @@ -40,15 +70,18 @@ def _use_color() -> bool: DEFAULT_LOGGING_CONFIG: dict[str, dict[str, Any] | Any] = { "formatters": { + # The "()" factory form forwards custom kwargs such as log_level. "vllm": { - "class": "vllm.logging_utils.NewLineFormatter", + "()": "vllm.logging_utils.NewLineFormatter", "datefmt": _DATE_FORMAT, "format": _FORMAT, + "log_level": envs.VLLM_LOGGING_LEVEL, }, "vllm_color": { - "class": "vllm.logging_utils.ColoredFormatter", + "()": "vllm.logging_utils.ColoredFormatter", "datefmt": _DATE_FORMAT, "format": _FORMAT, + "log_level": envs.VLLM_LOGGING_LEVEL, }, }, "handlers": { @@ -71,6 +104,8 @@ DEFAULT_LOGGING_CONFIG: dict[str, dict[str, Any] | Any] = { "disable_existing_loggers": False, } +_last_configured_logging_config: "LoggingConfig | None" = None + @lru_cache def _print_debug_once(logger: Logger, msg: str, *args: Hashable) -> None: @@ -165,36 +200,47 @@ _METHODS_TO_PATCH = { } -def _configure_vllm_root_logger() -> None: +def _configure_vllm_root_logger(config: "LoggingConfig | None" = None) -> None: + """Configure logging from explicit config or bootstrap environment values.""" logging_config: dict[str, dict[str, Any] | Any] = {} + if config is None: + configure_logging = envs.VLLM_CONFIGURE_LOGGING + log_level = envs.VLLM_LOGGING_LEVEL + log_config_file = envs.VLLM_LOGGING_CONFIG_PATH + else: + configure_logging = config.configure_logging + log_level = config.log_level + log_config_file = config.pylogging_config_file - if not envs.VLLM_CONFIGURE_LOGGING and envs.VLLM_LOGGING_CONFIG_PATH: + if not configure_logging and log_config_file: raise RuntimeError( - "VLLM_CONFIGURE_LOGGING evaluated to false, but " - "VLLM_LOGGING_CONFIG_PATH was given. VLLM_LOGGING_CONFIG_PATH " - "implies VLLM_CONFIGURE_LOGGING. Please enable " - "VLLM_CONFIGURE_LOGGING or unset VLLM_LOGGING_CONFIG_PATH." + "Logging configuration is disabled, but a Python logging config " + "file was given. pylogging_config_file requires " + "configure_logging to be enabled." ) - if envs.VLLM_CONFIGURE_LOGGING: - logging_config = DEFAULT_LOGGING_CONFIG + if configure_logging: + logging.setLogRecordFactory(_vllm_log_record_factory) + logging_config = deepcopy(DEFAULT_LOGGING_CONFIG) vllm_handler = logging_config["handlers"]["vllm"] # Refresh these values in case env vars have changed. - vllm_handler["level"] = envs.VLLM_LOGGING_LEVEL + vllm_handler["level"] = log_level vllm_handler["stream"] = envs.VLLM_LOGGING_STREAM vllm_handler["formatter"] = "vllm_color" if _use_color() else "vllm" vllm_loggers = logging_config["loggers"]["vllm"] - vllm_loggers["level"] = envs.VLLM_LOGGING_LEVEL + vllm_loggers["level"] = log_level + for formatter in logging_config["formatters"].values(): + formatter["log_level"] = log_level - if envs.VLLM_LOGGING_CONFIG_PATH: - if not path.exists(envs.VLLM_LOGGING_CONFIG_PATH): + if log_config_file: + if not path.exists(log_config_file): raise RuntimeError( "Could not load logging config. File does not exist: %s", - envs.VLLM_LOGGING_CONFIG_PATH, + log_config_file, ) - with open(envs.VLLM_LOGGING_CONFIG_PATH, encoding="utf-8") as file: + with open(log_config_file, encoding="utf-8") as file: custom_config = json.loads(file.read()) if not isinstance(custom_config, dict): @@ -212,11 +258,57 @@ def _configure_vllm_root_logger() -> None: if logging_config: dictConfig(logging_config) + # Transformers uses httpx to access the Hugging Face Hub. httpx is quite verbose, + # so we set its logging level to WARNING when vLLM's logging level is INFO. + # httpx2 is the successor huggingface_hub switches to in its 2.x releases. + if log_level == "INFO": + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpx2").setLevel(logging.WARNING) + + +def configure_logging(config: "LoggingConfig") -> None: + """Apply a logging configuration in the current process.""" + _configure_vllm_root_logger(config) + global _last_configured_logging_config + _last_configured_logging_config = config + _log_platform_warnings(config) + + +def _log_platform_warnings(config: "LoggingConfig") -> None: + """Emit platform diagnostics only after an enabled config is active.""" + if not config.configure_logging: + return + + # Import lazily because platform modules use init_logger during import. + from vllm.platforms import current_platform + + current_platform.log_warnings() + + +def configure_logging_if_needed(config: "LoggingConfig") -> None: + """Apply a logging configuration unless it is already active in this process.""" + if config != _last_configured_logging_config: + configure_logging(config) + + +def configure_logging_from_args(args: Any) -> "LoggingConfig": + """Apply parsed logging arguments and retain them for child processes.""" + from vllm.config.logging import LoggingConfig + + config = getattr(args, "logging_config", None) or LoggingConfig() + if hasattr(args, "log_level"): + config = replace(config, log_level=args.log_level) + if hasattr(args, "log_config_file"): + config = replace(config, pylogging_config_file=args.log_config_file) + + configure_logging_if_needed(config) + args.log_config_file = config.pylogging_config_file + args.logging_config = config + return config + def init_logger(name: str) -> _VllmLogger: - """The main purpose of this function is to ensure that loggers are - retrieved in such a way that we can be sure the root vllm logger has - already been configured.""" + """Retrieve a logger and add vLLM's convenience logging methods.""" logger = logging.getLogger(name) for method_name, method in _METHODS_TO_PATCH.items(): @@ -246,18 +338,6 @@ def current_formatter_type(logger: Logger) -> Literal["color", "newline", None]: return None -# The root logger is initialized when the module is imported. -# This is thread-safe as the module is only imported once, -# guaranteed by the Python GIL. -_configure_vllm_root_logger() - -# Transformers uses httpx to access the Hugging Face Hub. httpx is quite verbose, -# so we set its logging level to WARNING when vLLM's logging level is INFO. -# httpx2 is the successor huggingface_hub switches to in its 2.x releases. -if envs.VLLM_LOGGING_LEVEL == "INFO": - logging.getLogger("httpx").setLevel(logging.WARNING) - logging.getLogger("httpx2").setLevel(logging.WARNING) - logger = init_logger(__name__) diff --git a/vllm/logging_utils/formatter.py b/vllm/logging_utils/formatter.py index 4a90009266..205bb36405 100644 --- a/vllm/logging_utils/formatter.py +++ b/vllm/logging_utils/formatter.py @@ -4,16 +4,14 @@ import logging from pathlib import Path -from vllm import envs - class NewLineFormatter(logging.Formatter): """Adds logging prefix to newlines to align multi-line messages.""" - def __init__(self, fmt, datefmt=None, style="%"): + def __init__(self, fmt, datefmt=None, style="%", *, log_level: str | None = None): super().__init__(fmt, datefmt, style) - self.use_relpath = envs.VLLM_LOGGING_LEVEL == "DEBUG" + self.use_relpath = log_level == "DEBUG" if self.use_relpath: self.root_dir = Path(__file__).resolve().parent.parent.parent @@ -93,11 +91,20 @@ class ColoredFormatter(NewLineFormatter): "CRITICAL": "\033[35m", # Magenta } GREY = "\033[90m" # Grey for timestamp and file info + PROCESS_PREFIX_COLOR = "\033[0;36m" + PROCESS_PREFIX_RESET = "\033[0;0m" RESET = "\033[0m" - def __init__(self, fmt, datefmt=None, style="%"): + def __init__(self, fmt, datefmt=None, style="%", *, log_level: str | None = None): # Inject grey color codes into format string for timestamp and file info if fmt: + # Wrap process prefix with cyan + fmt = fmt.replace( + "(%(vllm_process_name)s pid=%(process)d)", + f"{self.PROCESS_PREFIX_COLOR}" + "(%(vllm_process_name)s pid=%(process)d)" + f"{self.PROCESS_PREFIX_RESET}", + ) # Wrap %(asctime)s with grey fmt = fmt.replace("%(asctime)s", f"{self.GREY}%(asctime)s{self.RESET}") # Wrap [%(fileinfo)s:%(lineno)d] with grey @@ -107,7 +114,7 @@ class ColoredFormatter(NewLineFormatter): ) # Call parent __init__ with potentially modified format string - super().__init__(fmt, datefmt, style) + super().__init__(fmt, datefmt, style, log_level=log_level) def format(self, record): # Store original levelname to restore later (in case record is reused) diff --git a/vllm/logits_process.py b/vllm/logits_process.py index 1bf97c2535..300ec3cacc 100644 --- a/vllm/logits_process.py +++ b/vllm/logits_process.py @@ -1,12 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable, Sequence +from collections.abc import Callable from typing import TypeAlias import torch -from vllm.tokenizers import TokenizerLike - LogitsProcessor: TypeAlias = ( Callable[[list[int], torch.Tensor], torch.Tensor] | Callable[[list[int], list[int], torch.Tensor], torch.Tensor] @@ -16,106 +14,3 @@ of previously generated tokens, the logits tensor for the next token and, optionally, prompt tokens as a first argument, and returns a modified tensor of logits to sample from.""" - - -def get_bad_words_logits_processors( - bad_words: list[str], tokenizer: TokenizerLike -) -> list[LogitsProcessor]: - bad_words_ids: list[list[int]] = list() - - for bad_word in bad_words: - # To prohibit words both at the beginning - # and in the middle of text - # (related to add_prefix_space tokenizer parameter) - for add_prefix_space in [False, True]: - prefix = " " if add_prefix_space else "" - prompt = prefix + bad_word.lstrip() - - prompt_token_ids = tokenizer.encode(text=prompt, add_special_tokens=False) - - # If no space at the beginning - # or if prefix space produces a new word token - if (not add_prefix_space) or ( - add_prefix_space - and prompt_token_ids[0] != bad_words_ids[-1][0] - and len(prompt_token_ids) == len(bad_words_ids[-1]) - ): - bad_words_ids.append(prompt_token_ids) - - return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)] - - -class NoBadWordsLogitsProcessor: - _SMALLEST_LOGIT = float("-inf") - _NEUTRAL_LOGIT = 0.0 - - def __init__(self, bad_words_ids: list[list[int]]): - self.bad_words_ids = bad_words_ids - self.word_bias: torch.FloatTensor = None - - def __call__( - self, - past_tokens_ids: Sequence[int], - logits: torch.FloatTensor, - ) -> torch.Tensor: - if self.word_bias is None: - self._init_word_bias(logits=logits) - - last_token_bias = torch.zeros_like(logits) - - for bad_word_ids in self.bad_words_ids: - if len(bad_word_ids) == 1: # 1-token words already processed - continue - - if len(bad_word_ids) > len(past_tokens_ids) + 1: - continue - - prefix_length = len(bad_word_ids) - 1 - last_token_id = bad_word_ids[-1] - actual_prefix = past_tokens_ids[-prefix_length:] - expected_prefix = bad_word_ids[:prefix_length] - - assert len(actual_prefix) == len(expected_prefix) - - is_match = tuple(actual_prefix) == tuple(expected_prefix) - last_token_bias[last_token_id] += ( - self._SMALLEST_LOGIT if is_match else self._NEUTRAL_LOGIT - ) - - logits = logits + self.word_bias + last_token_bias - - return logits - - def _init_word_bias(self, logits: torch.FloatTensor) -> None: - # Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501 - # from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py - - vocab_size = logits.shape[-1] - - self._check_token_ids_bounds(vocab_size=vocab_size) - - self.word_bias = torch.zeros( - (vocab_size,), dtype=torch.float, device=logits.device - ) - - for bad_word_ids in self.bad_words_ids: - if len(bad_word_ids) == 1: - bad_word_id = bad_word_ids[-1] - self.word_bias[bad_word_id] = self._SMALLEST_LOGIT - - def _check_token_ids_bounds(self, vocab_size: int) -> None: - invalid_token_ids = [] - - for bad_word_ids in self.bad_words_ids: - for token_id in bad_word_ids: - if token_id < 0 or token_id >= vocab_size: - invalid_token_ids.append(token_id) - - if len(invalid_token_ids) > 0: - raise ValueError( - f"The model vocabulary size is {vocab_size}," - f" but the following tokens" - f" were specified as bad: {invalid_token_ids}." - f" All token id values should be integers satisfying:" - f" 0 <= token_id < {vocab_size}." - ) diff --git a/vllm/lora/layers/base.py b/vllm/lora/layers/base.py index 5fba2998c8..70821be3d1 100644 --- a/vllm/lora/layers/base.py +++ b/vllm/lora/layers/base.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, overload import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig @@ -73,7 +73,7 @@ class BaseLayerWithLoRA(nn.Module): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: """Initializes lora matrices.""" ... @@ -103,7 +103,7 @@ class BaseLayerWithLoRA(nn.Module): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: """Returns True if the layer can be replaced by this LoRA layer.""" raise NotImplementedError diff --git a/vllm/lora/layers/base_linear.py b/vllm/lora/layers/base_linear.py index 36767376c4..12888cf3db 100644 --- a/vllm/lora/layers/base_linear.py +++ b/vllm/lora/layers/base_linear.py @@ -3,7 +3,7 @@ import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import envs from vllm.config import get_current_vllm_config @@ -105,7 +105,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: self.lora_config = lora_config if isinstance(self.base_layer, ReplicatedLinear): diff --git a/vllm/lora/layers/classifier.py b/vllm/lora/layers/classifier.py index aa42023763..58160d1931 100644 --- a/vllm/lora/layers/classifier.py +++ b/vllm/lora/layers/classifier.py @@ -3,7 +3,7 @@ import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.platforms import current_platform @@ -16,7 +16,7 @@ class ClassificationHeadWithLoRA(ReplicatedLinearWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: # Preserve ordinary LoRA A/B support for classification heads. super().create_lora_weights(max_loras, lora_config, model_config) diff --git a/vllm/lora/layers/column_parallel_linear.py b/vllm/lora/layers/column_parallel_linear.py index d015fd53ce..06be86d388 100644 --- a/vllm/lora/layers/column_parallel_linear.py +++ b/vllm/lora/layers/column_parallel_linear.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.distributed import tensor_model_parallel_all_gather @@ -164,7 +164,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: if type(source_layer) is maybe_get_oot_by_class(ColumnParallelLinear): return True @@ -208,7 +208,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: """The main reason for overriding this function is to enhance code maintainability. @@ -366,7 +366,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear) @@ -442,7 +442,7 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: return ( type(source_layer) is maybe_get_oot_by_class(QKVParallelLinear) @@ -488,7 +488,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: """The main reason for overloading this function is to handle inconsistent weight dimensions in qkv lora. @@ -502,7 +502,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: return ( @@ -543,7 +543,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: # specifying kwargs so they can be easily accessed in decorator @@ -585,7 +585,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: # specifying kwargs so they can be easily accessed in decorator @@ -621,7 +621,7 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: # specifying kwargs so they can be easily accessed in decorator return super().can_replace_layer( @@ -669,7 +669,7 @@ class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: # specifying kwargs so they can be easily accessed in decorator @@ -698,7 +698,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA( source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, decorate: bool = True, ) -> bool: # Support MergedColumnParallelLinear with 3 or more slices diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index ed0ccdf7ae..c58c18e71f 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import envs from vllm.config.lora import LoRAConfig @@ -245,7 +245,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: """Initializes lora matrices.""" self._verify_ep_fs(lora_config) @@ -465,7 +465,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: """Returns True if the layer can be replaced by this LoRA layer.""" # source_layer is MoERunner @@ -511,7 +511,7 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: """Initializes lora matrices.""" if model_config is None: @@ -624,7 +624,7 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: """Returns True if the layer can be replaced by this LoRA layer.""" # source_layer is MoERunner diff --git a/vllm/lora/layers/logits_processor.py b/vllm/lora/layers/logits_processor.py index 7613e9d165..6259fd7817 100644 --- a/vllm/lora/layers/logits_processor.py +++ b/vllm/lora/layers/logits_processor.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.distributed import ( @@ -85,7 +85,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: # TODO: Verify if this condition can be further relaxed if self.base_layer.vocab_size > 258048: @@ -221,7 +221,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: # Special handling for the LogitsProcessor. return False diff --git a/vllm/lora/layers/replicated_linear.py b/vllm/lora/layers/replicated_linear.py index f62dc60b16..b4d61caca2 100644 --- a/vllm/lora/layers/replicated_linear.py +++ b/vllm/lora/layers/replicated_linear.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.model_executor.custom_op import maybe_get_oot_by_class @@ -61,7 +61,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: return isinstance(source_layer, maybe_get_oot_by_class(ReplicatedLinear)) diff --git a/vllm/lora/layers/row_parallel_linear.py b/vllm/lora/layers/row_parallel_linear.py index 8f607dfbe5..48cbd31c1b 100644 --- a/vllm/lora/layers/row_parallel_linear.py +++ b/vllm/lora/layers/row_parallel_linear.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.distributed import ( @@ -89,7 +89,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: return type(source_layer) is maybe_get_oot_by_class(RowParallelLinear) @@ -165,7 +165,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: # specifying kwargs so they can be easily accessed in decorator return super().can_replace_layer( diff --git a/vllm/lora/layers/vocab_parallel_embedding.py b/vllm/lora/layers/vocab_parallel_embedding.py index c6f294efcd..b55f33e149 100644 --- a/vllm/lora/layers/vocab_parallel_embedding.py +++ b/vllm/lora/layers/vocab_parallel_embedding.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.lora import LoRAConfig from vllm.model_executor.custom_op import maybe_get_oot_by_class @@ -26,7 +26,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): self, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> None: self.lora_a_stacked = torch.zeros( ( @@ -113,7 +113,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> bool: # Accept both the in-tree and any OOT class, because layers built on top of # `VocabParallelEmbedding` (e.g. by the Transformers modelling backend) use the diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index f6ef6974dc..703f766889 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -420,6 +420,10 @@ def _run_fused_moe_lora_one_shot( npid = min(npid_occ, max_npid_by_budget) npid = max(1, min(npid, max(1, N_per_slice // 128))) + # see issue: https://github.com/intel/intel-xpu-backend-for-triton/issues/8121 + if current_platform.is_xpu(): + npid = 1 + # Robust defaults across the prefill regime (H100/H200/B200, bf16/fp16). # NPID > 1 is the small-M / under-saturated path -- more warps help # amortise the inner-N expand loop. ns=3 instead of 4: GB200 ncu showed diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py index b1321e9913..71a2e60dc7 100644 --- a/vllm/lora/utils.py +++ b/vllm/lora/utils.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING from huggingface_hub.utils import HfHubHTTPError, HFValidationError from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import envs from vllm.config.lora import LoRAConfig @@ -109,7 +109,7 @@ def from_layer( max_loras: int, lora_config: LoRAConfig, packed_modules_list: list, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> nn.Module: for lora_cls in _all_lora_classes: # specifying kwargs so they can be easily accessed in decorator @@ -130,7 +130,7 @@ def from_layer_logits_processor( lm_head: "ParallelLMHead", max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> LogitsProcessorWithLoRA: ret = LogitsProcessorWithLoRA( layer, @@ -147,7 +147,7 @@ def from_layer_classification( layer: nn.Module, max_loras: int, lora_config: LoRAConfig, - model_config: PretrainedConfig | None = None, + model_config: PreTrainedConfig | None = None, ) -> ClassificationHeadWithLoRA: instance_layer = ClassificationHeadWithLoRA(layer) instance_layer.create_lora_weights(max_loras, lora_config, model_config) diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 08035d0e38..308e8c4f42 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -1150,7 +1150,12 @@ def override_envs_for_invariance(): os.environ["NCCL_MIN_NCHANNELS"] = "1" os.environ["NCCL_MAX_NCHANNELS"] = "1" os.environ["NCCL_PROTO"] = "Simple" - os.environ["NCCL_ALGO"] = "allreduce:tree" + # NCCL >= 2.31 zero-fills the algorithm table of every collective when + # NCCL_ALGO is set and re-enables only the named ones; together with the + # NCCL_PROTO above, collectives not named here end up with no algorithm + # and fail with ncclInvalidUsage. Re-enable Ring and Tree for all + # collectives, then pin AllReduce to Tree for determinism. + os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree" os.environ["NCCL_NTHREADS"] = "1" os.environ["NCCL_SOCKET_NTHREADS"] = "1" diff --git a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py b/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py index 345c711ae6..aeeb2f63b4 100644 --- a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py +++ b/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py @@ -146,19 +146,21 @@ def _sparse_topk_remap_kernel( out_ptr, out_stride, k, + width, SBK: tl.constexpr, K_POW2: tl.constexpr, ): """Remap sparse top-k columns to request-local positions. Column ``j * SBK + o`` scores the token at - ``(si[row, j] - ks // SBK) * SBK + o``; -1 columns stay -1. + ``(si[row, j] - ks // SBK) * SBK + o``; columns outside ``[0, width)`` + (including DeepSelect's sentinel for NaN rows) become -1. """ row = tl.program_id(0) ks_base = tl.load(ks_ptr + row * ks_stride) // SBK cols = tl.arange(0, K_POW2) c = tl.load(col_ptr + row * col_stride + cols, mask=cols < k, other=-1) - valid = c >= 0 + valid = (c >= 0) & (c < width) cc = tl.where(valid, c, 0) blocks = tl.load(si_ptr + row * si_stride + cc // SBK) pos = (blocks - ks_base) * SBK + cc % SBK @@ -301,6 +303,7 @@ def sparse_topk_remap( topk_indices, topk_indices.stride(0), topk_tokens, + width, SBK=sparse_block_kv, K_POW2=triton.next_power_of_2(topk_tokens), ) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 764bb73e88..e5fed78c40 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -26,9 +26,6 @@ from vllm.model_executor.kernels.linear.mixed_precision import ( MPLinearKernel, MPLinearLayerConfig, ) -from vllm.model_executor.kernels.linear.mixed_precision.allspark import ( - AllSparkLinearKernel, -) from vllm.model_executor.kernels.linear.mixed_precision.conch import ( ConchLinearKernel, ) @@ -504,7 +501,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = { PlatformEnum.CUDA: [ CutlassW4A8LinearKernel, MacheteLinearKernel, - AllSparkLinearKernel, MarlinLinearKernel, ConchLinearKernel, ExllamaLinearKernel, @@ -884,10 +880,15 @@ def choose_mp_linear_kernel( ) -def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel: +def init_mxfp8_linear_kernel( + *, weight_shape: tuple[int, int], bmm_batch_size: int | None = None +) -> Mxfp8LinearKernel: """Select and instantiate the best MXFP8 linear kernel for the - current platform.""" - config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size) + current platform and `(N, K)` weight shape.""" + config = Mxfp8LinearLayerConfig( + weight_shape=weight_shape, + bmm_batch_size=bmm_batch_size, + ) platform = current_platform._enum possible: list[type[Mxfp8LinearKernel]] @@ -1272,7 +1273,6 @@ __all__ = [ "ZentorchWNA16LinearKernel", "MPLinearKernel", "MPLinearLayerConfig", - "AllSparkLinearKernel", "ConchLinearKernel", "CPUWNA16LinearKernel", "CutlassW4A8LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py index c1ff798a47..6c786eed35 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py @@ -1,9 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.model_executor.kernels.linear.mixed_precision.allspark import ( - AllSparkLinearKernel, -) from vllm.model_executor.kernels.linear.mixed_precision.conch import ( ConchLinearKernel, ) @@ -52,7 +49,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.zentorch import ( __all__ = [ "MPLinearKernel", "MPLinearLayerConfig", - "AllSparkLinearKernel", "ConchLinearKernel", "CPUWNA16LinearKernel", "CutlassW4A8LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mixed_precision/allspark.py b/vllm/model_executor/kernels/linear/mixed_precision/allspark.py deleted file mode 100644 index 704e661aed..0000000000 --- a/vllm/model_executor/kernels/linear/mixed_precision/allspark.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -import torch - -from vllm import _custom_ops as ops -from vllm.model_executor.layers.quantization.utils import replace_parameter -from vllm.model_executor.layers.quantization.utils.allspark_utils import ( - ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - check_allspark_supported_dtype_shape, -) -from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_ -from vllm.utils.platform_utils import num_compute_units - -from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig - - -class AllSparkLinearKernel(MPLinearKernel): - @classmethod - def get_min_capability(cls) -> int: - return 80 - - @classmethod - def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: - if c.zero_points: - return False, "Zero points currently not supported by AllSpark" - - return check_allspark_supported_dtype_shape( - c.partition_weight_shape[0], # in_features - c.partition_weight_shape[1], # out_features - c.group_size, - c.weight_type, - c.act_type, - ) - - # note assumes that - # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} - # `weight_scale` is: {input_dim = 0, output_dim = 1} - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - device = getattr(layer, self.w_q_name).device - c = self.config - - # prepare the parameters required for the kernel - properties = torch.cuda.get_device_properties(device.index) - sm_count = num_compute_units(device.index) - sm_version = properties.major * 10 + properties.minor - gemm_args = {} - gemm_args["sm_count"] = sm_count - gemm_args["sm_version"] = sm_version - - self.gemm_args = gemm_args - - # transform param weight, scale - old_weight_param = getattr(layer, self.w_q_name) - old_scale_param = getattr(layer, self.w_s_name) - - assert isinstance(old_weight_param, BasevLLMParameter) - permute_param_layout_(old_weight_param, input_dim=0, output_dim=1, packed_dim=0) - - assert isinstance(old_scale_param, BasevLLMParameter) - permute_param_layout_(old_scale_param, input_dim=0, output_dim=1) - - # unpack weight from K / 4 x N int32 to K x N uint8 - new_weight_param = torch.nn.Parameter( - old_weight_param.data, requires_grad=False - ) - new_weight_param.data = ( - new_weight_param.data.t().contiguous().view(dtype=torch.uint8) - ) - new_weight_param.data = new_weight_param.data.t().contiguous() - - new_scale_param = torch.nn.Parameter(old_scale_param.data, requires_grad=False) - - # reorder K x N weight as N32K16 format for Ampere W8A16 - new_weight_param.data, new_scale_param.data, _ = ops.allspark_repack_weight( - new_weight_param.data, new_scale_param.data, None, c.zero_points - ) - - replace_parameter(layer, self.w_q_name, new_weight_param.data) - replace_parameter(layer, self.w_s_name, new_scale_param.data) - - def apply_weights( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - c = self.config - gemm_args = self.gemm_args - w_q, w_s, _ = self._get_weight_params(layer) - - reshaped_x = x.reshape(-1, x.shape[-1]) - out_shape = x.shape[:-1] + (c.partition_weight_shape[1],) - - output = ops.allspark_w8a16_gemm( - a=reshaped_x, - b_qweight=w_q, - b_scales=w_s, - b_qzeros=None, - n=c.partition_weight_shape[1], - group_size=c.group_size, - sm_count=gemm_args["sm_count"], - sm_version=gemm_args["sm_version"], - CUBLAS_M_THRESHOLD=ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD, - has_zp=c.zero_points, - n32k16_reorder=True, - ) - - if bias is not None: - output.add_(bias) # In-place add - - return output.reshape(out_shape) diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index b98dfee18b..4655181f78 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel): ) name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name} + if self.w_zp_name is not None and hasattr(layer, self.w_zp_name): + name_map["zero_point"] = self.w_zp_name group_size = self.config.group_size quant_config = { "quant_method": "humming", "dtype": "int" + str(self.config.weight_type.size_bits), "group_size": 0 if group_size == -1 else group_size, + "has_zero_point": self.config.zero_points, } if self.config.zero_points: diff --git a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py index fdcfeb264b..3a8b886b4c 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py +++ b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py @@ -15,8 +15,13 @@ class Mxfp8LinearLayerConfig: All MXFP8 layers share the same structure: FP8-E4M3 weights with uint8 (E8M0) per-block scales at block size 32. + + Attributes: + weight_shape: The layer's `(out_features, in_features)`, i.e. `(N, K)`. + """ + weight_shape: tuple[int, int] bmm_batch_size: int | None = None diff --git a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py index cc9b35d34b..889b260435 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py +++ b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py @@ -24,6 +24,20 @@ from vllm.utils.flashinfer import has_flashinfer, has_flashinfer_cutedsl from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig +def _check_mm_mxfp8_shape(weight_shape: tuple[int, int]) -> tuple[bool, str | None]: + N, K = weight_shape + if K < 128: + return False, f"mm_mxfp8 requires K >= 128, got K={K}." + if K % MXFP8_BLOCK_SIZE != 0: + return ( + False, + f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}.", + ) + if N < 128: + return False, f"mm_mxfp8 requires N >= 128, got N={N}." + return True, None + + class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): """MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+).""" @@ -43,6 +57,15 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): @classmethod def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: + can_implement, reason = _check_mm_mxfp8_shape(c.weight_shape) + if not can_implement: + return False, reason + N, _ = c.weight_shape + if current_platform.is_device_capability_family(120) and N % 32 != 0: + return ( + False, + f"mm_mxfp8 on SM12x requires N to be divisible by 32, got N={N}.", + ) return True, None def input_quant_key(self) -> QuantKey: @@ -72,20 +95,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): weight_scale = layer.weight_scale N, K = weight.shape - min_dim = 128 - - assert min_dim <= K, ( - f"mm_mxfp8 requires K >= {min_dim}, got K={K}. " - f"in_features is too small for mm_mxfp8." - ) - assert K % MXFP8_BLOCK_SIZE == 0, ( - f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}." - ) - assert min_dim <= N, ( - f"mm_mxfp8 requires N >= {min_dim}, got N={N}. " - f"out_features is too small for mm_mxfp8." - ) - qa = as_quantized_activation(x, self.input_quant_key()) if qa is not None: input_mxfp8, input_scale = qa.data, qa.scale @@ -136,7 +145,7 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel): @classmethod def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: - return True, None + return _check_mm_mxfp8_shape(c.weight_shape) def input_quant_key(self) -> QuantKey: # Activations use FlashInfer's F8_128x4 swizzled scale layout. @@ -166,20 +175,6 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel): weight_scale = layer.weight_scale K, N = weight.shape - min_dim = 128 - - assert min_dim <= K, ( - f"mm_mxfp8 requires K >= {min_dim}, got K={K}. " - f"in_features is too small for mm_mxfp8." - ) - assert K % MXFP8_BLOCK_SIZE == 0, ( - f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}." - ) - assert min_dim <= N, ( - f"mm_mxfp8 requires N >= {min_dim}, got N={N}. " - f"out_features is too small for mm_mxfp8." - ) - qa = as_quantized_activation(x, self.input_quant_key()) if qa is not None: input_mxfp8, input_scale = qa.data, qa.scale diff --git a/vllm/model_executor/kernels/linear/mxfp8/marlin.py b/vllm/model_executor/kernels/linear/mxfp8/marlin.py index 6ebdbd6474..45824e0258 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/marlin.py +++ b/vllm/model_executor/kernels/linear/mxfp8/marlin.py @@ -3,6 +3,10 @@ import torch +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_BLOCK_SIZE, +) + from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig @@ -23,6 +27,16 @@ class MarlinMxfp8LinearKernel(Mxfp8LinearKernel): @classmethod def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: + N, K = c.weight_shape + if N <= 0 or K <= 0: + return False, f"Marlin MXFP8 requires positive N and K, got N={N}, K={K}." + if K % MXFP8_BLOCK_SIZE != 0: + return ( + False, + f"Marlin MXFP8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, " + f"got K={K}.", + ) + # Tile misalignment is fixed by zero-padding at weight prep. return True, None def process_weights_after_loading(self, layer: torch.nn.Module) -> None: diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py new file mode 100644 index 0000000000..875c5314be --- /dev/null +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py @@ -0,0 +1,737 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Portions adapted from ROCm/aiter (https://github.com/ROCm/aiter/pull/5750): +# SPDX-License-Identifier: MIT +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``). + +``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per +row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in +the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being +expanded to every row. Each weight scale byte is then read once per 32 output +rows, and small-M shapes can use the packed kernel below. + +Two kernels, picked per shape from a table tuned on MI355X: + +* a tiled kernel for larger M; +* a packed kernel for small M, which fills the MFMA's rows with K panels + instead of tokens and keeps only the matching-panel (block-diagonal) + products, so a handful of tokens still streams the weight at full width. + +Either can split K. The partials are normally summed in the same launch by +the last program of each output tile to finish, in split order, so the result +does not depend on scheduling. Otherwise a second launch reduces them. + +The packed kernel and the in-launch split-K reduction on one XCD +(``_split_tile``, ``_sum_splits``, ``_split_counters``, ``_k_partition``) are +adapted from the group32 GEMM in ROCm/aiter#5750. +""" + +from typing import NamedTuple + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + +BLOCK_ROWS = 32 + +# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is +# a multiple of the XCD count in every partition mode. _split_tile hardcodes +# the same stride. +_XCD_STRIDE = 8 +# One arrival counter per output tile of an in-launch split-K GEMM. +_SPLIT_COUNTERS = 1 << 16 + + +@triton.jit +def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr): + """(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on + one XCD, so they meet in that XCD's L2.""" + pid = tl.program_id(0) + tile = pid // (8 * SPLITS) * 8 + pid % 8 + grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N + return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS + + +@triton.jit +def _sum_splits( + acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr +): + """Publish this split's partial; the tile's last arrival sums all of them + in split order and re-arms the counter.""" + BM: tl.constexpr = acc.shape[0] + BN: tl.constexpr = acc.shape[1] + local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :] + tl.store(slot_ptr + split * BM * BN + local, acc) + # The partial must be in L2 before the arrival is counted. The splits + # share an XCD, so no device-scope release (an L2 writeback) is needed. + tl.inline_asm_elementwise( + "s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1 + ) + tl.debug_barrier() + if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1: + total = tl.zeros((BM, BN), dtype=tl.float32) + for s in tl.static_range(SPLITS): + total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv") + tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask) + tl.store(count_ptr, 0) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_tiled_kernel( + x_ptr, + xs_ptr, + w_ptr, + ws_ptr, + out_ptr, + slot_ptr, + count_ptr, + M, + N: tl.constexpr, + K: tl.constexpr, + stride_om, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + K_PER_SPLIT: tl.constexpr, + N_FIRST: tl.constexpr, + EVEN_K: tl.constexpr, + FUSED_SPLITS: tl.constexpr, +): + if FUSED_SPLITS > 1: + pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS) + if pid_m * BLOCK_M >= M: + return + else: + # N_FIRST launches output-column tiles fastest, so consecutive + # programs share an activation tile. + if N_FIRST: + pid_n, pid_m = tl.program_id(0), tl.program_id(1) + else: + pid_m, pid_n = tl.program_id(0), tl.program_id(1) + pid_k = tl.program_id(2) + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + m_mask = offs_m < M + n_mask = offs_n < N + k0 = pid_k * K_PER_SPLIT + offs_k = k0 + tl.arange(0, BLOCK_K) + offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32) + x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :] + xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :] + w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :] + ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :] + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for kk in range(0, K_PER_SPLIT, BLOCK_K): + if EVEN_K: + x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0) + xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127) + w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0) + ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127) + else: + k_ok = (offs_k + kk) < K + s_ok = (offs_sk + kk // 32) < K // 32 + x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0) + xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127) + w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0) + ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127) + acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) + x_ptrs += BLOCK_K + w_ptrs += BLOCK_K + xs_ptrs += BLOCK_K // 32 + ws_ptrs += BLOCK_K // 32 + out_mask = m_mask[:, None] & n_mask[None, :] + if FUSED_SPLITS > 1: + _sum_splits( + acc, + out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], + out_mask, + slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N), + count_ptr + tile, + pid_k, + FUSED_SPLITS, + ) + else: + out_ptrs = ( + out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :] + ) + tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_packed_kernel( + x_ptr, + xs_ptr, + w_ptr, + ws_ptr, + out_ptr, + slot_ptr, + count_ptr, + M, + N: tl.constexpr, + K: tl.constexpr, + stride_om, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + K_PACK: tl.constexpr, + K_PER_SPLIT: tl.constexpr, + EVEN_K: tl.constexpr, + FUSED_SPLITS: tl.constexpr, +): + if FUSED_SPLITS > 1: + pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS) + if pid_m * BLOCK_M >= M: + return + k_start = pid_k * K_PER_SPLIT + k_stop = tl.minimum(k_start + K_PER_SPLIT, K) + else: + pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2) + if K_PER_SPLIT >= K: + k_start = 0 + k_stop = K + else: + k_start = pid_k * K_PER_SPLIT + k_stop = tl.minimum(k_start + K_PER_SPLIT, K) + # Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise + # for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices. + rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK + x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK + cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK + w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK + ks = tl.arange(0, BLOCK_K) + gs = tl.arange(0, BLOCK_K // 32) + row_ok = rows < M + col_ok = cols < N + acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32) + for base in range(k_start, k_stop, BLOCK_K * K_PACK): + xk = base + x_panel[:, None] * BLOCK_K + ks[None, :] + wk = base + w_panel[:, None] * BLOCK_K + ks[None, :] + xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :] + wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :] + x_mask = row_ok[:, None] + w_mask = col_ok[:, None] + xs_mask = row_ok[:, None] + ws_mask = col_ok[:, None] + if not EVEN_K: + x_mask = x_mask & (xk < K) + w_mask = w_mask & (wk < K) + xs_mask = xs_mask & (xg < K // 32) + ws_mask = ws_mask & (wg < K // 32) + x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0) + w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0) + xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127) + ws = tl.load( + ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127 + ) + acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) + # Keep only products of matching K panels: the block diagonal. + blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3) + pair = tl.arange(0, K_PACK) + diagonal = tl.where( + pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0 + ) + y = tl.sum(tl.sum(diagonal, 3), 2) + out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :] + out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N) + if FUSED_SPLITS > 1: + _sum_splits( + y, + out_ptrs, + out_mask, + slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N), + count_ptr + tile, + pid_k, + FUSED_SPLITS, + ) + else: + tl.store( + out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask + ) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_splitk_reduce_kernel( + partial_ptr, + out_ptr, + M, + N: tl.constexpr, + stride_om, + SPLIT_K: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask = (offs_m[:, None] < M) & (offs_n[None, :] < N) + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for split in tl.static_range(SPLIT_K): + acc += tl.load( + partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :], + mask=mask, + other=0.0, + ) + tl.store( + out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=mask, + ) + + +class _Config(NamedTuple): + packed: bool + block_m: int + block_n: int + block_k: int + # K panels per MFMA tile (packed) or K partitions reduced by a second + # launch (tiled). + k_split: int + n_first: bool + num_warps: int + num_stages: int + matrix_instr_nonkdim: int + waves_per_eu: int + # K partitions summed within the launch. + fused_splits: int = 1 + + +def _tiled( + bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1 +) -> _Config: + return _Config( + False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused + ) + + +def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config: + return _Config( + True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused + ) + + +# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at +# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A +# tier serves every M up to its bound; larger M reuse the last tier. +_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = { + # shared expert gate_up, TP4 + (1152, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)), + (2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)), + (4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)), + (8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)), + (16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)), + (24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)), + (32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)), + (48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)), + (64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)), + (96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)), + (128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)), + (160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)), + (192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)), + (256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)), + (384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)), + (512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)), + (1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)), + (2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + ], + # fused wq_a|wkv (replicated) + (1792, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)), + (8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)), + (16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)), + (32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)), + (48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)), + (64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)), + (96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)), + (128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)), + (192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)), + (384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)), + ], + # shared expert gate_up, TP2 + (2304, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)), + (8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)), + (16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)), + (24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)), + (32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)), + (48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)), + (64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)), + (128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)), + (256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)), + (4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)), + ], + # indexer wq_b + (4096, 1280): [ + (1, _packed(4, 16, 256, 4, 4, 2, 16, 1)), + (2, _packed(4, 16, 256, 4, 4, 2, 16, 0)), + (4, _packed(4, 16, 256, 4, 4, 2, 16, 1)), + (8, _packed(8, 16, 256, 4, 4, 3, 16, 1)), + (16, _packed(16, 16, 512, 2, 2, 3, 16, 1)), + (32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)), + (64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)), + (128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)), + (192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)), + (256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)), + ], + # shared expert down, TP4 + (5120, 576): [ + (8, _packed(8, 32, 256, 2, 2, 2, 16, 1)), + (16, _packed(16, 32, 512, 1, 2, 2, 16, 0)), + (32, _packed(32, 32, 256, 1, 8, 3, 16, 1)), + (64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)), + (96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)), + (128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)), + (192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)), + (256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)), + (768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + ], + # shared expert down, TP2 + (5120, 1152): [ + (1, _packed(4, 32, 128, 4, 2, 3, 16, 1)), + (8, _packed(8, 32, 256, 2, 4, 3, 16, 0)), + (32, _packed(32, 32, 512, 1, 2, 3, 16, 1)), + (128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)), + (1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)), + ], + # wo_b, TP4 + (5120, 2048): [ + (1, _packed(4, 32, 256, 4, 2, 2, 16, 0)), + (4, _packed(4, 32, 256, 4, 2, 2, 16, 1)), + (8, _packed(8, 32, 256, 4, 2, 3, 16, 0)), + (16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)), + (32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)), + (64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)), + (96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)), + (128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)), + ], + # wo_b, TP2 + (5120, 4096): [ + (4, _packed(4, 32, 512, 4, 2, 2, 16, 0)), + (8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)), + (16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)), + (24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)), + (32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)), + (48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)), + (64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)), + (128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)), + (3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + ], + # wq_b, TP4 + (8192, 1280): [ + (1, _packed(8, 32, 256, 2, 4, 3, 16, 1)), + (8, _packed(8, 32, 256, 2, 4, 3, 16, 0)), + (16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)), + (32, _packed(32, 32, 512, 1, 2, 3, 16, 0)), + (48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)), + (64, _packed(32, 64, 256, 1, 4, 3, 16, 1)), + (128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + ], + # wq_b, TP2 + (16384, 1280): [ + (2, _packed(4, 32, 128, 4, 2, 3, 16, 1)), + (4, _packed(4, 32, 128, 4, 4, 2, 16, 0)), + (8, _packed(8, 32, 256, 2, 4, 2, 16, 0)), + (32, _packed(32, 64, 256, 1, 4, 3, 16, 1)), + (64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)), + (128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + ], +} + + +def _default_config(M: int, N: int, K: int) -> _Config: + """Untuned shapes: the tiers most tuned shapes settle on.""" + if M <= 8: + return _packed(8, 16, 256, 2, 2, 3, 16, 0) + if M <= 32: + return _packed(32, 32, 256, 1, 2, 3, 16, 0) + if M <= 256: + return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0) + return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0) + + +def _config(M: int, N: int, K: int) -> _Config: + tiers = _TUNED.get((N, K)) + if tiers is None: + return _default_config(M, N, K) + for max_m, cfg in tiers: + if max_m >= M: + return cfg + return tiers[-1][1] + + +_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {} + + +def _split_counters(device: torch.device) -> torch.Tensor | None: + """Zeroed arrival counters, one set per stream so concurrent launches never + share a tile's counter. Each launch leaves the counters it used at zero. + + None when a stream first needs them inside a CUDA graph capture: that + allocation would come from the graph's pool, where it can take the address + of an intermediate that every replay overwrites. vLLM warms up on its + capture streams first, so this only affects captures without a warmup. + """ + key = (device.index, torch.accelerator.current_stream(device).stream_id) + counters = _split_counters_by_stream.get(key) + if counters is None: + if torch.cuda.is_current_stream_capturing(): + return None + counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device) + _split_counters_by_stream[key] = counters + return counters + + +def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]: + """K per split, rounded up to whole steps, and the number of non-empty + splits.""" + size = triton.cdiv(triton.cdiv(K, splits), step) * step + return size, triton.cdiv(K, size) + + +def _launch( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out: torch.Tensor, + cfg: _Config, +) -> None: + M, K = x.shape + N = weight.shape[0] + opts = dict( + num_warps=cfg.num_warps, + num_stages=cfg.num_stages, + matrix_instr_nonkdim=cfg.matrix_instr_nonkdim, + waves_per_eu=cfg.waves_per_eu, + ) + grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n) + tiles = grid_m * grid_n + step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k + splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split + k_per_split, splits = _k_partition(K, splits, step) + counters = None + if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS: + counters = _split_counters(out.device) + if counters is not None: + fused = splits + target = out + slots = torch.empty( + tiles * fused * cfg.block_m * cfg.block_n, + dtype=torch.float32, + device=out.device, + ) + grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,) + else: + # Without counters, split K through a second launch. + fused = 1 + target = ( + out + if splits == 1 + else torch.empty((splits, M, N), dtype=torch.float32, device=x.device) + ) + slots = counters = target # Unused by the kernels. + if cfg.packed or not cfg.n_first: + grid = (grid_m, grid_n, splits) + else: + grid = (grid_n, grid_m, splits) + args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K) + stride = out.stride(0) if splits == 1 or fused > 1 else N + if cfg.packed: + _block32_packed_kernel[grid]( + *args, + stride, + BLOCK_M=cfg.block_m, + BLOCK_N=cfg.block_n, + BLOCK_K=cfg.block_k, + K_PACK=cfg.k_split, + K_PER_SPLIT=k_per_split, + EVEN_K=K % step == 0, + FUSED_SPLITS=fused, + **opts, + ) + else: + _block32_tiled_kernel[grid]( + *args, + stride, + BLOCK_M=cfg.block_m, + BLOCK_N=cfg.block_n, + BLOCK_K=cfg.block_k, + K_PER_SPLIT=k_per_split, + N_FIRST=cfg.n_first, + EVEN_K=K % k_per_split == 0, + FUSED_SPLITS=fused, + **opts, + ) + if fused == 1 and splits > 1: + _block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))]( + target, + out, + M, + N, + out.stride(0), + SPLIT_K=splits, + BLOCK_M=32, + BLOCK_N=64, + ) + + +def _rocm_mxfp8_block32_gemm_impl( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + M, K = x.shape + out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device) + if M > 0: + _launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K)) + return out + + +def _rocm_mxfp8_block32_gemm_fake( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype) + + +# An opaque op: the kernel and tile choice branch on the token count, which a +# compiled caller must not freeze at its trace-time value. +direct_register_custom_op( + op_name="rocm_mxfp8_block32_gemm", + op_func=_rocm_mxfp8_block32_gemm_impl, + fake_impl=_rocm_mxfp8_block32_gemm_fake, +) + + +def rocm_mxfp8_block32_gemm( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + """``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight. + + Args: + x: [M, K] e4m3 activation, contiguous. + x_scale: [M, K / 32] E8M0 (uint8) activation scales. + weight: [N, K] e4m3 weight, contiguous. + weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales. + out_dtype: Output dtype. + + Returns: + The [M, N] product. + + """ + return torch.ops.vllm.rocm_mxfp8_block32_gemm( + x, x_scale, weight, weight_scale, out_dtype + ) diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py index c7e6537f49..faf1e180fa 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py @@ -22,6 +22,7 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig +from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm @triton.jit @@ -119,12 +120,10 @@ def _mxfp8_dot_scaled_linear( return out -# Triton 3.8 enables TRITON_HIP_USE_ASYNC_COPY by default on gfx950; its extra LDS -# buffer leaves no room for num_stages=3 at BLOCK_K=256. -# TODO(rasmith)(Rohan138): Remove the 3.8 check once -# https://github.com/vllm-project/vllm/pull/50605 merges. +# gfx950's async-copy LDS (default in the triton 3.8 this branch pins) leaves no +# room for num_stages=3 at BLOCK_K=256; use 2 there. _BK256_STAGES = 3 -if triton.__version__.startswith("3.8") and current_platform.is_rocm(): +if current_platform.is_rocm(): from vllm.platforms.rocm import on_gfx950 if on_gfx950(): @@ -209,6 +208,17 @@ def _select_cfg(M, N, K): _DOT_SCALED_K_ALIGN = 128 +def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None: + """The [N / 32, K / 32] block scales, if every 32 rows share their scales.""" + N = weight_scale.shape[0] + if N % BLOCK_ROWS != 0: + return None + blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1) + if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)): + return None + return blocks[:, 0].contiguous() + + class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): """Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``.""" @@ -236,7 +246,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): N, K = weight.shape scale_k = K // MXFP8_BLOCK_SIZE weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous() - if K % _DOT_SCALED_K_ALIGN != 0: + block_scale = _as_block32_scale(weight_scale) + if block_scale is not None: + # Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them + # back from the loader expanded per row; keep the blocks and use + # the block-scaled GEMM, which also handles any K % 32 == 0. + weight_scale = block_scale + elif K % _DOT_SCALED_K_ALIGN != 0: weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale) layer.weight = Parameter(weight.contiguous(), requires_grad=False) layer.weight_scale = Parameter(weight_scale, requires_grad=False) @@ -254,7 +270,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): ) out_shape = (*x.shape[:-1], layer.weight.shape[0]) x2d = x.reshape(-1, x.shape[-1]) - if layer.weight.element_size() >= 2: + if layer.weight_scale.shape[0] != layer.weight.shape[0]: + # One scale row per 32 weight rows (see process_weights_after_loading). + x_q, x_scale = mxfp8_e4m3_quantize(x2d) + out = rocm_mxfp8_block32_gemm( + x_q, x_scale, layer.weight, layer.weight_scale, x.dtype + ) + elif layer.weight.element_size() >= 2: out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype)) elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0: out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale) diff --git a/vllm/model_executor/kernels/mhc/cpu.py b/vllm/model_executor/kernels/mhc/cpu.py index 4659540698..d12b0eae18 100644 --- a/vllm/model_executor/kernels/mhc/cpu.py +++ b/vllm/model_executor/kernels/mhc/cpu.py @@ -81,7 +81,7 @@ def hc_head_fused_cpu( hc_eps: float, ) -> torch.Tensor: """CPU-ported HC head reduction (see `test_hc_head_cpu` in - tests/kernels/test_mhc_kernels.py for the eager reference this is tested + tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested against). The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the diff --git a/vllm/model_executor/kernels/mhc/tilelang_kernels.py b/vllm/model_executor/kernels/mhc/tilelang_kernels.py index 33de3b9ce5..f7f560f50d 100644 --- a/vllm/model_executor/kernels/mhc/tilelang_kernels.py +++ b/vllm/model_executor/kernels/mhc/tilelang_kernels.py @@ -23,6 +23,23 @@ from vllm.utils.math_utils import cdiv ENABLE_PDL = current_platform.is_arch_support_pdl() and current_platform.is_cuda() +# TileLang's HIP AllReduce shuffle phase ignores a nonzero reduction-group +# offset, so the 64-thread RMSNorm reduction must be wave64-aligned and use +# threads 0..63. +_MIX_ON_TRAILING_THREADS = current_platform.is_rocm() + + +def _is_mix_thread(): + if _MIX_ON_TRAILING_THREADS: + return T.get_thread_binding() >= 64 + return T.get_thread_binding() < 32 + + +def _is_collapse_thread(): + if _MIX_ON_TRAILING_THREADS: + return T.get_thread_binding() < 64 + return T.get_thread_binding() >= 32 + @cache def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int: @@ -353,7 +370,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( mixes[j] *= rms[0] T.copy(mixes, mixes_shared) - if split_mode != "input" and T.get_thread_binding() < 32: + if split_mode != "input" and _is_mix_thread(): cm = T.alloc_fragment((hc_mult, hc_mult), T.float32) for j in T.Parallel(hc_mult): if save_pre_mix: @@ -399,7 +416,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( for j, k in T.Parallel(hc_mult, hc_mult): comb_mix[i, j * hc_mult + k] = cm[j, k] - elif split_mode != "stats" and T.get_thread_binding() >= 32: + elif split_mode != "stats" and _is_collapse_thread(): pre_mix_shared = T.alloc_shared(hc_mult, T.float32) for j in T.Parallel(hc_mult): if use_pre_mix_in: @@ -546,7 +563,7 @@ def mhc_pre_big_fuse_broadcast_with_norm_tilelang( mixes_shared = T.alloc_shared(hc_mult3, T.float32) T.copy(mixes, mixes_shared) - if T.get_thread_binding() < 32: + if _is_mix_thread(): cm = T.alloc_fragment((hc_mult, hc_mult), T.float32) for j in T.Parallel(hc_mult): post_mix[i, j] = ( diff --git a/vllm/model_executor/layers/activation.py b/vllm/model_executor/layers/activation.py index 35a6fdf9b0..808d6e7112 100644 --- a/vllm/model_executor/layers/activation.py +++ b/vllm/model_executor/layers/activation.py @@ -233,12 +233,8 @@ class SiluAndMulWithClamp(CustomOp): self.swiglu_limit = float(swiglu_limit) self.alpha = float(alpha) self.beta = float(beta) - if current_platform.is_rocm() or current_platform.is_xpu(): - self._forward_method = self.forward_native - elif current_platform.is_cuda_alike(): + if current_platform.is_cuda_alike(): self.op = torch.ops._C.silu_and_mul_with_clamp - elif current_platform.is_cpu(): - self._forward_method = self.forward_native def forward_native(self, x: torch.Tensor) -> torch.Tensor: d = x.shape[-1] // 2 @@ -253,7 +249,11 @@ class SiluAndMulWithClamp(CustomOp): self.op(out, x, self.swiglu_limit, self.alpha, self.beta) return out - def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + def forward_hip(self, x: torch.Tensor) -> torch.Tensor: + # Limit the ROCm _C path to the default case to avoid the + # precision loss MiniMax saw without fp32 intermediates. + if self.alpha == 1.0 and self.beta == 0.0: + return self.forward_cuda(x) return self.forward_native(x) def extra_repr(self) -> str: diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 79012a4a5c..bec1856d93 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -8,7 +8,10 @@ import torch.nn as nn import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CacheConfig, get_current_vllm_config +from vllm.config import ( + CacheConfig, + get_current_vllm_config, +) from vllm.config.vllm import VllmConfig from vllm.forward_context import ForwardContext, get_forward_context from vllm.logger import init_logger @@ -100,6 +103,7 @@ def _largest_kernel_block_within( per_token_bytes: int, page_budget: int, fallback: int, + kv_cache_spec: KVCacheSpec | None = None, ) -> int: """Largest supported kernel block size whose page fits in ``page_budget``. @@ -111,7 +115,7 @@ def _largest_kernel_block_within( """ from vllm.v1.attention.backend import MultipleOf - sizes = attn_backend.get_supported_kernel_block_sizes() + sizes = attn_backend.get_supported_kernel_block_sizes(kv_cache_spec) max_block_size = page_budget // per_token_bytes candidates = [s for s in sizes if isinstance(s, int)] candidates.extend( @@ -632,7 +636,7 @@ class Attention(nn.Module, AttentionLayerBase): # ``unify`` scales it up by an integer ratio. shared_page = vllm_config.cache_config.skip_page_size_padded # The backend owns its packing - sw_per_token = self.attn_backend.customize_spec( + kv_cache_spec = self.attn_backend.customize_spec( SlidingWindowSpec( block_size=1, num_kv_heads=self.num_kv_heads, @@ -642,10 +646,15 @@ class Attention(nn.Module, AttentionLayerBase): kv_quant_mode=quant_mode, sliding_window=self.sliding_window, ) - ).real_page_size_bytes + ) + sw_per_token = kv_cache_spec.real_page_size_bytes page_budget = shared_page or sw_per_token * block_size sw_block_size = _largest_kernel_block_within( - self.attn_backend, sw_per_token, page_budget, block_size + self.attn_backend, + sw_per_token, + page_budget, + block_size, + kv_cache_spec, ) return SlidingWindowSpec( block_size=sw_block_size, diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index c37801e887..2ae111c1c3 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -404,6 +404,24 @@ def _get_kv_b_proj_input_dtype( return weight_dtype +def split_kv_b_proj( + kv_b_proj: nn.Module, + out_dtype: torch.dtype, + kv_lora_rank: int, + num_heads: int, + qk_nope_head_dim: int, + v_head_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Dequantize ``kv_b_proj`` and return ``W_UK [L,N,P]``, ``W_UV [L,N,V]``.""" + weight = get_and_maybe_dequant_weights(kv_b_proj, out_dtype=out_dtype).T + assert weight.shape == ( + kv_lora_rank, + num_heads * (qk_nope_head_dim + v_head_dim), + ), f"kv_b_proj weight {tuple(weight.shape)} vs {kv_lora_rank=} {num_heads=}" + weight = weight.view(kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim) + return weight.split([qk_nope_head_dim, v_head_dim], dim=-1) + + class MLAAttention(nn.Module, AttentionLayerBase): """Multi-Head Latent Attention layer. @@ -717,13 +735,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): self.kv_cache = kv_cache.squeeze(1) if ( self._vllm_config.kernel_config.enable_jit_warmup - and self.attn_backend.get_name() - in ( - "FLASHMLA_SPARSE", - "FLASHINFER_MLA_SPARSE", - "FLASHINFER_MLA_SPARSE_SM120", - "DEEPSEEK_V32_INDEXER", - ) + and self._uses_flat_kv_cache() ): from vllm.v1.attention.backends.mla.sparse_utils import ( _CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL, @@ -736,6 +748,13 @@ class MLAAttention(nn.Module, AttentionLayerBase): block_stride_rows=self.kv_cache.stride(0) // row_width, ) + def _uses_flat_kv_cache(self) -> bool: + backend = self.attn_backend.get_name() + return backend == "FLASHINFER_MLA_SPARSE" or ( + backend == "FLASHMLA_SPARSE" + and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla") + ) + @property def chunked_prefill_workspace_size(self) -> int: if self._chunked_prefill_workspace_size is None: @@ -1025,9 +1044,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): mqa_q_pe = mqa_pe_padded if self.is_aiter_triton_fp4_bmm_enabled: - from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 - - mqa_ql_nope = batched_gemm_a16wfp4( + mqa_ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4( mqa_q_nope, self.W_K, self.W_K_scale, @@ -1223,13 +1240,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): ) return - # we currently do not have quantized bmm's which are needed for - # `W_UV` and `W_UK_T`, we just store fp16/bf16 copies and perform - # the bmm's in 16-bit, the extra memory overhead of this is fairly low - kv_b_proj_weight = get_and_maybe_dequant_weights( - self.kv_b_proj, out_dtype=act_dtype - ).T - if self.dcp_q_replicate: # qrep wired here: validate unsupported decode backends once. assert self.q_pad_num_heads in (None, self.num_heads), ( @@ -1245,24 +1255,13 @@ class MLAAttention(nn.Module, AttentionLayerBase): "FP4/FP8 MLA BMM paths." ) - assert kv_b_proj_weight.shape == ( - self.kv_lora_rank, - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), - ), ( - f"{kv_b_proj_weight.shape=}, " - f"{self.kv_lora_rank=}, " - f"{self.num_heads=}, " - f"{self.qk_nope_head_dim=}, " - f"{self.v_head_dim=}" - ) - kv_b_proj_weight = kv_b_proj_weight.view( + W_UK, W_UV = split_kv_b_proj( + self.kv_b_proj, + act_dtype, self.kv_lora_rank, self.num_heads, - self.qk_nope_head_dim + self.v_head_dim, - ) - - W_UK, W_UV = kv_b_proj_weight.split( - [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + self.qk_nope_head_dim, + self.v_head_dim, ) # If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported @@ -1367,11 +1366,20 @@ class MLAAttention(nn.Module, AttentionLayerBase): **common_kwargs, sliding_window=self.sliding_window, ) - return MLAAttentionSpec( + spec = MLAAttentionSpec( **common_kwargs, is_index_group_leader=self.indexer is not None, non_causal_multi_token_decode=self.non_causal_multi_token_decode, ) + # SM100 FlashMLA paged kernels also express TMA coordinates in token rows. + uses_tma_rows = ( + self.attn_backend.get_name() == "FLASHMLA_SPARSE" + and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla") + and current_platform.is_device_capability_family(100) + ) + if self._uses_flat_kv_cache() or uses_tma_rows: + spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes) + return spec def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor): # Convert from (B, N, L) to (N, B, L) @@ -2473,6 +2481,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> MLACommonDecodeMetadata: return MLACommonDecodeMetadata( @@ -2637,6 +2646,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1], query_start_loc_device=query_start_loc[: num_decodes + 1], num_decode_tokens=num_decode_tokens, + max_query_len=min( + common_attn_metadata.max_query_len, self.reorder_batch_threshold + ), dcp_tot_seq_lens_device=dcp_tot_seq_lens_device, ) diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index aed5afdcf9..ee7409b190 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -291,17 +291,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): self, common_attn_metadata: "CommonAttentionMetadata", ) -> torch.Tensor: - num_tokens = common_attn_metadata.num_actual_tokens - starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32) - seg_lengths = np.diff(starts) - req_id_per_token = np.repeat( - np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths - ) - self.req_id_per_token_buffer.fill_(0) - self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( - np_to_pinned_tensor(req_id_per_token), non_blocking=True - ) - return self.req_id_per_token_buffer[:num_tokens] + return common_attn_metadata.token_to_req_indices(self.req_id_per_token_buffer) def _build_chunked_context_fields( self, diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index eb7edaef53..3186b5d3c4 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -48,9 +48,18 @@ class FlashInferOneSidedDispatchLayout: def flashinfer_one_sided_dispatch_layout( - hidden_dim: int, quant_config: FusedMoEQuantConfig + hidden_dim: int, + quant_config: FusedMoEQuantConfig, + input_dtype: torch.dtype | None = None, ) -> FlashInferOneSidedDispatchLayout: """Return the one-sided activation payload layout.""" + if input_dtype is not None: + if input_dtype not in (torch.float16, torch.bfloat16): + raise ValueError( + "flashinfer_nvlink_one_sided unpacked inputs must be float16 " + f"or bfloat16, got {input_dtype}" + ) + return FlashInferOneSidedDispatchLayout(hidden_dim * input_dtype.itemsize, 0) if quant_config.quant_dtype is None: return FlashInferOneSidedDispatchLayout(hidden_dim * 2, 0) if quant_config.quant_dtype == "nvfp4": @@ -174,6 +183,7 @@ def maybe_make_prepare_finalize( allow_new_interface: bool = False, use_monolithic: bool = False, all2all_manager: Any | None = None, + input_dtype: torch.dtype | None = None, ) -> FusedMoEPrepareAndFinalize | None: if not moe.moe_parallel_config.use_all2all_kernels: if not allow_new_interface: @@ -367,7 +377,7 @@ def maybe_make_prepare_finalize( get_current_vllm_config().scheduler_config.max_num_batched_tokens ) dispatch_layout = flashinfer_one_sided_dispatch_layout( - moe.hidden_dim, quant_config + moe.hidden_dim, quant_config, input_dtype=input_dtype ) prepare_finalize = FlashInferNVLinkOneSidedPrepareAndFinalize( max_num_tokens=max_num_tokens, diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 0c82d28096..9221e93824 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import IntEnum from typing import Union @@ -1290,13 +1290,12 @@ class FusedMoEConfig: # Only honored on the non-reduced (late-AR) TP path. Default False. skip_final_all_reduce: bool = False - # When True, experts that can stop after GEMM2 are allowed to hand back an - # UnfinalizedMoEOutput instead of finalized states, leaving the top-k - # reduction to fuse into the consumer. Set by layers that have such a - # consumer; read through `use_deferred_moe_finalize`, which applies the - # guards. Kernels without the capability ignore it. Default False. - defer_moe_finalize: bool = False - # Optional consumer capacity for deferred finalize. Negative means unbounded. + # Requested through `defer_moe_finalize()`, read through + # `should_defer_moe_finalize()`. + _defer_moe_finalize: bool = field(default=False, init=False) + # Most tokens a deferred call covers: the consumer's capacity, lowered by + # experts that would split a larger call across kernel launches. Negative + # means unbounded. defer_moe_finalize_max_num_tokens: int = -1 # SwiGLU clamp limit. When set, backends that do not implement the clamp @@ -1422,21 +1421,57 @@ class FusedMoEConfig: def use_deferred_moe_finalize(self) -> bool: """Whether experts may return an unfinalized output on this deployment. - Evaluated on read rather than in ``__post_init__`` because - ``defer_moe_finalize`` is set after construction, like - ``skip_final_all_reduce``. + Evaluated on read rather than in ``__post_init__`` because deferral is + requested after construction, like ``skip_final_all_reduce``. """ # The consumer fuses a TP all-reduce. Other parallel modes require a - # combine or reduce-scatter after the experts and cannot defer it. + # combine or reduce-scatter after the experts and cannot defer it, and + # the consumer has no way to strip hidden-dim padding from GEMM2 rows. return ( - self.defer_moe_finalize + self._defer_moe_finalize and self.tp_size > 1 and self.dp_size == 1 and self.ep_size == 1 and self.pcp_size == 1 and not self.is_sequence_parallel + and self.hidden_dim == self.hidden_dim_unpadded ) + def defer_moe_finalize(self, max_num_tokens: int = -1) -> None: + """Ask the experts to leave the top-k reduction to the layer's consumer. + + A layer whose consumer can fuse the top-k reduction (e.g. into its TP + all-reduce) calls this once while the model is built, and only when its + experts are TRTLLM-Gen ones that can stop after GEMM2 (the + ``do_finalize=False`` path). Other experts ignore the request and + always finalize, which ``should_defer_moe_finalize`` can't see, so the + layer checks the quant method's ``experts_cls`` first, as Kimi-K3 does. + From then on: + + - The experts return an ``UnfinalizedMoEOutput`` instead of finalized + states for every call ``should_defer_moe_finalize`` accepts. + - ``should_defer_moe_finalize(num_tokens)`` is the answer for a call: + it requires a TP-only deployment without hidden-dim padding + (``use_deferred_moe_finalize``), a non-empty call and at most + ``defer_moe_finalize_max_num_tokens`` tokens. The cap starts at + ``max_num_tokens`` and only ever goes down: experts that would split a + larger call across kernel launches lower it to their single-launch + size when they are built, since each launch permutes into its own + buffer. + - The model asks ``should_defer_moe_finalize`` before each call and takes + the matching path. A deferred call runs the runner's ``_forward_impl`` + directly, since the MoE custom op returns tensors only, and the + consumer then owns the top-k reduction, the shared-expert add and the + all-reduce. + + Args: + max_num_tokens: Most tokens per call the consumer can take in + deferred form. Negative means no limit of its own. + + """ + self._defer_moe_finalize = True + self.limit_deferred_moe_finalize(max_num_tokens) + def should_defer_moe_finalize(self, num_tokens: int) -> bool: """Return whether this invocation may defer the top-k reduction.""" max_num_tokens = self.defer_moe_finalize_max_num_tokens @@ -1446,6 +1481,15 @@ class FusedMoEConfig: and (max_num_tokens < 0 or num_tokens <= max_num_tokens) ) + def limit_deferred_moe_finalize(self, max_num_tokens: int) -> None: + """Finalize calls above ``max_num_tokens`` even when deferring. + + Negative means no limit, and leaves the current one in place. + """ + current = self.defer_moe_finalize_max_num_tokens + if max_num_tokens >= 0 and (current < 0 or max_num_tokens < current): + self.defer_moe_finalize_max_num_tokens = max_num_tokens + @property def use_deepep_ht_kernels(self): return self.moe_parallel_config.use_deepep_ht_kernels diff --git a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py index cd7da3e1e3..a03651c0a9 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py @@ -1326,7 +1326,9 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular): @staticmethod def _supports_current_device() -> bool: - return cutlass_group_gemm_supported() + return ( + current_platform.is_device_capability(90) and cutlass_group_gemm_supported() + ) @staticmethod def _supports_no_act_and_mul() -> bool: diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index 27fdcf8bb2..5bf5065858 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -1294,9 +1294,6 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts): if triton_kernels_version == "3.8": # Activation and topk-reduce kept unfused from the matmuls (3.8 has # no in-kernel scatter+reduce); combine via external index_add. - assert self._lora_context is None, ( - "tk38 unfused MoE path does not support LoRA yet" - ) act_out_dim = self.adjust_N_for_activation(w1.shape[2], activation) M, K = hidden_states.shape[-2:] a_ragged = routing_data.ragged @@ -1313,6 +1310,23 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts): quant_config.w1_precision, gammas=gammas if apply_router_weight_on_input else None, ) + + lora_context = self._lora_context + if lora_context is not None: + s_ids = e_ids = n_pad = t_map = None + s_ids, e_ids, n_pad, t_map = self.apply_w13_lora( + lora_context, + y=inter, + x=hidden_states, + topk_ids=global_topk_ids, + topk_weights=topk_weights, + expert_map=expert_map, + w1=w1, + w2=w2, + num_tokens=M, + top_k_num=topk, + ) + act_out = torch.empty( (routing_data.n_valid, act_out_dim), dtype=hidden_states.dtype, @@ -1330,6 +1344,32 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts): quant_config.w2_precision, gammas=None if apply_router_weight_on_input else gammas, ) + + if lora_context is not None: + # w2 LoRA works in (num_tokens, topk, K) token-topk order. Scatter + # the ragged rows there via the same expert-sort permutation + # (flat idx = token*topk + slot = argsort(flat_e)), apply, then + # gather the LoRA-updated rows back to ragged order for the combine. + flat_e = topk_ids.reshape(-1).to(torch.int64).clamp(min=0) + order = torch.argsort(flat_e, stable=True) + down_tt = down.new_zeros((M * topk, K)) + down_tt[order] = down + self.apply_w2_lora( + lora_context, + y=down_tt.view(M, topk, K), + x=act_out, + topk_weights=topk_weights, + sorted_token_ids_lora=s_ids, + expert_ids_lora=e_ids, + num_tokens_post_padded_lora=n_pad, + token_lora_mapping=t_map, + num_tokens=M, + w1=w1, + w2=w2, + top_k_num=topk, + ) + down = down_tt[order] + acc = torch.zeros((M, K), dtype=torch.float32, device=hidden_states.device) acc.index_add_(0, gather_tok.to(torch.int64), down.to(torch.float32)) output.view(M, K).copy_(acc.to(output.dtype)) diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 62df395f69..4f2f760ad3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -385,10 +385,11 @@ def rocm_aiter_fused_experts( gate_mode = "" if activation == MoEActivation.SITU: - # SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4 - # fp4 activations with separated gate/up weights (AITER #4463); - # default a16w4 SiTU also stays separated. - gate_mode = GateMode.SEPARATED.value + gate_mode = ( + GateMode.INTERLEAVE.value + if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() + else GateMode.SEPARATED.value + ) elif quant_config.use_mxfp4_w4a16: gate_mode = GateMode.INTERLEAVE.value elif activation_interleave is not None: diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 1e21a4fe2c..ee3b9b3f4b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -337,6 +337,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) + # Under EP, drop the top-k slots routed to remote experts at alignment + # time instead of launching `off_experts == -1` GEMM blocks that only + # write zeros; the pad-aware `moe_sum` below skips the same slots. + # Their rows in the workspace caches are then never written, so every + # consumer of those caches must be row-local: a dynamic per-tensor + # scale for the second GEMM's activation would take its amax over the + # untouched rows, and LoRA aligns the full `topk_ids` itself. + a2_scale_is_global = ( + self.quant_dtype is not None + and a2_scale is None + and self.block_shape is None + and (not self.per_act_token_quant or self.quantization_emulation) + ) + skip_invalid = ( + expert_map is not None + and self._lora_context is None + and not a2_scale_is_global + ) + # Include fused shared-expert rows while preserving EP remapping. num_align_experts = w1.shape[0] if expert_map is None else global_num_experts sorted_token_ids, expert_ids, num_tokens_post_padded = ( @@ -350,6 +369,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): use_int8_w8a16=self.quant_config.use_int8_w8a16, use_int4_w4a16=self.quant_config.use_int4_w4a16, block_shape=self.block_shape, + ignore_invalid_experts=skip_invalid, ) ) @@ -585,10 +605,21 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) # separate function is required for MoE + LoRA - self.moe_sum(intermediate_cache3, output) + self.moe_sum(intermediate_cache3, output, topk_ids, expert_map) - def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None: - ops.moe_sum(input, output) + def moe_sum( + self, + input: torch.Tensor, + output: torch.Tensor, + topk_ids: torch.Tensor | None = None, + expert_map: torch.Tensor | None = None, + ) -> None: + if expert_map is not None: + # Skip the slots whose expert is not on this rank: the rows the + # alignment dropped, or the zeros the `-1` blocks wrote. + ops.moe_sum(input, output, topk_ids, expert_map) + else: + ops.moe_sum(input, output) class TritonWNA16Experts(TritonExperts): diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py index e860597bcb..fb52445d84 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py @@ -12,6 +12,10 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, RoutingMethodType, ) +from vllm.model_executor.layers.fused_moe.moe_output import ( + UnfinalizedMoEOutput, + convert_flashinfer_moe_output, +) from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) @@ -273,7 +277,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular): workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, apply_router_weight_on_input: bool, - ): + ) -> UnfinalizedMoEOutput | None: import flashinfer from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout @@ -297,7 +301,9 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular): weight_layout = WeightLayout.BlockMajorK hidden_states_scale = prepare_deepseek_fp8_x_sf(hidden_states, a1q_scale) - flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe( + num_tokens = hidden_states.shape[0] + defer = self.moe_config.should_defer_moe_finalize(num_tokens) + flashinfer_output = flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe( topk_ids=(topk_ids, topk_weights), routing_bias=None, hidden_states=hidden_states, @@ -321,9 +327,20 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular): use_shuffled_weight=use_shuffled_weight, weight_layout=weight_layout, fp8_quantization_type=fp8_quant_type, - output=output, + do_finalize=not defer, + output=None if defer else output, tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), ) + if not defer: + return None + routed_output = convert_flashinfer_moe_output( + flashinfer_output, + do_finalize=False, + num_tokens=num_tokens, + top_k=topk_ids.size(1), + ) + assert isinstance(routed_output, UnfinalizedMoEOutput) + return routed_output class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic): diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py index 1c20bc1cbc..e1557a8f0d 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py @@ -289,6 +289,18 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula Moved from trtllm_moe.py. """ + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + **kwargs, + ): + super().__init__(moe_config, quant_config, **kwargs) + # Each launch permutes into its own buffer, so a chunked call finalizes. + moe_config.limit_deferred_moe_finalize( + self._max_supported_tokens(self.topk, moe_config.num_experts) + ) + @staticmethod def _supports_parallel_config( moe_parallel_config: FusedMoEParallelConfig, @@ -343,7 +355,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula def _invoke_kernel( self, - output: torch.Tensor, + output: torch.Tensor | None, x_quant: torch.Tensor, x_scale: torch.Tensor | None, topk_ids: torch.Tensor, @@ -355,10 +367,11 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula local_num_experts: int, local_expert_offset: int, topk: int, - ) -> None: + ) -> UnfinalizedMoEOutput | None: + """Finalize into ``output``, or stop after GEMM2 when it is None.""" from flashinfer import trtllm_fp4_block_scale_routed_moe - trtllm_fp4_block_scale_routed_moe( + flashinfer_output = trtllm_fp4_block_scale_routed_moe( topk_ids=(topk_ids, topk_weights), routing_bias=None, hidden_states=x_quant, @@ -386,12 +399,22 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula # Modular kernel receives pre-routed tokens, so routing is already # done. Use Renormalize as a safe default the TRTLLM kernel supports. routing_method_type=RoutingMethodType.Renormalize, - do_finalize=True, + do_finalize=output is not None, enable_pdl=True, activation_type=self._flashinfer_activation_type(activation), output=output, tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), ) + if output is not None: + return None + routed_output = convert_flashinfer_moe_output( + flashinfer_output, + do_finalize=False, + num_tokens=x_quant.shape[0], + top_k=topk, + ) + assert isinstance(routed_output, UnfinalizedMoEOutput) + return routed_output def apply( self, @@ -410,7 +433,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, apply_router_weight_on_input: bool, - ): + ) -> UnfinalizedMoEOutput | None: topk_ids = topk_ids.to(dtype=torch.int32) topk = topk_ids.size(-1) @@ -428,13 +451,17 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula assert self.w1_scale is not None assert self.w2_scale is not None - # Chunk tokens so the batched-GEMM grid stays within CUDA limits. + # Chunk tokens so the batched-GEMM grid stays within CUDA limits. Each + # launch permutes into its own buffer, so only a run that fits in one + # launch can leave the top-k reduction to a deferring consumer. M = x_quant.size(0) chunk_size = self._max_supported_tokens(topk, global_num_experts) + defer = chunk_size >= M and self.moe_config.should_defer_moe_finalize(M) + unfinalized: UnfinalizedMoEOutput | None = None for start in range(0, M, chunk_size): end = min(start + chunk_size, M) - self._invoke_kernel( - output[start:end], + unfinalized = self._invoke_kernel( + None if defer else output[start:end], x_quant[start:end], None if x_scale is None else x_scale[start:end], topk_ids[start:end], @@ -448,4 +475,4 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula topk, ) - return output + return unfinalized diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index cea985830d..f9cca18a9c 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kNvfp4Dynamic, + kNvfp4DynamicToken, kNvfp4Static, ) from vllm.model_executor.utils import is_weights_pre_processed @@ -124,6 +125,10 @@ class TrtLlmNvFp4ExpertsBase: self.gemm1_beta = _per_expert(situ_linear_beta) self.gemm1_clamp_limit = None + # Unfolded values: process_weights_after_loading reruns on weight reload. + self._gemm1_clamp_limit_unfolded = self.gemm1_clamp_limit + self._gemm1_beta_unfolded = self.gemm1_beta + logger.debug_once( "activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s", moe_config.activation, @@ -163,15 +168,15 @@ class TrtLlmNvFp4ExpertsBase: # Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar) # division so the TRTLLM kernel receives the raw-GEMM-space clamp - # directly. g1_alphas is set once here in process_weights_after_loading - # (via the in-place mul above) and never changes again, so this is a - # static, per-expert constant. Register on the layer so EPLB - # rearranges it alongside the other expert tensors. + # directly. Fold from the unfolded value, since g1_alphas changes when + # weights are reloaded. Register on the layer so EPLB rearranges it + # alongside the other expert tensors. # SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the # raw GEMM1 accumulator, so they are registered as-is without the # g1_alphas fold used by the SwiGLU-OAI clamp/beta below. - if self.gemm1_clamp_limit is not None and not self.is_situ: - gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas + clamp_limit = self._gemm1_clamp_limit_unfolded + if clamp_limit is not None and not self.is_situ: + gemm1_clamp_limit = clamp_limit / self.quant_config.g1_alphas layer.register_parameter( "gemm1_clamp_limit", torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False), @@ -182,12 +187,9 @@ class TrtLlmNvFp4ExpertsBase: # clamp limit. alpha is applied to the dequantized gate, so it stays # raw. Register both on the layer so EPLB rearranges them with the # other per-expert tensors. - if self.gemm1_beta is not None: - gemm1_beta = ( - self.gemm1_beta - if self.is_situ - else self.gemm1_beta / self.quant_config.g1_alphas - ) + beta = self._gemm1_beta_unfolded + if beta is not None: + gemm1_beta = beta if self.is_situ else beta / self.quant_config.g1_alphas layer.register_parameter( "gemm1_beta", torch.nn.Parameter(gemm1_beta, requires_grad=False), @@ -222,10 +224,10 @@ class TrtLlmNvFp4ExpertsBase: activation_key: QuantKey | None, ) -> bool: """Supports Nvfp4 quantization.""" - SUPPORTED_W_A = [ - (kNvfp4Static, kNvfp4Dynamic), - ] - return (weight_key, activation_key) in SUPPORTED_W_A + return weight_key == kNvfp4Static and activation_key in ( + kNvfp4Dynamic, + kNvfp4DynamicToken, + ) @staticmethod def _supports_activation(activation: MoEActivation) -> bool: @@ -300,6 +302,16 @@ class TrtLlmNvFp4ExpertsBase: class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular): """Modular version of the implementation (just the experts).""" + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + per_token_activation: bool = False, + ): + super().__init__(moe_config, quant_config, per_token_activation) + # Each launch permutes into its own buffer, so a chunked call finalizes. + moe_config.limit_deferred_moe_finalize(self._get_chunk_size()) + @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: """The modular implementation supports all parallel configs.""" @@ -316,22 +328,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula expert_tokens_meta: mk.ExpertTokensMetadata | None, activation: MoEActivation, ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - if self.per_token_activation: - # Deferred input quant leaves K unpacked here, breaking the - # workspace assumptions below. Per-token NVFP4 is only supported on - # the monolithic (non-EP) path for now. - raise NotImplementedError( - "NVFP4 per-token activation is only supported on the monolithic " - "(non-EP) FlashInfer TRTLLM MoE path." - ) - # The workspaces for this implementation are managed by flashinfer. workspace1 = (0,) workspace2 = (0,) - # Hidden states are Nvfp4, packed into int8 dtype, so we - # need to multiply K by 2 to get the output shape right. - assert self.hidden_dim == K * 2 + # Per-token inputs are unpacked; otherwise each byte holds two FP4 values. + assert self.hidden_dim == (K if self.expects_unquantized_inputs else K * 2) output = (M, self.hidden_dim) return (workspace1, workspace2, output) @@ -341,7 +343,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula def _invoke_kernel( self, - output: torch.Tensor, + output: torch.Tensor | None, hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, @@ -349,8 +351,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula topk_ids: torch.Tensor, activation: MoEActivation, global_num_experts: int, - a1q_scale: torch.Tensor, - ): + a1q_scale: torch.Tensor | None, + ) -> UnfinalizedMoEOutput | None: + """Finalize into ``output``, or stop after GEMM2 when it is None.""" import flashinfer assert self.quant_config.w1_scale is not None @@ -367,7 +370,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula output1_scale_gate_scalar = self.quant_config.g1_alphas # Invoke kernel. - flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe( + flashinfer_output = flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe( topk_ids=(topk_ids, topk_weights), routing_bias=None, hidden_states=hidden_states, @@ -395,7 +398,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula local_num_experts=self.local_num_experts, routed_scaling_factor=None, routing_method_type=1, # not used - do_finalize=True, + do_finalize=output is not None, activation_type=activation_to_flashinfer_int(activation), per_token_scale=per_token_scale, output=output, @@ -403,6 +406,16 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula fi_moe_largest_bucket(self.moe_config), self._get_chunk_size() ), ) + if output is not None: + return None + routed_output = convert_flashinfer_moe_output( + flashinfer_output, + do_finalize=False, + num_tokens=hidden_states.shape[0], + top_k=self.topk, + ) + assert isinstance(routed_output, UnfinalizedMoEOutput) + return routed_output def apply( self, @@ -421,7 +434,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, apply_router_weight_on_input: bool, - ): + ) -> UnfinalizedMoEOutput | None: assert self._supports_activation(activation) # Per-token defers input quant to _invoke_kernel, so a1q_scale is None. assert a1q_scale is not None or self.per_token_activation @@ -433,8 +446,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula chunk_size = self._get_chunk_size() if chunk_size >= M: - self._invoke_kernel( - output, + # Each launch permutes into its own buffer, so only a run that fits + # in one launch can leave the top-k reduction to a deferring + # consumer. + defer = self.moe_config.should_defer_moe_finalize(M) + return self._invoke_kernel( + None if defer else output, hidden_states, w1, w2, @@ -458,6 +475,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula global_num_experts, None if a1q_scale is None else a1q_scale[start:end], ) + return None class TrtLlmNvFp4ExpertsMonolithic( diff --git a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py b/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py index 2533c3a895..98a2577967 100644 --- a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py @@ -7,11 +7,6 @@ import json import os import torch -from aiter.fused_moe import moe_sorting as aiter_moe_sorting -from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( - flydsl_a16w4_gemm1, - flydsl_a16w4_gemm2, -) from vllm.logger import init_logger from vllm.utils.platform_utils import get_device_name_as_file_name @@ -81,6 +76,10 @@ def moe_sorting( model_dim: int, block_m: int, ): + # Imported lazily so aiter's import-time sort-backend env vars are read + # when this is called, not whenever this module happens to be imported. + from aiter.fused_moe import moe_sorting as aiter_moe_sorting + topk_ids_i32 = topk_ids.to(torch.int32) topk_w_f32 = topk_weights.to(torch.float32) sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = ( @@ -185,6 +184,12 @@ def fused_flydsl_moe_impl( tile_n2: int | None = None, tile_k2: int | None = None, ) -> torch.Tensor: + # Imported lazily -- see the comment in moe_sorting() above. + from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( + flydsl_a16w4_gemm1, + flydsl_a16w4_gemm2, + ) + device = hidden_states.device tokens = hidden_states.shape[0] model_dim = hidden_states.shape[1] diff --git a/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py b/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py index c64aeadcc7..d83ed2c507 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py @@ -148,7 +148,7 @@ class FusedMoEMethodBase(QuantizeMethodBase): topk_ids: torch.Tensor, shared_experts: "SharedExperts | None", shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: """Apply the MoE operation using modular kernels. Args: @@ -160,7 +160,7 @@ class FusedMoEMethodBase(QuantizeMethodBase): shared_experts: The shared experts module, if any Returns: - Output tensor from routed experts. + Finalized routed states or a deferred-finalize output. """ raise NotImplementedError diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index fba16f67b0..f563c819e4 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -238,6 +238,15 @@ class FusedMoEPrepareAndFinalize(ABC): """ return False + def supports_deferred_moe_finalize(self) -> bool: + """Whether ``finalize`` can be skipped for a deferring consumer. + + An implementation opts in only if everything it does in ``finalize`` + -- the top-k reduction, and any combine or reduce-scatter -- is work + the consumer takes over. + """ + return False + # TODO: pass FusedMoEParallelConfig in as ctor parameter? class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize): @@ -404,9 +413,6 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize): described above for the monolithic case. """ - def supports_deferred_moe_finalize(self) -> bool: - return False - @abstractmethod def prepare( self, @@ -895,10 +901,14 @@ class FusedMoEExpertsModular(FusedMoEExperts): workspace2: torch.Tensor, expert_tokens_meta: ExpertTokensMetadata | None, apply_router_weight_on_input: bool, - ) -> None: + ) -> UnfinalizedMoEOutput | None: """This function computes the intermediate result of a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2. + Writes into `output` and returns None, unless the implementation stopped + after GEMM2 and left the top-k reduction to a fused consumer, in which + case `output` is untouched and the unfinalized result is returned. + Args: output: (torch.Tensor): The unweighted, unreduced output tensor. hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE @@ -1270,7 +1280,7 @@ class FusedMoEKernelModularImpl: apply_router_weight_on_input: bool, expert_tokens_meta: ExpertTokensMetadata | None, output_alias: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: _, M_full, N, K, top_k = self.fused_experts.moe_problem_size( a1q, w1, w2, topk_ids ) @@ -1318,7 +1328,7 @@ class FusedMoEKernelModularImpl: elif use_output_alias: fused_out = output_alias - self.fused_experts.apply( + unfinalized = self.fused_experts.apply( output=fused_out, hidden_states=a1q, w1=w1, @@ -1336,6 +1346,11 @@ class FusedMoEKernelModularImpl: apply_router_weight_on_input=apply_router_weight_on_input, ) + # Experts that stopped after GEMM2 wrote nothing into `fused_out`; the + # top-k reduction they left open belongs to whoever takes this. + if isinstance(unfinalized, UnfinalizedMoEOutput): + return unfinalized + return fused_out def _finalize( @@ -1426,7 +1441,7 @@ class FusedMoEKernelModularImpl: apply_router_weight_on_input: bool = False, shared_experts: SharedExperts | None = None, shared_experts_input: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: """This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism. @@ -1453,7 +1468,9 @@ class FusedMoEKernelModularImpl: hidden_states before latent projection. Returns: - torch.Tensor: The output tensor after applying the MoE layer. + torch.Tensor: The output tensor after applying the MoE layer, or + the unfinalized output when the experts left the top-k reduction to + a fused consumer. """ output = torch.empty_like(hidden_states) @@ -1498,6 +1515,17 @@ class FusedMoEKernelModularImpl: if lora_ctx is not None: lora_ctx.original_hidden_states = None + if isinstance(fused_out, UnfinalizedMoEOutput): + # Nothing below can run on an unfinalized output: finalize is the + # local top-k reduction (and, for DP/EP, the combine) that the + # consumer takes over. + if not self.prepare_finalize.supports_deferred_moe_finalize(): + raise RuntimeError( + f"{type(self.prepare_finalize).__name__} cannot pass through " + "a deferred MoE output." + ) + return fused_out + return self._finalize( output, fused_out, @@ -1653,10 +1681,7 @@ class FusedMoEKernel: return self.prepare_finalize.output_is_reduced() def supports_deferred_moe_finalize(self) -> bool: - return ( - isinstance(self.prepare_finalize, FusedMoEPrepareAndFinalizeMonolithic) - and self.prepare_finalize.supports_deferred_moe_finalize() - ) + return self.prepare_finalize.supports_deferred_moe_finalize() def apply_monolithic( self, @@ -1703,7 +1728,7 @@ class FusedMoEKernel: apply_router_weight_on_input: bool, shared_experts: SharedExperts | None = None, shared_experts_input: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert isinstance(self.impl, FusedMoEKernelModularImpl) return self.impl.apply( hidden_states=hidden_states, diff --git a/vllm/model_executor/layers/fused_moe/moe_output.py b/vllm/model_executor/layers/fused_moe/moe_output.py index 04081cf6d1..8f9662b442 100644 --- a/vllm/model_executor/layers/fused_moe/moe_output.py +++ b/vllm/model_executor/layers/fused_moe/moe_output.py @@ -9,7 +9,7 @@ import torch @dataclass class UnfinalizedMoEOutput: - """Unfinalized output of a monolithic MoE kernel. + """Unfinalized output of a MoE kernel. Kernels that can stop after GEMM2 (the TRTLLM-Gen ``do_finalize=False`` path) hand back their permuted, unweighted output plus the routing weights and the @@ -24,10 +24,11 @@ class UnfinalizedMoEOutput: # [num_permuted_rows, hidden_dim] permuted, unweighted GEMM2 output. gemm2_permuted: torch.Tensor - # [num_tokens, top_k] routing weights, in the activation dtype -- consumers - # index them as such, and a buffer typed wider than its contents reads as - # garbage. These already carry routed_scaling_factor for routing methods that - # fold it in. + # [num_tokens, top_k] routing weights, in the activation dtype when the + # kernel routed itself and in the router's dtype for modular experts -- + # consumers index them as such, and a buffer typed wider than its contents + # reads as garbage. These already carry routed_scaling_factor for routing + # methods that fold it in. expert_weights: torch.Tensor # [num_tokens, top_k] int32 permute map; -1 = expert not local to this rank. expanded_idx_to_permuted_idx: torch.Tensor diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 5450f73f85..56ba3981b7 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( fp4_dtype = torch.float4_e2m1fn_x2 e8m0_dtype = torch.float8_e8m0fnu - # SiTUv2 flydsl uses separated gate/up layout (a4w4). - guinterleave = False + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() w13 = rocm_aiter_ops.shuffle_weight_a16w4( w13_weight.data.view(fp4_dtype), 16, guinterleave ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 0d794584d7..0fecb65e4b 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -605,6 +605,7 @@ def make_nvfp4_moe_kernel( routing_tables=routing_tables, allow_new_interface=True, use_monolithic=issubclass(experts_cls, mk.FusedMoEExpertsMonolithic), + input_dtype=moe_config.in_dtype if per_token_activation else None, ) assert prepare_finalize is not None diff --git a/vllm/model_executor/layers/fused_moe/oracle/w4a8.py b/vllm/model_executor/layers/fused_moe/oracle/w4a8.py index d8e026ba31..5df5b099af 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/w4a8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/w4a8.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Oracle for W4A8 (INT4, FP8) MoE from enum import Enum from typing import TYPE_CHECKING @@ -22,26 +23,37 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) if TYPE_CHECKING: - from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( - CutlassExpertsW4A8Fp8, - ) + from vllm.model_executor.layers.fused_moe import RoutedExperts logger = init_logger(__name__) class W4A8MoeBackend(Enum): CUTLASS = "CUTLASS" + HUMMING = "HUMMING" def backend_to_kernel_cls( backend: W4A8MoeBackend, -) -> list[type["CutlassExpertsW4A8Fp8"]]: +) -> list[type[mk.FusedMoEExperts]]: if backend == W4A8MoeBackend.CUTLASS: from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsW4A8Fp8, ) return [CutlassExpertsW4A8Fp8] + elif backend == W4A8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] else: raise ValueError(f"Unknown W4A8 MoE backend: {backend.value}") @@ -50,8 +62,19 @@ def select_w4a8_moe_backend( config: FusedMoEConfig, weight_key: QuantKey | None = kInt4Static, activation_key: QuantKey | None = kFp8DynamicTokenSym, -) -> tuple[W4A8MoeBackend, type["CutlassExpertsW4A8Fp8"]]: - backend = W4A8MoeBackend.CUTLASS +) -> tuple[W4A8MoeBackend, type[mk.FusedMoEExperts]]: + backends = [W4A8MoeBackend.CUTLASS, W4A8MoeBackend.HUMMING] + if config.moe_backend != "auto": + backend_map = { + "cutlass": W4A8MoeBackend.CUTLASS, + "humming": W4A8MoeBackend.HUMMING, + } + if config.moe_backend not in backend_map: + raise ValueError( + f"moe_backend='{config.moe_backend}' is not supported for W4A8 MoE. " + f"Expected one of {list(backend_map)}." + ) + backends = [backend_map[config.moe_backend]] activation_format = ( mk.FusedMoEActivationFormat.BatchedExperts @@ -59,29 +82,33 @@ def select_w4a8_moe_backend( else mk.FusedMoEActivationFormat.Standard ) - last_reason: str | None = None - for kernel_cls in backend_to_kernel_cls(backend): - supported, reason = kernel_cls.is_supported_config( - kernel_cls, - config, - weight_key, - activation_key, - activation_format, - ) - if supported: - logger.info_once("Using %s W4A8 MoE backend.", backend.value) - return backend, kernel_cls - last_reason = reason + reasons = [] + for backend in backends: + for kernel_cls in backend_to_kernel_cls(backend): + supported, reason = kernel_cls.is_supported_config( + kernel_cls, + config, + weight_key, + activation_key, + activation_format, + ) + if supported: + logger.info_once("Using %s W4A8 MoE backend.", backend.value) + return backend, kernel_cls + reasons.append(f"{kernel_cls.__name__}: {reason}") raise NotImplementedError( - f"W4A8 MoE backend {backend.value} does not support the " - f"deployment configuration: {last_reason}." + "No W4A8 MoE backend supports the deployment configuration: " + + "; ".join(reasons) ) def convert_to_w4a8_moe_kernel_format( - w13_weight_packed: torch.Tensor, - w2_weight_packed: torch.Tensor, + backend: W4A8MoeBackend, + layer: "RoutedExperts", + group_size: int, + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, w13_weight_scale: torch.Tensor, w2_weight_scale: torch.Tensor, ) -> tuple[ @@ -89,11 +116,40 @@ def convert_to_w4a8_moe_kernel_format( torch.Tensor, torch.Tensor, torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, ]: + if backend == W4A8MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming import ( + convert_to_humming_moe_kernel_format, + ) + from vllm.utils.humming import ( + HummingInputSchema, + HummingWeightSchema, + dtypes, + ) + + convert_to_humming_moe_kernel_format( + layer, + weight_schema=HummingWeightSchema( + b_dtype=dtypes.uint4, weight_scale_group_size=group_size + ), + input_schema=HummingInputSchema(a_dtype=dtypes.float8e4m3), + allow_input_schema_fallback=False, + ) + return ( + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale, + layer.w2_weight_scale, + None, + None, + None, + None, + ) + from vllm import _custom_ops as ops from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 from vllm.model_executor.layers.quantization.utils.quant_utils import ( @@ -104,18 +160,18 @@ def convert_to_w4a8_moe_kernel_format( quant_fp8 = QuantFP8(static=False, group_shape=GroupShape.PER_TOKEN) - convert_packed_uint4b8_to_signed_int4_inplace(w13_weight_packed) + convert_packed_uint4b8_to_signed_int4_inplace(w13_weight) # Mirror the sync in CutlassW4A8LinearKernel; required for TP>1 correctness. torch.accelerator.synchronize() w13_weight_shuffled, b_strides1 = ops.cutlass_encode_and_reorder_int4b_grouped( - w13_weight_packed + w13_weight ) - convert_packed_uint4b8_to_signed_int4_inplace(w2_weight_packed) + convert_packed_uint4b8_to_signed_int4_inplace(w2_weight) # Mirror the sync in CutlassW4A8LinearKernel; required for TP>1 correctness. torch.accelerator.synchronize() w2_weight_shuffled, b_strides2 = ops.cutlass_encode_and_reorder_int4b_grouped( - w2_weight_packed + w2_weight ) w13_weight_scale, w13_weight_chan_scale = convert_bf16_scales_to_fp8( @@ -147,11 +203,23 @@ def convert_to_w4a8_moe_kernel_format( def make_w4a8_moe_quant_config( + backend: W4A8MoeBackend, w1_scale: torch.Tensor, w2_scale: torch.Tensor, - g1_alphas: torch.Tensor, - g2_alphas: torch.Tensor, + g1_alphas: torch.Tensor | None, + g2_alphas: torch.Tensor | None, + layer: torch.nn.Module | None = None, ) -> FusedMoEQuantConfig: + if backend == W4A8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.quantization.utils.humming import ( + get_humming_moe_quant_config, + ) + + assert isinstance(layer, RoutedExperts) + return get_humming_moe_quant_config(layer) + + assert g1_alphas is not None and g2_alphas is not None return int4_w4afp8_moe_quant_config( w1_scale=w1_scale, w2_scale=w2_scale, @@ -163,14 +231,30 @@ def make_w4a8_moe_quant_config( def make_w4a8_moe_kernel( + backend: W4A8MoeBackend, moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, - experts_cls: type["CutlassExpertsW4A8Fp8"], - b_strides1: torch.Tensor, - b_strides2: torch.Tensor, + experts_cls: type[mk.FusedMoEExperts], + b_strides1: torch.Tensor | None, + b_strides2: torch.Tensor | None, group_size: int, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, ) -> mk.FusedMoEKernel: + if backend == W4A8MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming import ( + make_humming_moe_kernel, + ) + + return make_humming_moe_kernel( + moe_quant_config, moe_config, experts_cls, routing_tables=routing_tables + ) + + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsW4A8Fp8, + ) + + assert issubclass(experts_cls, CutlassExpertsW4A8Fp8) + assert b_strides1 is not None and b_strides2 is not None prepare_finalize = maybe_make_prepare_finalize( moe=moe_config, quant_config=moe_quant_config, diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/no_dp_ep.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/no_dp_ep.py index d13bc20673..660807fea3 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/no_dp_ep.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/no_dp_ep.py @@ -38,6 +38,11 @@ def _quantize_input( class MoEPrepareAndFinalizeNoDPEPModular(mk.FusedMoEPrepareAndFinalizeModular): + def supports_deferred_moe_finalize(self) -> bool: + # finalize() here is the top-k weight-and-reduce and nothing else, + # which is exactly the work a deferring consumer takes over. + return True + @property def activation_format(self) -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index e3a287ddc2..1d411ecc72 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -36,6 +36,28 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def _index_expert_mapping( + mapping: list[tuple[str, str, int, str]], +) -> dict[str, list[tuple[str, str, int, str]]]: + """Index by logical checkpoint ID, preserving entry order and EPLB replicas. + + An empty index keeps the full scan for unsupported mapping names. + """ + mapping_by_expert: dict[str, list[tuple[str, str, int, str]]] = {} + for entry in mapping: + prefix, _, suffix = entry[1].partition(".") + expert_key, sep, _ = suffix.partition(".") + if ( + prefix != "experts" + or not expert_key + or (expert_key.isdecimal() and not sep) + ): + return {} + if expert_key.isdecimal(): + mapping_by_expert.setdefault(expert_key, []).append(entry) + return mapping_by_expert + + class FusedMoeWeightScaleSupported(Enum): TENSOR = "tensor" CHANNEL = "channel" @@ -889,14 +911,22 @@ class RoutedExperts(PluggableLayer): self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[str]: expert_mapping = self.get_expert_mapping(include_fused=True) + mapping_by_expert = _index_expert_mapping(expert_mapping) + for expert_name, loaded_weight in weights: qual_name = f"{self.layer_name}.{expert_name}" - # Fused expert weights can be identified by their 3D tensors is_fused = loaded_weight.dim() == 3 + # Fused tensors and ambiguous names keep the full mapping. + candidates = expert_mapping + if not is_fused and qual_name.count("experts.") == 1: + expert_key = qual_name.partition("experts.")[2].partition(".")[0] + candidates = mapping_by_expert.get(expert_key, expert_mapping) + matched = False - for param_name, weight_name, expert_id, shard_id in expert_mapping: + for param_name, weight_name, expert_id, shard_id in candidates: if weight_name not in qual_name: if matched and is_fused: + # Fused tensors use the first contiguous run of matches. break continue matched = True @@ -1224,7 +1254,7 @@ class RoutedExperts(PluggableLayer): topk_ids: torch.Tensor, shared_experts: "SharedExperts | None" = None, shared_experts_input: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: """Execute routed experts using the quantization method's apply function. This is called by the runner after router selection (for modular kernels) @@ -1239,7 +1269,7 @@ class RoutedExperts(PluggableLayer): shared_experts_input: Input for shared experts (if any) Returns: - Output tensor from routed experts. + Finalized routed states or a deferred-finalize output. """ assert not self.quant_method.is_monolithic diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index f546dfd98c..f91e9c449e 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -5,7 +5,11 @@ from torch.nn.parameter import Parameter import vllm._custom_ops as ops from vllm.model_executor.custom_op import PluggableLayer -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ( + ReplicatedLinear, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op @@ -25,12 +29,18 @@ class GateLinear(ReplicatedLinear): The ``out_dtype`` attribute is mutable and can be set after init (e.g. when the required dtype depends on the expert quantization method which is only known later). + + A ``quant_config`` that actually quantizes the gate disables every + specialized tier, leaving plain ``ReplicatedLinear`` behavior. """ # (hidden_size, num_experts) pairs with an instantiated fp32 kernel: # (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3 FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)} FP32_MAX_TOKENS = 32 + # Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses + # whose router shape stays ahead further up the range may raise it. + LL_BF16_MAX_TOKENS = 16 def __init__( self, @@ -40,7 +50,10 @@ class GateLinear(ReplicatedLinear): out_dtype: torch.dtype | None = None, params_dtype: torch.dtype | None = None, force_fp32_compute: bool = False, + skip_bias_add: bool = False, + quant_config: QuantizationConfig | None = None, prefix: str = "", + return_bias: bool = True, ): is_hopper = current_platform.is_device_capability((9, 0)) is_blackwell = current_platform.is_device_capability_family(100) @@ -72,17 +85,25 @@ class GateLinear(ReplicatedLinear): input_size, output_size, bias=bias, + skip_bias_add=skip_bias_add, params_dtype=params_dtype, - quant_config=None, + quant_config=quant_config, prefix=prefix, + return_bias=return_bias, ) self.out_dtype = out_dtype + # A quantized gate exposes no plain ``weight``, so every specialized + # tier below is disabled and forward falls back to ReplicatedLinear. + self.is_unquantized = isinstance(self.quant_method, UnquantizedLinearMethod) + can_use_specialized_kernels &= self.is_unquantized + self.allow_specialized_router_gemm = can_use_specialized_kernels # fp32 specialized kernel eligibility (exact dims, fp32 weight) self.allow_fp32_router_gemm = ( - not bias + self.is_unquantized + and not bias and self.weight.dtype == torch.float32 and ( ( @@ -94,7 +115,8 @@ class GateLinear(ReplicatedLinear): ) ) self.allow_bf16x3_router_gemm = ( - not bias + self.is_unquantized + and not bias and self.weight.dtype == torch.float32 and current_platform.is_cuda() and is_blackwell @@ -108,7 +130,7 @@ class GateLinear(ReplicatedLinear): # applies on any CUDA-alike device (no bias, since torch.mm has no bias # term). The specialized-kernel gate above excludes family-120 Blackwell # (GB10 / DGX Spark), which this tier still covers. See #49921. - self._router_gemm_no_bias = not bias + self._router_gemm_no_bias = self.is_unquantized and not bias self._router_gemm_cublas_capable = ( current_platform.is_cuda() or current_platform.is_rocm() ) and self._router_gemm_no_bias @@ -162,17 +184,26 @@ class GateLinear(ReplicatedLinear): and is_available() ) + def _return( + self, output: torch.Tensor + ) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]: + return output if not self.return_bias else (output, None) + def forward( self, x: torch.Tensor ) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]: # Tier 1: cuteDSL ll_bf16_gemm (SM90+, any dims) - if self.allow_ll_bf16_gemm and x.shape[0] <= 16 and x.dtype == torch.bfloat16: + if ( + self.allow_ll_bf16_gemm + and x.shape[0] <= self.LL_BF16_MAX_TOKENS + and x.dtype == torch.bfloat16 + ): from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ( ll_bf16_gemm, ) output = ll_bf16_gemm(x, self.weight) - return output, None + return self._return(output) # Tier 2: fp32 specialized kernel (model-specific shapes, M<=32) # Dispatch is wrapped in a custom op so that torch.compile/CUDA-graph @@ -184,7 +215,7 @@ class GateLinear(ReplicatedLinear): output = torch.ops.vllm.fp32_router_gemm_dispatch( x, self.weight, self.allow_bf16x3_router_gemm ) - return output, None + return self._return(output) # Tier 3: bf16x3 CuteDSL kernel for fp32 router weights if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: @@ -193,20 +224,29 @@ class GateLinear(ReplicatedLinear): ) output = bf16x3_router_gemm(x, self.weight) - return output, None + return self._return(output) # Tier 4: cuBLAS bf16→fp32 if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16: output = torch.mm(x, self.weight.T, out_dtype=torch.float32) - return output, None + return self._return(output) # Tier 5: F.linear (ReplicatedLinear) - if self.out_dtype is not None and x.dtype != self.weight.dtype: + if ( + self.out_dtype is not None + and self.is_unquantized + and x.dtype != self.weight.dtype + ): x = x.to(self.weight.dtype) - output, output_bias = super().forward(x) + base_output = super().forward(x) + output_bias: Parameter | None = None + if isinstance(base_output, tuple): + output, output_bias = base_output + else: + output = base_output if self.out_dtype is not None and output.dtype != self.out_dtype: output = output.to(self.out_dtype) - return output, output_bias + return output if not self.return_bias else (output, output_bias) _FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 870d04c0c8..08279d1c89 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -657,7 +657,11 @@ class MoERunner(MoERunnerInterface): """ ctx = get_forward_context() return ( - ctx.dp_metadata.sp_local_sizes(self.moe_config.sp_size) + ctx.dp_metadata.sp_local_sizes( + self.moe_config.sp_size, + pcp_size=self.moe_config.pcp_size, + use_ep=self.moe_config.use_ep, + ) if ctx.dp_metadata else nullcontext() ) diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index 34e116dfbc..a98e181cae 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -453,35 +453,6 @@ def normalize_batched_scales_shape( return scales -@triton.jit -def _pack_topk_ids_weights_kernel( - topk_ids_ptr, - topk_weights_ptr, - output_ptr, - n_elements, - BLOCK_SIZE: tl.constexpr, - USE_GDC: tl.constexpr, - launch_pdl: tl.constexpr, # triton metadata -): - pid = tl.program_id(axis=0) - offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < n_elements - if USE_GDC: - tl.extra.cuda.gdc_launch_dependents() - tl.extra.cuda.gdc_wait() - expert_id = tl.load(topk_ids_ptr + offsets, mask=mask, other=0).to(tl.int32) - expert_id_shifted = expert_id << 16 - - weight = tl.load(topk_weights_ptr + offsets, mask=mask, other=0.0) - weight_bf16 = weight.to(tl.bfloat16) - weight_int16 = weight_bf16.to(tl.int16, bitcast=True) - - weight_int32 = weight_int16.to(tl.int32) & 0xFFFF - - packed = expert_id_shifted | weight_int32 - tl.store(output_ptr + offsets, packed, mask=mask) - - def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int: """Estimate FlashInfer's MoE autotuning maximum token count. diff --git a/vllm/model_executor/layers/fusion/mm_input_norm.py b/vllm/model_executor/layers/fusion/mm_input_norm.py new file mode 100644 index 0000000000..a21828033d --- /dev/null +++ b/vllm/model_executor/layers/fusion/mm_input_norm.py @@ -0,0 +1,361 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Fused Normalisation on the Device. + +Equivalent to:: + + output = (input * rescale_factor - image_mean) / image_std + +This is implemented as a single per-channel affine transform:: + + output = input * weight[c] + bias[c] + +where:: + + weight = rescale_factor / image_std + bias = -image_mean / image_std +""" + +import math +from typing import Any, NamedTuple + +import torch +from torch import nn + +from vllm.config import ModelConfig +from vllm.model_executor.custom_op import CustomOp +from vllm.transformers_utils.processor import cached_get_processor, get_processor_config +from vllm.triton_utils import tl, triton + + +@triton.jit +def _fused_mm_input_norm_kernel( + x_ptr, + y_ptr, + w_ptr, + b_ptr, + numel, + L, + C: tl.constexpr, + BLOCK: tl.constexpr, +): + # 1D grid over the flattened (N, C, L) tensor. Each program processes + # BLOCK contiguous elements; the channel index is recovered from the + # flat offset via `(offs // L) % C`. int32 offsets: int64 division is + # ~2x slower here (measured on GB200); the wrapper asserts + # numel < 2**31 so the flat index never overflows. + pid = tl.program_id(0) + offs = pid * BLOCK + tl.arange(0, BLOCK) + mask = offs < numel + c = (offs // L) % C + + x = tl.load( + x_ptr + offs, + mask=mask, + other=0, + eviction_policy="evict_first", + ).to(tl.float32) + w = tl.load(w_ptr + c, mask=mask, other=0).to(tl.float32) + b = tl.load(b_ptr + c, mask=mask, other=0).to(tl.float32) + tl.store( + y_ptr + offs, + x * w + b, + mask=mask, + eviction_policy="evict_first", + ) + + +def fused_mm_input_norm_triton( + inputs: torch.Tensor, + outputs: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, +) -> torch.Tensor: + """Fused per-channel affine transform for normalisation. + + Args: + inputs: Input tensor, shape ``(N, C, L)``. A contiguous copy is + materialized internally if needed. + outputs: Output tensor, contiguous and shaped exactly ``(N, C, L)``. + weight: Per-channel scale, shape ``(C,)``. + bias: Per-channel shift, shape ``(C,)``. + + Returns: + ``outputs``, for chaining. + + """ + N, C, L = inputs.shape + assert outputs.shape == (N, C, L), ( + f"outputs shape {tuple(outputs.shape)} != inputs shape {(N, C, L)}" + ) + assert weight.numel() == C and bias.numel() == C, ( + f"weight/bias must have {C} elements, got {weight.numel()} / {bias.numel()}" + ) + # The flat 1D kernel writes outputs by raw offset; a non-contiguous + # buffer would be written at the wrong locations and cannot be + # auto-materialized without breaking the caller's aliasing. + assert outputs.is_contiguous(), "outputs must be contiguous" + + # Tile size along the flattened element axis. 1024 with 4 warps gives + # 8 elements per lane and benchmarks fastest across realistic image + # sizes; larger tiles starve the SMs on small inputs. + BLOCK_SIZE = 1024 + + numel = N * C * L + # The kernel indexes with int32 offsets (int64 division costs ~2x + # throughput); guard the flat index range here. + assert numel < 2**31, f"numel={numel} exceeds the int32 index range" + + grid = (triton.cdiv(numel, BLOCK_SIZE),) + + _fused_mm_input_norm_kernel[grid]( + inputs.contiguous(), + outputs, + weight.contiguous(), + bias.contiguous(), + numel, + L, + C=C, + BLOCK=BLOCK_SIZE, + num_warps=4, # 8 elements per lane at BLOCK_SIZE=1024 + ) + return outputs + + +class NormParams(NamedTuple): + """Resolved per-channel affine parameters (flags already folded in).""" + + image_mean: list[float] + image_std: list[float] + rescale_factor: float + + @property + def is_identity(self) -> bool: + """Whether ``weight = rescale/std`` and ``bias = -mean/std`` are + numerically the identity transform (mirrors ``torch.allclose``'s + default fp32 tolerances).""" + return all( + math.isclose(self.rescale_factor / s, 1.0, rel_tol=1e-5, abs_tol=1e-8) + and math.isclose(m / s, 0.0, abs_tol=1e-8) + for m, s in zip(self.image_mean, self.image_std) + ) + + +def _load_norm_params(model_config: ModelConfig) -> NormParams: + """Resolve the per-channel affine parameters ``(image_mean, image_std, + rescale_factor)`` from the processor config, falling back to the image + processor object.""" + model = model_config.model + revision = model_config.revision + + config = get_processor_config(model, revision=revision) + # NOTE: do not use cached_image_processor_from_config here — it merges + # mm_processor_kwargs, which mm_device_do_normalize poisons with + # do_normalize=False. + image_processor = cached_get_processor( + model, revision=revision, trust_remote_code=model_config.trust_remote_code + ).image_processor + + def resolve(key: str) -> Any: + """Processor config value, falling back to the image_processor.""" + if (value := config.get(key)) is not None: + return value + return getattr(image_processor, key, None) + + do_rescale = bool(resolve("do_rescale")) + do_normalize = bool(resolve("do_normalize")) + + # Parameters whose flag is off are unused; default them to no-ops + # without resolving them. + rescale_factor = resolve("rescale_factor") if do_rescale else 1.0 + image_mean = resolve("image_mean") if do_normalize else [0.0] * 3 + image_std = resolve("image_std") if do_normalize else [1.0] * 3 + + assert rescale_factor is not None, "rescale_factor is still None after resolution." + assert image_mean is not None, "image_mean is still None after resolution." + assert image_std is not None, "image_std is still None after resolution." + assert len(image_mean) == len(image_std), ( + f"image_mean and image_std have different lengths: " + f"{len(image_mean)} vs {len(image_std)}" + ) + + return NormParams( + image_mean=[float(v) for v in image_mean], + image_std=[float(v) for v in image_std], + rescale_factor=float(rescale_factor), + ) + + +class IdentityInputNorm(nn.Module): + """Stand-in used when the processor requires no rescale/normalise. + + Not a no-op: with ``mm_device_do_normalize`` enabled, raw ``uint8`` + pixels travel to the device unprocessed and must be cast to + ``visual_dtype`` here. + """ + + def forward( + self, pixel_values: torch.Tensor, visual_dtype: torch.dtype + ) -> torch.Tensor: + return pixel_values.to(visual_dtype, copy=False) + + +@CustomOp.register("fused_mm_input_norm") +class FusedMMInputNorm(CustomOp): + """Module that applies rescaling and normalisation to input images. + Equivalent to: output = (input * rescale_factor - mean) / std + + Dtype semantics: + + * Input dtype — the dtype of the ``pixel_values`` argument to ``forward_*``. + It is ``uint8`` when ``mm_device_do_normalize`` is enabled (raw bytes + travel to the device unprocessed) and equals ``visual_dtype`` + otherwise. + * Output dtype — the ``visual_dtype`` argument of ``forward_*``. The + computation itself is always fp32, independent of the output dtype + (e.g. compute fp32, emit bf16). + + Platform dispatch: + + * ``forward_native`` — pure PyTorch eager path, used as the semantic + reference and default fallback on all platforms. + * ``forward_cuda`` — Triton kernel path for CUDA devices. + * ``forward_xpu`` — custom XPU kernel path. + * ``forward_oot`` — out-of-tree platform override entry point; falls back + to ``forward_native`` unless a plugin overrides it. + """ + + def __init__( + self, + image_mean: list[float], + image_std: list[float], + rescale_factor: float, + channel: int = 3, + ): + super().__init__() + + assert len(image_mean) == len(image_std) == channel, ( + f"image_mean/image_std must have {channel} entries, " + f"got {len(image_mean)} / {len(image_std)}" + ) + assert rescale_factor != 0.0, "rescale_factor must be non-zero" + + self.channel = channel + + # Model construction can set the accelerator as PyTorch's default + # device; build the buffers on CPU first, then move them over. + mean = torch.tensor(image_mean, dtype=torch.float32, device="cpu") + std = torch.tensor(image_std, dtype=torch.float32, device="cpu") + device = torch.get_default_device() + self.register_buffer("weight", (rescale_factor / std).to(device)) + self.register_buffer("bias", (-mean / std).to(device)) + + # ------------------------------------------------------------------ + # Internal helpers shared by the platform-specific forward_* methods + # ------------------------------------------------------------------ + + @staticmethod + def _unpack_2d(pixel_values: torch.Tensor) -> tuple[int, int]: + assert pixel_values.ndim == 2, ( + f"pixel_values must be 2D (patches, size), got {pixel_values.dim()}D " + f"with shape {tuple(pixel_values.shape)}" + ) + patches, size = pixel_values.shape + return patches, size + + def _patch_size(self, size: int) -> int: + assert size % self.channel == 0, ( + f"size={size} is not divisible by channel={self.channel}" + ) + return size // self.channel + + # ------------------------------------------------------------------ + # Platform-specific implementations + # ------------------------------------------------------------------ + + def forward_native( + self, pixel_values: torch.Tensor, visual_dtype: torch.dtype + ) -> torch.Tensor: + """Pure PyTorch eager implementation. + + This is the semantic reference implementation and the fallback used + on any platform without a specialised kernel. + """ + patches, size = self._unpack_2d(pixel_values) + patch_size = self._patch_size(size) + + # weight/bias are fp32, so type promotion makes the arithmetic fp32 + x = pixel_values.reshape(patches, self.channel, patch_size) + x = x * self.weight.view(1, self.channel, 1) + self.bias.view( + 1, self.channel, 1 + ) + return x.view(patches, size).to(visual_dtype) + + def forward_cuda( + self, pixel_values: torch.Tensor, visual_dtype: torch.dtype + ) -> torch.Tensor: + """Triton kernel path for CUDA devices.""" + patches, size = self._unpack_2d(pixel_values) + patch_size = self._patch_size(size) + + x3 = pixel_values.reshape(patches, self.channel, patch_size) + y = torch.empty((patches, size), dtype=visual_dtype, device=pixel_values.device) + y3 = y.view(patches, self.channel, patch_size) + + fused_mm_input_norm_triton(x3, y3, self.weight, self.bias) + return y + + def forward_xpu( + self, pixel_values: torch.Tensor, visual_dtype: torch.dtype + ) -> torch.Tensor: + """XPU fused custom kernel path. + + On XPU, fuse the whole rescale + normalise into a single custom + kernel. The eager path materializes an fp32 intermediate and then + casts back, which adds device-side compute that cancels the + bandwidth saving of transferring uint8 pixel_values. The fused + kernel reads uint8 directly and writes ``visual_dtype`` in one pass. + """ + # The out-of-tree XPU kernel only supports the uint8 input that + # device-side normalisation guarantees; fail loudly otherwise. + assert pixel_values.dtype == torch.uint8, ( + f"xpu_fused_input_norm requires uint8 input, got {pixel_values.dtype}" + ) + return torch.ops.vllm.xpu_fused_input_norm( + pixel_values, self.weight, self.bias, visual_dtype + ) + + def forward_oot( + self, pixel_values: torch.Tensor, visual_dtype: torch.dtype + ) -> torch.Tensor: + """Out-of-tree platform override entrypoint.""" + return self.forward_native(pixel_values, visual_dtype) + + +def build_mm_input_norm(model_config: ModelConfig) -> nn.Module: + """Build the input normalisation module for a model. + + Returns an ``IdentityInputNorm`` when device-side normalisation is + disabled or the processor's rescale/normalise is numerically the identity + transform; otherwise a ``FusedMMInputNorm`` built from the processor's + parameters. + """ + mm_config = getattr(model_config, "multimodal_config", None) + if not getattr(mm_config, "mm_device_do_normalize", False): + return IdentityInputNorm() + + params = _load_norm_params(model_config) + + # Flags off are already defaulted to identity values by + # ``_load_norm_params``; this also catches configs that request + # rescale/normalise but are numerically the identity transform. + if params.is_identity: + return IdentityInputNorm() + + return FusedMMInputNorm( + image_mean=params.image_mean, + image_std=params.image_std, + rescale_factor=params.rescale_factor, + channel=len(params.image_mean), + ) diff --git a/vllm/model_executor/layers/logits_processor.py b/vllm/model_executor/layers/logits_processor.py index 10f425a503..04541212fe 100644 --- a/vllm/model_executor/layers/logits_processor.py +++ b/vllm/model_executor/layers/logits_processor.py @@ -256,6 +256,8 @@ class LogitsProcessor(PluggableLayer): hidden_states: torch.Tensor, k: int, embedding_bias: torch.Tensor | None = None, + *, + return_log_probs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """Vocab-parallel top-k without all-gathering full logits. @@ -263,7 +265,9 @@ class LogitsProcessor(PluggableLayer): the values as well as the global ids. Communication is O(batch * 2k * tp_size) rather than O(batch * vocab_size). - Scale and soft cap are applied to the k selected values rather than + With return_log_probs, values use the full-vocabulary log-partition. + + Otherwise scale and soft cap are applied to the k selected values rather than the whole vocabulary; both are monotonic, so the selection is the same and only k entries are touched. """ @@ -274,12 +278,26 @@ class LogitsProcessor(PluggableLayer): ) logits = self._apply_head(lm_head, hidden_states, embedding_bias) + if return_log_probs: + if self.soft_cap is not None: + logits = torch.tanh(logits / self.soft_cap) * self.soft_cap + if self.scale != 1.0: + logits = logits * self.scale # Mask out padding entries beyond org_vocab_size on this shard. num_pad = lm_head.shard_indices.num_org_vocab_padding if num_pad > 0: logits[..., -num_pad:] = -float("inf") + if return_log_probs: + log_partition = torch.logsumexp(logits.float(), dim=-1, keepdim=True) + if lm_head.tp_size > 1: + log_partition = torch.logsumexp( + tensor_model_parallel_all_gather(log_partition, dim=-1), + dim=-1, + keepdim=True, + ) + values, ids = _topk(logits, k) # Convert shard-local indices to global vocab indices. ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index @@ -291,6 +309,8 @@ class LogitsProcessor(PluggableLayer): ids = ids.gather(-1, selected) values = values.float() + if return_log_probs: + return ids, values - log_partition if self.scale != 1.0: values = values * self.scale if self.soft_cap is not None: diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 95ced2e0a4..249d56685e 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import ( VllmConfig, @@ -24,7 +24,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = "", ) -> None: diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 681ff98ae6..9d5bcc89da 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -59,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import ( from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig -from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, @@ -228,6 +227,7 @@ def fi_chunk_gated_delta_rule( initial_state=fi_state, output_final_state=output_final_state, cu_seqlens=cu_seqlens, + backend="flashinfer", ) # FlashInfer returns (output, state) when output_final_state=True, # or just output when output_final_state=False. @@ -1961,74 +1961,3 @@ direct_register_custom_op( op_func=qwen_gdn_attention_core_fused_norm_packed, mutates_args=["core_attn_out"], ) - - -@triton.jit -def fused_gdn_gating_kernel( - g, - beta_output, - A_log, - a, - b, - dt_bias, - seq_len, - NUM_HEADS: tl.constexpr, - beta: tl.constexpr, - threshold: tl.constexpr, - BLK_HEADS: tl.constexpr, -): - i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2) - head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS) - off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off - mask = head_off < NUM_HEADS - blk_A_log = tl.load(A_log + head_off, mask=mask) - blk_a = tl.load(a + off, mask=mask) - blk_b = tl.load(b + off, mask=mask) - blk_bias = tl.load(dt_bias + head_off, mask=mask) - # If the model is loaded in fp16, without the .float() here, A might be -inf - x = blk_a.to(tl.float32) + blk_bias.to(tl.float32) - softplus_x = tl.where( - beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x - ) - blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x - tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask) - # compute beta_output = sigmoid(b) - blk_beta_output = tl.sigmoid(blk_b.to(tl.float32)) - tl.store( - beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask - ) - - -def fused_gdn_gating( - A_log: torch.Tensor, - a: torch.Tensor, - b: torch.Tensor, - dt_bias: torch.Tensor, - beta: float = 1.0, - threshold: float = 20.0, -) -> tuple[torch.Tensor, torch.Tensor]: - """Fused computation of g and beta for Gated Delta Net. - g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias) - beta_output = b.sigmoid() - TODO maybe use torch.compile to replace this triton kernel - """ - batch, num_heads = a.shape - seq_len = 1 - grid = (batch, seq_len, triton.cdiv(num_heads, 8)) - g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device) - beta_output = torch.empty(1, batch, num_heads, dtype=b.dtype, device=b.device) - fused_gdn_gating_kernel[grid]( - g, - beta_output, - A_log, - a, - b, - dt_bias, - seq_len, - num_heads, - beta, - threshold, - 8, - num_warps=1, - ) - return g, beta_output diff --git a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py index 75a7d097a8..a4283e2482 100644 --- a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py @@ -4,7 +4,7 @@ import copy import torch import torch.nn.functional as F -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.config import ( VllmConfig, @@ -37,7 +37,7 @@ from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata -def _build_rope_parameters(config: PretrainedConfig) -> dict | None: +def _build_rope_parameters(config: PreTrainedConfig) -> dict | None: rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {} if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"): rope_parameters["rope_theta"] = config.rope_theta @@ -385,7 +385,7 @@ class BailingMoELinearAttention(LinearAttention): # --8<-- [end:bailing_moe_linear_attention] def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = "linear_attn", ): diff --git a/vllm/model_executor/layers/mamba/linear/base.py b/vllm/model_executor/layers/mamba/linear/base.py index 73df071873..da6808984b 100644 --- a/vllm/model_executor/layers/mamba/linear/base.py +++ b/vllm/model_executor/layers/mamba/linear/base.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import ( VllmConfig, @@ -24,7 +24,7 @@ class LinearAttention(PluggableLayer, MambaBase): """Base class for Linear attention layer.""" def __init__( - self, config: PretrainedConfig, vllm_config: VllmConfig, prefix: str = "" + self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = "" ): super().__init__() self.layer_idx = extract_layer_index(prefix) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index db85ca0dee..b567d188eb 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -56,7 +56,6 @@ from vllm.model_executor.model_loader.weight_utils import ( from vllm.model_executor.parameter import BasevLLMParameter from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform -from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, @@ -927,67 +926,54 @@ class MambaMixer2(MambaBase, PluggableLayer): # then chunk_stride = 2 chunk_stride = mamba_block_size // chunk_size - # The per-sequence loop below uses these as Python scalars. - # TODO avoid sync here? - with gpu_sync_allowed(): - block_idx_first_cpu = block_idx_first_scheduled_token_p.tolist() - block_idx_last_cpu = block_idx_last_scheduled_token_p.tolist() - num_computed_tokens_p_cpu = num_computed_tokens_p.tolist() - last_chunk_indices_p_cpu = last_chunk_indices_p.tolist() + # Save state for sequences with more than just final state, + # batched over sequences so that the number of kernel launches + # does not scale with the number of prefills. + n_blocks_to_fill = ( + block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p + ).clamp(min=0) + total_fill = attn_metadata.num_state_writes_p + if total_fill > 0: + first_chunk = torch.zeros_like(last_chunk_indices_p) + first_chunk[1:] = last_chunk_indices_p[:-1] + 1 - # Save state for sequences with more than just final state - for seq_idx in range(num_prefills): - # Block index for the first scheduled token - block_idx_first_scheduled_token = block_idx_first_cpu[seq_idx] - - # Block index for the last scheduled token - block_idx_last_scheduled_token = block_idx_last_cpu[seq_idx] - - # Number of blocks that need to be written - n_blocks_to_fill = ( - block_idx_last_scheduled_token - block_idx_first_scheduled_token - ) - - # Skip sequences that don't have any blocks to fill - if n_blocks_to_fill == 0: - continue - - # Look up the state indices - cache_blocks_to_fill = state_indices_tensor_p[ - seq_idx, - block_idx_first_scheduled_token:block_idx_last_scheduled_token, - ] - - # First chunk index for this sequence - if seq_idx == 0: - first_chunk = 0 - else: - first_chunk = 1 + last_chunk_indices_p_cpu[seq_idx - 1] - - # First chunk that is aligned on the mamba block boundary - first_aligned_chunk = first_chunk + chunk_stride - 1 - - # Calculate the number of computed tokens that were not - # already cached + # The chunk that completes this sequence's first mamba block. + # A block spans chunk_stride chunks. When the sequence resumed + # mid-block, the chunks before this step are not in varlen_states, + # so subtract them: the boundary sits earlier by their count. num_unaligned_computed_tokens = ( - num_computed_tokens_p_cpu[seq_idx] % mamba_block_size + num_computed_tokens_p % mamba_block_size + ) + unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size + first_aligned_chunk = ( + first_chunk + (chunk_stride - 1) - unaligned_chunk_shift ) - if num_unaligned_computed_tokens > 0: - # If the number of computed tokens is not block aligned, - # then we need to shift the index accordingly - first_aligned_chunk -= ( - num_unaligned_computed_tokens // chunk_size - ) + # Flatten all writes into one list: fill_counts=[2,1,2] + # -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1] + fill_counts = n_blocks_to_fill.long() + # output_size, or repeat_interleave reads the size off the device + seq_ids = torch.repeat_interleave( + torch.arange(num_prefills, device=ssm_state.device), + fill_counts, + output_size=total_fill, + ) + write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts + write_offsets = torch.arange( + total_fill, device=ssm_state.device + ) - torch.repeat_interleave( + write_starts, fill_counts, output_size=total_fill + ) - # Get states to write - from_where = varlen_states[ - first_aligned_chunk : first_aligned_chunk - + n_blocks_to_fill * chunk_stride : chunk_stride + cache_blocks_to_fill = state_indices_tensor_p[ + seq_ids, + block_idx_first_scheduled_token_p[seq_ids] + write_offsets, + ] + # Write the states. Each row goes to its own cache block, + # so the indexed copy has no duplicate destinations. + ssm_state[cache_blocks_to_fill] = varlen_states[ + first_aligned_chunk[seq_ids] + write_offsets * chunk_stride ] - - # Write the states - ssm_state[cache_blocks_to_fill] = from_where # For all seqs, store the last state (note: might be partial): assert state_indices_tensor_p is not None diff --git a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py index b5c1d11a16..f3f3a33ab1 100644 --- a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py +++ b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py @@ -27,6 +27,104 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata _CPU_GDN_ATTENTION_OPS_REGISTERED = False +def _stage_ds_conv_states( + conv_states: torch.Tensor, state_indices: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + storage = conv_states.transpose(1, 2).index_select(0, state_indices) + scratch = storage.transpose(1, 2) + identity_indices = torch.arange( + state_indices.numel(), dtype=torch.int32, device=state_indices.device + ) + return scratch, identity_indices + + +def _scatter_ds_conv_states( + conv_states: torch.Tensor, + state_indices: torch.Tensor, + scratch: torch.Tensor, +) -> None: + conv_states.index_copy_(0, state_indices.to(torch.int64), scratch) + + +def _causal_conv1d_fwd_cpu( + *, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + conv_states: torch.Tensor, + query_start_loc: torch.Tensor, + cache_indices: torch.Tensor, + has_initial_state: torch.Tensor, + silu_activation: bool, + is_vnni: bool, +) -> torch.Tensor: + if not is_conv_state_dim_first(): + return ops.causal_conv1d_fwd_cpu( + x=x, + weight=weight, + bias=bias, + conv_states=conv_states, + query_start_loc=query_start_loc, + cache_indices=cache_indices, + has_initial_state=has_initial_state, + silu_activation=silu_activation, + is_vnni=is_vnni, + ) + + scratch, identity_indices = _stage_ds_conv_states(conv_states, cache_indices) + out = ops.causal_conv1d_fwd_cpu( + x=x, + weight=weight, + bias=bias, + conv_states=scratch, + query_start_loc=query_start_loc, + cache_indices=identity_indices, + has_initial_state=has_initial_state, + silu_activation=silu_activation, + is_vnni=is_vnni, + ) + _scatter_ds_conv_states(conv_states, cache_indices, scratch) + return out + + +def _causal_conv1d_update_cpu( + *, + x: torch.Tensor, + conv_states: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + silu_activation: bool, + conv_state_indices: torch.Tensor, + is_vnni: bool, + num_accepted_tokens: torch.Tensor | None = None, +) -> torch.Tensor: + if not is_conv_state_dim_first(): + return ops.causal_conv1d_update_cpu( + x=x, + conv_states=conv_states, + weight=weight, + bias=bias, + silu_activation=silu_activation, + conv_state_indices=conv_state_indices, + is_vnni=is_vnni, + num_accepted_tokens=num_accepted_tokens, + ) + + scratch, identity_indices = _stage_ds_conv_states(conv_states, conv_state_indices) + out = ops.causal_conv1d_update_cpu( + x=x, + conv_states=scratch, + weight=weight, + bias=bias, + silu_activation=silu_activation, + conv_state_indices=identity_indices, + is_vnni=is_vnni, + num_accepted_tokens=num_accepted_tokens, + ) + _scatter_ds_conv_states(conv_states, conv_state_indices, scratch) + return out + + def cpu_gdn_attention_core( mixed_qkv: torch.Tensor, b: torch.Tensor, @@ -109,10 +207,8 @@ def _cpu_gdn_attention_nonspec( conv_state = layer.kv_cache[0] if use_cpp_conv: - # C++ conv requires [num_allocated_slots, kernel - 1, conv_dim] (SD). - if is_conv_state_dim_first(): - raise RuntimeError("C++ CPU GDN attention requires `SD` conv_state layout.") - conv_state = conv_state.transpose(1, 2) + if not is_conv_state_dim_first(): + conv_state = conv_state.transpose(1, 2) else: if not is_conv_state_dim_first(): conv_state = conv_state.transpose(-1, -2) @@ -141,7 +237,7 @@ def _cpu_gdn_attention_nonspec( decode_a = a[:num_decode_tokens] decode_state_indices = state_indices_tensor[:num_decodes] if use_cpp_conv: - decode_mixed_qkv = ops.causal_conv1d_update_cpu( + decode_mixed_qkv = _causal_conv1d_update_cpu( x=decode_mixed_qkv, conv_states=conv_state, weight=layer.conv1d.weight, @@ -210,7 +306,7 @@ def _cpu_gdn_attention_nonspec( ] if use_cpp_conv: - prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu( + prefill_mixed_qkv = _causal_conv1d_fwd_cpu( x=prefill_mixed_qkv.transpose(0, 1), weight=layer.conv1d.weight, bias=layer.conv1d.bias, @@ -416,7 +512,6 @@ def _spec_forward( can_use_native_conv = ( torch.cpu._is_amx_tile_supported() - and not is_conv_state_dim_first() and width == 4 and num_spec_decodes > 0 and bool(torch.all(seq_lens == seq_lens[0]).item()) @@ -424,7 +519,7 @@ def _spec_forward( ) if can_use_native_conv: q_i = int(seq_lens[0].item()) - conv_out = ops.causal_conv1d_update_cpu( + conv_out = _causal_conv1d_update_cpu( x=mixed_qkv_spec.view(num_spec_decodes, q_i, dim), conv_states=conv_buf, weight=layer.conv1d.weight, @@ -509,8 +604,6 @@ def _spec_aware_nonspec( state_indices_tensor = state_indices_tensor.contiguous() is_amx = torch.cpu._is_amx_tile_supported() - if is_amx and is_conv_state_dim_first(): - raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.") if not is_amx: conv_weights = _unpacked_conv_weight(layer) @@ -526,7 +619,7 @@ def _spec_aware_nonspec( decode_a = a[:num_decode_tokens] decode_state_indices = state_indices_tensor[:num_decodes] if is_amx: - decode_mixed_qkv = ops.causal_conv1d_update_cpu( + decode_mixed_qkv = _causal_conv1d_update_cpu( x=decode_mixed_qkv, conv_states=conv_buf, weight=layer.conv1d.weight, @@ -593,7 +686,7 @@ def _spec_aware_nonspec( num_decodes : num_decodes + num_prefills ] if is_amx: - prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu( + prefill_mixed_qkv = _causal_conv1d_fwd_cpu( x=prefill_mixed_qkv.transpose(0, 1), weight=layer.conv1d.weight, bias=layer.conv1d.bias, @@ -664,11 +757,9 @@ def _spec_aware_nonspec_subset( prefill_state_indices = prefill_state_indices.contiguous() is_amx = torch.cpu._is_amx_tile_supported() - if is_amx and is_conv_state_dim_first(): - raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.") if is_amx: - conv_out = ops.causal_conv1d_fwd_cpu( + conv_out = _causal_conv1d_fwd_cpu( x=mixed_qkv.transpose(0, 1), weight=layer.conv1d.weight, bias=layer.conv1d.bias, diff --git a/vllm/model_executor/layers/pooler/activations.py b/vllm/model_executor/layers/pooler/activations.py index f0c64d2720..fc47684a12 100644 --- a/vllm/model_executor/layers/pooler/activations.py +++ b/vllm/model_executor/layers/pooler/activations.py @@ -7,7 +7,7 @@ from typing import TypeVar import torch import torch.nn as nn import torch.nn.functional as F -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import ModelConfig from vllm.logger import init_logger @@ -17,7 +17,7 @@ logger = init_logger(__name__) def get_act_fn( - config: PretrainedConfig, + config: PreTrainedConfig, static_num_labels: bool = True, ) -> "PoolerActivation": # get classification act_fn diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index b17a17d083..0c3e980551 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Union import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE from torch.nn import Parameter -from transformers import PretrainedConfig +from transformers import PreTrainedConfig import vllm.model_executor.layers.fused_moe # noqa from vllm import _custom_ops as ops @@ -370,7 +370,7 @@ class AutoAWQConfig(QuantizationConfig): def maybe_update_config( self, model_name: str, - hf_config: PretrainedConfig | None = None, + hf_config: PreTrainedConfig | None = None, revision: str | None = None, ): if self.modules_to_not_convert: diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index 960ee95d60..ff57f3b8c5 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -6,7 +6,7 @@ from typing import Any import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE -from transformers import PretrainedConfig +from transformers import PreTrainedConfig import vllm.model_executor.layers.fused_moe # noqa from vllm.logger import init_logger @@ -273,7 +273,7 @@ class AutoGPTQConfig(QuantizationConfig): def maybe_update_config( self, model_name: str, - hf_config: PretrainedConfig | None = None, + hf_config: PreTrainedConfig | None = None, revision: str | None = None, ): if self.modules_in_block_to_quantize: diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py index a2c6eb2a45..1b0457854e 100644 --- a/vllm/model_executor/layers/quantization/base_config.py +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any import regex as re import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig if TYPE_CHECKING: from vllm.model_executor.layers.quantization import QuantizationMethods @@ -265,7 +265,7 @@ class QuantizationConfig(ABC): def maybe_update_config( # noqa: B027 self, model_name: str, - hf_config: PretrainedConfig | None = None, + hf_config: PreTrainedConfig | None = None, revision: str | None = None, ): """Interface to update values after config initialization. diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 3603fd8bde..cb69a1d665 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -511,6 +511,13 @@ class CompressedTensorsConfig(QuantizationConfig): def _is_dynamic_token_w4a8_int( weight_quant: QuantizationArgs, input_quant: QuantizationArgs ) -> bool: + if not weight_quant or not input_quant: + return False + if ( + weight_quant.type != QuantizationType.INT + or input_quant.type != QuantizationType.INT + ): + return False is_weight_4_bits = weight_quant.num_bits == 4 is_activation_8_bits = input_quant.num_bits == 8 weight_strategy = ( @@ -575,6 +582,11 @@ class CompressedTensorsConfig(QuantizationConfig): ) -> bool: if not weight_quant or not input_quant: return False + if ( + weight_quant.type != QuantizationType.INT + or input_quant.type != QuantizationType.FLOAT + ): + return False is_weight_4_bits = weight_quant.num_bits == 4 is_activation_8_bits = input_quant.num_bits == 8 weight_strategy = weight_quant.strategy == QuantizationStrategy.GROUP.value @@ -787,6 +799,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, @@ -799,6 +812,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py index f2d2a9c945..d965127b81 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py @@ -164,7 +164,7 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): return CompressedTensorsW8A8Int8MoEMethod( weight_quant, input_quant, layer.moe_config ) - elif quant_config._is_fp8_w4a8_sm90(weight_quant, input_quant): + elif quant_config._is_fp8_w4a8(weight_quant, input_quant): from .compressed_tensors_moe_w4a8_fp8 import ( CompressedTensorsW4A8Fp8MoEMethod, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index f7d12c8635..643fd75e82 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( XPUExpertsMxFp4, ) +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( B12X_BACKENDS, Mxfp4MoeBackend, @@ -227,7 +228,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( x, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index 54f5aa86b3..51b7d0460a 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -303,7 +303,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( x, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py index 8f98cb4813..4a2223aab8 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py @@ -143,57 +143,49 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): layer.register_parameter("w13_weight_shape", w13_weight_shape) set_weight_attrs(w13_weight_shape, extra_weight_attrs) - w13_weight_chan_scale = torch.nn.Parameter( - torch.ones( - num_experts, - self.moe.w13_num_shards * intermediate_size_per_partition, - dtype=torch.float32, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight_chan_scale", w13_weight_chan_scale) - - w2_weight_chan_scale = torch.nn.Parameter( - torch.ones(num_experts, hidden_size, dtype=torch.float32), - requires_grad=False, - ) - layer.register_parameter("w2_weight_chan_scale", w2_weight_chan_scale) - # don't use input scales layer.w13_input_scale = None layer.w2_input_scale = None def process_weights_after_loading(self, layer: RoutedExperts) -> None: + for name in ("w13", "w2"): + setattr(layer, f"{name}_weight", getattr(layer, f"{name}_weight_packed")) + delattr(layer, f"{name}_weight_packed") + ( - w13_weight_packed, - w2_weight_packed, + w13_weight, + w2_weight, w13_weight_scale, w2_weight_scale, w13_weight_chan_scale, w2_weight_chan_scale, - b_strides1, - b_strides2, + self.b_strides1, + self.b_strides2, ) = convert_to_w4a8_moe_kernel_format( - w13_weight_packed=layer.w13_weight_packed, - w2_weight_packed=layer.w2_weight_packed, + backend=self.w4a8_backend, + layer=layer, + group_size=self.group_size, + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, w13_weight_scale=layer.w13_weight_scale, w2_weight_scale=layer.w2_weight_scale, ) - replace_parameter(layer, "w13_weight_packed", w13_weight_packed) - replace_parameter(layer, "w2_weight_packed", w2_weight_packed) - replace_parameter(layer, "w13_weight_scale", w13_weight_scale) - replace_parameter(layer, "w2_weight_scale", w2_weight_scale) - replace_parameter(layer, "w13_weight_chan_scale", w13_weight_chan_scale) - replace_parameter(layer, "w2_weight_chan_scale", w2_weight_chan_scale) - - self.b_strides1 = b_strides1 - self.b_strides2 = b_strides2 + for name, tensor in ( + ("w13_weight", w13_weight), + ("w2_weight", w2_weight), + ("w13_weight_scale", w13_weight_scale), + ("w2_weight_scale", w2_weight_scale), + ("w13_weight_chan_scale", w13_weight_chan_scale), + ("w2_weight_chan_scale", w2_weight_chan_scale), + ): + replace_parameter(layer, name, tensor) self.moe_quant_config = self.get_fused_moe_quant_config(layer) if self.moe_quant_config is not None: assert self.experts_cls is not None self.moe_kernel = make_w4a8_moe_kernel( + backend=self.w4a8_backend, moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, @@ -205,10 +197,12 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: return make_w4a8_moe_quant_config( + backend=self.w4a8_backend, w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, g1_alphas=layer.w13_weight_chan_scale, g2_alphas=layer.w2_weight_chan_scale, + layer=layer, ) def apply( @@ -224,8 +218,8 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): assert self.moe_kernel is not None return self.moe_kernel.apply( hidden_states=x, - w1=layer.w13_weight_packed, - w2=layer.w2_weight_packed, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 262cd307a8..82f3ada446 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, @@ -375,7 +376,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py index ee3b321363..b5326e1875 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, @@ -188,7 +189,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index cb9cb34158..ce4520cbbe 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -153,12 +153,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): layer.logical_widths, getattr(layer, "input_scale", None), ) - weight = weight.t() elif self.strategy == QuantizationStrategy.CHANNEL: weight, weight_scale, input_scale = process_fp8_weight_channel_strategy( layer.weight, layer.weight_scale, getattr(layer, "input_scale", None) ) - weight = weight.t() elif self.strategy == QuantizationStrategy.BLOCK: assert self.is_static_input_scheme is False @@ -177,8 +175,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): # required by torch.compile to be torch.nn.Parameter layer.weight = Parameter(weight.data, requires_grad=False) - # Preserve the dim tags dropped by the transpose so layout-aware - # kernels (humming) see (K, N) instead of assuming (N, K). + # The helpers return (K, N); retag so layout-aware kernels see it. layer.weight.input_dim = 0 layer.weight.output_dim = 1 layer.weight_scale = Parameter(weight_scale.data, requires_grad=False) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_mxfp8.py index 0f72be13a5..83bf4bb9cb 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_mxfp8.py @@ -33,9 +33,6 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme): - Activations dynamically quantized to MXFP8 during inference """ - def __init__(self): - self.kernel = init_mxfp8_linear_kernel() - @classmethod def get_min_capability(cls) -> int: return 75 @@ -79,6 +76,8 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index 42688ab834..bf14f8a60b 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = { 7: scalar_types.uint7b64, 8: scalar_types.uint8b128, } -WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8} +WNA16_ZP_SUPPORTED_TYPES_MAP = { + 2: scalar_types.uint2, + 3: scalar_types.uint3, + 4: scalar_types.uint4, + 5: scalar_types.uint5, + 6: scalar_types.uint6, + 7: scalar_types.uint7, + 8: scalar_types.uint8, +} WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys()) @@ -111,8 +119,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): layer.output_size_per_partition = output_size_per_partition layer.output_partition_sizes = output_partition_sizes layer.params_dtype = params_dtype - if not hasattr(layer, "has_bias"): - layer.has_bias = False mp_linear_kernel_config = MPLinearLayerConfig( full_weight_shape=(input_size, output_size), diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py index 427c92310f..465f21a827 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming. +"""Weight N-bit INT scheme with INT4 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT4 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA4Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -131,8 +147,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): layer.output_size_per_partition = output_size_per_partition layer.output_partition_sizes = output_partition_sizes layer.params_dtype = params_dtype - if not hasattr(layer, "has_bias"): - layer.has_bias = False mp_config = MPLinearLayerConfig( full_weight_shape=(input_size, output_size), @@ -143,7 +157,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +165,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +175,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +198,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +215,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +251,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py index a14077d899..8c6a2c84ff 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming. +"""Weight N-bit INT scheme with INT8 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT8 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA8Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -131,8 +147,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): layer.output_size_per_partition = output_size_per_partition layer.output_partition_sizes = output_partition_sizes layer.params_dtype = params_dtype - if not hasattr(layer, "has_bias"): - layer.has_bias = False mp_config = MPLinearLayerConfig( full_weight_shape=(input_size, output_size), @@ -143,7 +157,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +165,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +175,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +198,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +215,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +251,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py index 187b6438ab..dd1b986dc3 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py @@ -97,12 +97,8 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme): output_size_per_partition = sum(output_partition_sizes) layer.input_size_per_partition = input_size_per_partition layer.output_size_per_partition = output_size_per_partition - # Set for kernels' weight prep; also covers ParallelLMHead, which does - # not set these in __init__. layer.output_partition_sizes = output_partition_sizes layer.params_dtype = params_dtype - if not hasattr(layer, "has_bias"): - layer.has_bias = False mp_config = MPLinearLayerConfig( full_weight_shape=(input_size, output_size), diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py b/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py index ae8f143a61..8552eeb652 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py @@ -95,13 +95,13 @@ class HadamardTransform(torch.nn.Module): if self.transforms[part_id].scheme.head_dim is not None: weight_size = self.transforms[part_id].scheme.head_dim value = value.unflatten(-1, (-1, weight_size)) - value = ops.hadacore_transform(value) + value = ops.hadacore_transform(value, inplace=False) value = value.flatten(-2, -1) return value # sylvester transforms are symmetric, inv => transpose => original - return ops.hadacore_transform(value) + return ops.hadacore_transform(value, inplace=False) # fall back to dense else: diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index c85a2c288e..37bb9c6840 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe import ( UnquantizedFusedMoEMethod, ) from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, @@ -93,7 +94,7 @@ class Fp8Config(QuantizationConfig): def __init__( self, - is_checkpoint_fp8_serialized: bool = False, + is_checkpoint_fp8_serialized: bool = True, activation_scheme: str = "dynamic", ignored_layers: list[str] | None = None, weight_block_size: list[int] | None = None, @@ -102,6 +103,13 @@ class Fp8Config(QuantizationConfig): super().__init__() self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + if not is_checkpoint_fp8_serialized: + raise ValueError( + "The `fp8` quantization method no longer supports online " + "quantization. Please use `--quantization fp8_per_tensor` " + "instead. See " + "https://docs.vllm.ai/en/stable/features/quantization/online/" + ) if activation_scheme not in ACTIVATION_SCHEMES: raise ValueError(f"Unsupported activation scheme {activation_scheme}") @@ -112,11 +120,6 @@ class Fp8Config(QuantizationConfig): ) self.store_dtype = store_dtype if weight_block_size is not None: - if not is_checkpoint_fp8_serialized: - raise ValueError( - "The block-wise quantization only supports fp8-serialized " - "checkpoint for now." - ) if len(weight_block_size) != 2: raise ValueError( "The quantization block size of weight must have 2 " @@ -182,18 +185,9 @@ class Fp8Config(QuantizationConfig): match_mode=self.ignored_layers_match_mode, ): return UnquantizedLinearMethod() - if not self.is_checkpoint_fp8_serialized: - from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineLinearMethod, - ) - - online_method = Fp8PerTensorOnlineLinearMethod() - online_method.marlin_input_dtype = get_marlin_input_dtype(prefix) - return online_method - else: - offline_method = Fp8LinearMethod(self) - offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix) - return offline_method + offline_method = Fp8LinearMethod(self) + offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix) + return offline_method elif isinstance(layer, RoutedExperts): if is_layer_skipped( prefix=prefix, @@ -208,14 +202,7 @@ class Fp8Config(QuantizationConfig): ) return Mxfp4MoEMethod(layer.moe_config) - if self.is_checkpoint_fp8_serialized: - return Fp8MoEMethod(self, layer) - else: - from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineMoEMethod, - ) - - return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config) + return Fp8MoEMethod(self, layer) elif isinstance(layer, Attention): return Fp8KVCacheMethod(self) return None @@ -416,7 +403,6 @@ class Fp8LinearMethod(LinearMethodBase): if self.act_q_static: assert input_scale is not None input_scale = input_scale.max() - weight = weight.t() # Update layer with new values. replace_parameter(layer, "weight", weight.data) @@ -549,7 +535,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.orig_dtype = params_dtype layer.weight_block_size = None - assert self.quant_config.is_checkpoint_fp8_serialized params_dtype = torch.float8_e4m3fn if self.block_quant: @@ -843,7 +828,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py index b1a8a294f1..53e8ca2b01 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( B12X_BACKENDS, Mxfp4MoeBackend, @@ -228,7 +229,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( x, diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py index 775b5f5bba..b084c697ea 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py @@ -17,9 +17,6 @@ from .inc_scheme import INCLinearScheme class INCMxfp8LinearScheme(INCLinearScheme): - def __init__(self) -> None: - self.kernel = init_mxfp8_linear_kernel() - @classmethod def get_min_capability(cls) -> int: return 75 @@ -72,6 +69,8 @@ class INCMxfp8LinearScheme(INCLinearScheme): ) layer.register_parameter("weight_scale", weight_scale) + self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py index 2c9a06b798..ed7bd9fed0 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, @@ -179,7 +180,7 @@ class INCMxfp8MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( x, diff --git a/vllm/model_executor/layers/quantization/input_quant_fp8.py b/vllm/model_executor/layers/quantization/input_quant_fp8.py index cf45552735..161baf446d 100644 --- a/vllm/model_executor/layers/quantization/input_quant_fp8.py +++ b/vllm/model_executor/layers/quantization/input_quant_fp8.py @@ -155,8 +155,11 @@ class QuantFP8(CustomOp): if use_aiter_per_token_quant: return rocm_aiter_ops.per_token_quant(x, _FP8_DTYPE, scale) - # Fallback to CUDA implementation - return self.forward_cuda(x, scale, scale_ub) + # Fall back to the CUDA implementation. Dispatch on the class, not + # through self: a subclass may override forward_cuda with a different + # signature (_DecodeConcatQuantFP8 does), and a virtual call would + # re-enter it with mismatched arguments. + return QuantFP8.forward_cuda(self, x, scale, scale_ub) def forward_xpu( self, @@ -175,7 +178,8 @@ class QuantFP8(CustomOp): dtype=_FP8_DTYPE, use_ue8m0=self.use_ue8m0, ) - return self.forward_cuda(x, scale, scale_ub, use_triton) + # Dispatch on the class, for the same reason as in forward_hip. + return QuantFP8.forward_cuda(self, x, scale, scale_ub, use_triton) def forward_native( self, diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 364c52b4c8..90ac2b8c79 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -690,7 +690,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( @@ -1102,7 +1102,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( @@ -1471,7 +1471,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( @@ -2129,7 +2129,7 @@ class KFp8StaticChannel(QuantKeyScheme): weight, weight_scale, _ = process_fp8_weight_channel_strategy( layer.weight, layer.weight_scale.data ) - layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight = Parameter(weight, requires_grad=False) layer.weight_scale = Parameter(weight_scale, requires_grad=False) @@ -2320,7 +2320,8 @@ def select_linear_kernel( return init_nvfp4_linear_kernel(use_a16=spec.activation is None) if w.scale.dtype == MXFP8_SCALE_DTYPE: return init_mxfp8_linear_kernel( - bmm_batch_size=getattr(layer, "bmm_batch_size", None) + weight_shape=weight_shape or layer.weight.shape, + bmm_batch_size=getattr(layer, "bmm_batch_size", None), ) # fp8 family: init_fp8 routes block-vs-plain itself off the activation key, # and needs a real key -- weight-only fp8 is not a ModelOpt format. @@ -2508,12 +2509,6 @@ class ModelOptLinearMethod(LinearMethodBase): layer.logical_widths = output_partition_sizes layer.input_size_per_partition = input_size_per_partition layer.output_size_per_partition = sum(output_partition_sizes) - # Humming reads both off the layer in - # prepare_humming_linear_layer_config. LinearBase sets them itself; - # ParallelLMHead does not, so supply them here. - layer.output_partition_sizes = output_partition_sizes - if not hasattr(layer, "has_bias"): - layer.has_bias = getattr(layer, "bias", None) is not None shapes = Shapes(output_partition_sizes, input_size_per_partition, params_dtype) self.wkey.create_weights(layer, WEIGHT, self.ctx, shapes, weight_loader) @@ -2532,7 +2527,10 @@ class ModelOptLinearMethod(LinearMethodBase): def process_weights_after_loading(self, layer) -> None: if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False): - self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size) + self.kernel = init_mxfp8_linear_kernel( + weight_shape=(layer.weight.shape[-2], layer.weight.shape[-1]), + bmm_batch_size=layer.bmm_batch_size, + ) if is_weights_pre_processed(): if not self.supports_pre_processed_weights: raise RuntimeError( diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 33f6afac80..dcba98c7e4 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -433,7 +433,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( @@ -860,7 +860,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index 866c88b3d6..414c4aba5b 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -166,7 +166,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert not self.is_monolithic assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 5165762d5b..9d5814206c 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -42,10 +42,6 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase): FP8 with block-32 scales) during weight loading. """ - def __init__(self): - super().__init__() - self.kernel = init_mxfp8_linear_kernel() - def create_weights( self, layer: torch.nn.Module, @@ -73,6 +69,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase): **extra_weight_attrs, ) + self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape) + def process_weights_after_loading(self, layer: Module) -> None: if getattr(layer, "_already_called_process_weights_after_loading", False): return diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py index 7abefce257..efc97705c5 100644 --- a/vllm/model_executor/layers/quantization/online/nvfp4.py +++ b/vllm/model_executor/layers/quantization/online/nvfp4.py @@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( amax_for_moe_weight_quant, - kNvfp4Dynamic, + kNvfp4DynamicToken, kNvfp4Static, weight_amax, ) @@ -100,7 +100,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( config=self.moe, weight_key=kNvfp4Static, - activation_key=kNvfp4Dynamic, + activation_key=kNvfp4DynamicToken, ) def process_weights_after_loading(self, layer: Module) -> None: @@ -178,6 +178,14 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): routing_tables=layer._expert_routing_tables(), per_token_activation=True, ) + else: + # Reload creates new scale tensors; derived kernel scales must use + # their new values before layerwise reload restores captured storage. + assert self.moe_quant_config is not None + assert self.moe_quant_config.g1_alphas is not None + assert self.moe_quant_config.g2_alphas is not None + self.moe_quant_config.g1_alphas.copy_(w13_scale_2) + self.moe_quant_config.g2_alphas.copy_(w2_scale_2) self.moe_kernel.fused_experts.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index a3a3a36f00..2951f578b0 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -5,7 +5,6 @@ import fnmatch from typing import TYPE_CHECKING, Any, NamedTuple, cast import torch -from transformers import PretrainedConfig from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -46,6 +45,8 @@ from vllm.model_executor.layers.quantization.quark.utils import ( from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( _ACTIVATION_QUANT_KEY_MAP, _WEIGHT_QUANT_KEY_MAP, + OCP_MX_BLOCK_SIZE, + ocp_mx_weight_dtype_and_rows, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -82,10 +83,6 @@ __all__ = ["QuarkLinearMethod"] logger = init_logger(__name__) -# model_type values that use dynamic MXFP4 re-quantization for -# OCP MX fp4 Quark checkpoints -_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"}) - class QuantKeyMatch(NamedTuple): matches: bool @@ -119,37 +116,6 @@ class QuarkConfig(QuantizationConfig): self.kv_cache_group = kv_cache_group self.kv_cache_config = kv_cache_config self.pack_method = pack_method - # Note : this flag is kept disabled because the overhead of - # dynamic mxfp4 quantization negates the performance gains - # that come from shifting to mxfp4. It is left here in case - # we want to re-enable it in the future. - self.dynamic_mxfp4_quant = False - - def maybe_update_config( - self, - model_name: str, - hf_config: PretrainedConfig | None = None, - revision: str | None = None, - ): - """Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints.""" - if hf_config is None: - return - - if ( - getattr(hf_config, "model_type", None) - not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES - ): - return - - quant_config = getattr(hf_config, "quantization_config", None) - if isinstance(quant_config, dict): - quant_dtype = ( - quant_config.get("global_quant_config", {}) - .get("weight", {}) - .get("dtype") - ) - if quant_dtype == "fp4": - self.dynamic_mxfp4_quant = True def get_linear_method(self) -> "QuarkLinearMethod": return QuarkLinearMethod(self) @@ -194,6 +160,20 @@ class QuarkConfig(QuantizationConfig): self.quant_config = quant_config_with_hf_to_vllm_mapper + def _scale_block_rows(self, prefix: str, layer_type: type[torch.nn.Module]) -> int: + """How many weight rows one checkpoint scale row covers. + + 1 for the canonical 1-D per-group MX spelling; ``block_size[0]`` for + MXFP8 exports that spell the scale 2-D (DeepSeek-V4.1 uses 32). The + scheme cannot work this out itself: it has no handle on the layer name, + and tensor-parallel sharding narrows the loaded scale so the ratio is + not recoverable from shapes either. + """ + config = self._find_matched_config(prefix, layer_type) + weight = self._unwrap_single_quant_config(config.get("weight")) + normalized = ocp_mx_weight_dtype_and_rows(weight) + return normalized[1] if normalized is not None else 1 + def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": @@ -201,17 +181,6 @@ class QuarkConfig(QuantizationConfig): self.get_quant_method_target(prefix, type(layer)) ) - exclude_layers = cast(list[str], self.quant_config.get("exclude")) - is_ignored = should_ignore_layer( - prefix, - ignore=exclude_layers, - fused_mapping=self.packed_modules_mapping, - check_children=isinstance(layer, RoutedExperts), - ) - dynamic_mxfp4_quant = ( - is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant - ) - if method_cls is UnquantizedFusedMoEMethod: return UnquantizedFusedMoEMethod(layer.moe_config) if method_cls is UnquantizedLinearMethod: @@ -222,7 +191,11 @@ class QuarkConfig(QuantizationConfig): scheme_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, - dynamic_mxfp4_quant=dynamic_mxfp4_quant, + scale_block_rows=( + self._scale_block_rows(prefix, type(layer)) + if scheme_cls is QuarkOCP_MX + else 1 + ), weight_config=self._find_matched_config(prefix, type(layer)).get( "weight" ), @@ -274,11 +247,6 @@ class QuarkConfig(QuantizationConfig): if is_routed_experts: return None, None, UnquantizedFusedMoEMethod if issubclass(layer_type, LinearBase): - if "self_attn" in prefix and self.dynamic_mxfp4_quant: - weight_quant_key, activation_key, _ = self.get_scheme_cls( - layer_type, prefix - ) - return weight_quant_key, activation_key, QuarkLinearMethod return None, None, UnquantizedLinearMethod return None, None, None if issubclass(layer_type, LinearBase): @@ -693,52 +661,18 @@ class QuarkConfig(QuantizationConfig): The rationale for checking only the weight type is that the model loading concept and process primarily concerns the weights themselves. """ - # Confirm weights quantized. - if not isinstance(weight_quant, dict): + # Confirm weights quantized, and normalize the two OCP MX config + # dialects (1-D per_group, and MXFP8's 2-D per_block) to a dtype. + normalized = ocp_mx_weight_dtype_and_rows(weight_quant) + if normalized is None: logger.debug( - "Quark model's weight quantization is incompatible with OCP_MX format: " - "weight_quant is not a dictionary." + "Quark model's weight quantization is incompatible with OCP MX " + "format: %s", + weight_quant, ) return QuantKeyMatch(False, None, None) - # Input and weight qscheme needs to be per group. - if weight_quant.get("qscheme") != "per_group": - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "weight is not per_group." - ) - return QuantKeyMatch(False, None, None) - - # Input and weight group size needs to be 32. - if weight_quant.get("group_size") != 32: - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "group_size of weight is not 32." - ) - return QuantKeyMatch(False, None, None) - - # Activations and weight scales need to be in e8m0 format. - if weight_quant.get("scale_format") != "e8m0": - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "scale_format of weight is not e8m0." - ) - return QuantKeyMatch(False, None, None) - - # Input and weight dtypes need to be any of fp4, - # fp6_e3m2 or fp6_e3m2, possibly mixed. - if weight_quant.get("dtype") not in { - "fp4", - "fp6_e3m2", - "fp6_e2m3", - }: - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "dtype is not in {fp4, fp6_e3m2, fp6_e2m3}." - ) - return QuantKeyMatch(False, None, None) - - weight_dtype = weight_quant["dtype"].replace("fp", "mxfp") + weight_dtype, _ = normalized weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype] if input_quant is None: activation_quant_key = None @@ -756,6 +690,14 @@ class QuarkConfig(QuantizationConfig): "input scales." ) return QuantKeyMatch(False, None, None) + elif ( + input_quant["dtype"] == "fp8_e4m3" + and input_quant.get("qscheme") == "per_group" + and input_quant.get("group_size") == OCP_MX_BLOCK_SIZE + ): + # MXFP8 activations. Must be checked before the per-tensor FP8 + # branch below, which would otherwise claim them. + activation_quant_key = _ACTIVATION_QUANT_KEY_MAP["mxfp8_e4m3"] elif input_quant["dtype"] == "fp8_e4m3": activation_quant_key = kFp8DynamicTensorSym else: @@ -956,7 +898,7 @@ class QuarkConfig(QuantizationConfig): scheme_cls: type["QuarkScheme"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, + scale_block_rows: int = 1, weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" @@ -997,7 +939,7 @@ class QuarkConfig(QuantizationConfig): kwargs["weight_quant_key"] = weight_quant_key if scheme_cls is QuarkOCP_MX: - kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant + kwargs["scale_block_rows"] = scale_block_rows scheme = scheme_cls(**kwargs) diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 3a9f6d6366..8da0aba97c 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -976,7 +976,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( hidden_states=x, @@ -1587,7 +1587,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): raise ValueError( f"Unsupported OCP MX dtype combination for MoE: " f"input_dtype={self.input_dtype}, weight_dtype={self.weight_dtype}. " - f"Please check that the combination is supported in OCP_MX_Scheme." + f"MXFP8 experts are not supported through the MoE path; only " + f"the linear path handles MXFP8. Supported MoE weight dtypes " + f"are mxfp4, mxfp6_e3m2, and mxfp6_e2m3." ) # TODO(bowenbao): refactor and introduce backends for other OCP MX schemes, @@ -1676,10 +1678,16 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): if quant_dtype == "mxfp4": assert dim % 2 == 0 return dim // 2 - else: + elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}: # FP6 packs 4 * 6 = 24 bits on 3 bytes. assert (dim * 3) % 4 == 0 return (dim * 3) // 4 + else: + raise NotImplementedError( + f"Unsupported quant_dtype in QuarkOCP_MX_MoEMethod." + f"get_packed_dim: {quant_dtype}. MXFP8 is not supported for " + f"MoE experts." + ) def create_weights( self, @@ -1935,7 +1943,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( hidden_states=x, @@ -2182,7 +2190,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: + ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.moe_kernel is not None return self.moe_kernel.apply( x, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 30758782c3..d12dd62e01 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -10,26 +10,33 @@ from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MxFp4LinearKernel, MxFp6LinearKernel, + Mxfp8LinearKernel, init_mxfp4_linear_kernel, init_mxfp6_linear_kernel, + init_mxfp8_linear_kernel, +) +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_SCALE_DTYPE, + MXFP8_VALUE_DTYPE, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( _ACTIVATION_QUANT_KEY_MAP, _WEIGHT_QUANT_KEY_MAP, OCP_MX_BLOCK_SIZE, + OCP_MX_UNPACKED_DTYPES, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kMxfp4Static, kMxfp6E2M3Static, kMxfp6E3M2Static, + kMxfp8Static, ) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, ModelWeightParameter, PackedvLLMParameter, ) -from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from .quark_scheme import QuarkScheme @@ -38,7 +45,7 @@ logger = init_logger(__name__) class QuarkOCP_MX(QuarkScheme): - ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel + ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None] supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()] @@ -46,10 +53,10 @@ class QuarkOCP_MX(QuarkScheme): self, weight_quant_key: QuantKey, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, + scale_block_rows: int = 1, ): super().__init__(weight_quant_key, activation_quant_key) - self.dynamic_mxfp4_quant = dynamic_mxfp4_quant + self.scale_block_rows = scale_block_rows self.weight_dtype = next( dtype for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() @@ -65,11 +72,21 @@ class QuarkOCP_MX(QuarkScheme): else None ) - if self.weight_dtype == "mxfp4": - self.packed_factor: int | Fraction = 2 + self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES + + if self.is_unpacked: + self.packed_factor: int | Fraction = 1 + elif self.weight_dtype == "mxfp4": + self.packed_factor = 2 else: self.packed_factor = Fraction(numerator=8, denominator=6) + if self.is_unpacked: + # MXFP8 runs on the shared kernel abstraction, which picks a native + # backend where one exists and emulates otherwise. The MXFP4/MXFP6 + # warnings below do not apply. + return + if not current_platform.supports_mx(): logger.warning_once( "The current platform does not support native MXFP4/MXFP6 " @@ -91,7 +108,9 @@ class QuarkOCP_MX(QuarkScheme): ) def get_packed_dim(self, dim: int, quant_dtype: str): - if quant_dtype == "mxfp4": + if quant_dtype in OCP_MX_UNPACKED_DTYPES: + return dim + elif quant_dtype == "mxfp4": assert dim % 2 == 0 return dim // 2 elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}: @@ -105,25 +124,33 @@ class QuarkOCP_MX(QuarkScheme): "open an issue." ) + def _expanding_scale_loader(self, weight_loader: Callable) -> Callable: + """Expand a 2-D block scale to one row per weight row. + + A ``block_size=[R, 32]`` export carries one e8m0 value per R weight + rows; the kernels want one per row. At ``R == 1`` the layouts already + coincide, so the loader is passed through untouched. + """ + if self.scale_block_rows == 1: + return weight_loader + + rows = self.scale_block_rows + + def expanding_loader(param, loaded_weight, *args, **kwargs): + loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave( + rows, dim=0 + ) + return weight_loader(param, loaded_weight, *args, **kwargs) + + return expanding_loader + @classmethod def get_min_capability(cls) -> int: return 70 - def process_dynamic_mxfp4_weights_after_loading( - self, layer: torch.nn.Module - ) -> None: - from aiter.ops.triton.quant import dynamic_mxfp4_quant - - w_q, w_s = dynamic_mxfp4_quant(layer.weight) - layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False) - layer.weight = torch.nn.Parameter(w_q, requires_grad=False) - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False) - if self.dynamic_mxfp4_quant: - self.process_dynamic_mxfp4_weights_after_loading(layer) - self.ocp_mx_linear.process_weights_after_loading(layer) def create_weights( @@ -145,20 +172,37 @@ class QuarkOCP_MX(QuarkScheme): "tensor-parallel sharding for this layer." ) - if self.dynamic_mxfp4_quant: + if self.is_unpacked: + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.params_dtype = params_dtype + weight = ModelWeightParameter( data=torch.empty( - sum(output_partition_sizes), + output_size_per_partition, input_size_per_partition, - dtype=params_dtype, + dtype=MXFP8_VALUE_DTYPE, ), input_dim=1, output_dim=0, weight_loader=weight_loader, ) - layer.register_parameter("weight", weight) - set_weight_attrs(weight, kwargs) + + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=MXFP8_SCALE_DTYPE, + ), + input_dim=1, + output_dim=0, + weight_loader=self._expanding_scale_loader(weight_loader), + ) + layer.register_parameter("weight_scale", weight_scale) + layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE] else: output_size_per_partition = sum(output_partition_sizes) layer.logical_widths = output_partition_sizes @@ -200,6 +244,16 @@ class QuarkOCP_MX(QuarkScheme): weight_quant_key=self.weight_quant_key, activation_quant_key=self.activation_quant_key, ) + elif self.weight_quant_key == kMxfp8Static: + self.ocp_mx_linear = init_mxfp8_linear_kernel( + weight_shape=layer.weight.shape + ) + else: + raise NotImplementedError( + "No OCP MX linear kernel for weight_quant_key=" + f"{self.weight_quant_key}. Something is wrong, please open an " + "issue." + ) def apply_weights( self, diff --git a/vllm/model_executor/layers/quantization/utils/allspark_utils.py b/vllm/model_executor/layers/quantization/utils/allspark_utils.py deleted file mode 100644 index 4c324682e5..0000000000 --- a/vllm/model_executor/layers/quantization/utils/allspark_utils.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.platforms import current_platform -from vllm.scalar_type import ScalarType, scalar_types - -ALLSPARK_AMPERE_M_CUBLAS_THRESHOLD = 1024 -ALLSPARK_SUPPORTED_QUANT_TYPES = [scalar_types.uint8b128] -ALLSPARK_AMPERE_N_ALIGN = 16 -ALLSPARK_AMPERE_K_ALIGN = 16 - - -def check_allspark_supported_dtype_shape( - input_size_per_partition: int, - output_size_per_partition: int, - group_size: int, - weight_dtype: ScalarType, - act_dtype: torch.dtype, -): - capability_tuple = current_platform.get_device_capability() - device_capability = -1 if capability_tuple is None else capability_tuple.to_int() - - # For Ampere GPU - if device_capability >= 80 and device_capability < 90: - if group_size != -1: - return ( - False, - "For Ampere GPU, AllSpark does not support group_size " - f"= {group_size}. Only group_size = -1 are supported.", - ) - - if weight_dtype not in ALLSPARK_SUPPORTED_QUANT_TYPES: - return ( - False, - "For Ampere GPU, AllSpark does not support " - f"quant type ({weight_dtype}). Only quant type " - f"({ALLSPARK_SUPPORTED_QUANT_TYPES}) are supported.", - ) - - if ( - input_size_per_partition % ALLSPARK_AMPERE_K_ALIGN != 0 - or output_size_per_partition % ALLSPARK_AMPERE_N_ALIGN != 0 - ): - return ( - False, - "AllSpark needs input_size_per_partition % " - f"{ALLSPARK_AMPERE_K_ALIGN} = 0 and " - f"output_size_per_partition % {ALLSPARK_AMPERE_N_ALIGN} = 0 " - "for Ampere GPU optimized kernels.", - ) - - if act_dtype != torch.float16 and act_dtype != torch.bfloat16: - return ( - False, - "AllSpark only supports act_dtype = float16 or bfloat16," - f"for Ampere GPU, but got act_dtype = {act_dtype}.", - ) - else: - return ( - False, - "AllSpark currently does not support " - f"device_capability = {device_capability}.", - ) - - return True, None diff --git a/vllm/model_executor/layers/quantization/utils/config_utils.py b/vllm/model_executor/layers/quantization/utils/config_utils.py index 1beb988ac2..eb50ed1a23 100644 --- a/vllm/model_executor/layers/quantization/utils/config_utils.py +++ b/vllm/model_executor/layers/quantization/utils/config_utils.py @@ -44,7 +44,12 @@ def get_quark_ocp_mx_group_size( return None assert weight_quant is not None - return int(weight_quant["group_size"]) + group_size = weight_quant.get("group_size") + if group_size is None: + block_size = weight_quant.get("block_size") + if isinstance(block_size, list | tuple) and len(block_size) >= 2: + group_size = block_size[1] + return int(group_size) if group_size is not None else None def is_shared_expert_quant_fse_compatible( diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index 79859a69d4..c9888dc3cb 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -1373,7 +1373,7 @@ def process_fp8_weight_tensor_strategy( logical_widths: list[int], input_scale: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: - """Process weights for tensor-wise quantization strategy.""" + """Requantize fused shards to one scale and return ``(K, N)`` weight.""" from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( normalize_e4m3fn_to_e4m3fnuz, requantize_with_max_scale, @@ -1391,7 +1391,7 @@ def process_fp8_weight_tensor_strategy( logical_widths=logical_widths, ) - weight = _maybe_pad_fp8_weight(weight) + weight = _maybe_pad_fp8_weight(weight).t() return weight, weight_scale, input_scale @@ -1400,7 +1400,7 @@ def process_fp8_weight_channel_strategy( weight_scale: torch.Tensor, input_scale: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: - """Process weights for channel-wise quantization strategy.""" + """Normalize FNUZ if needed and return ``(K, N)`` weight.""" from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( normalize_e4m3fn_to_e4m3fnuz, ) @@ -1410,14 +1410,14 @@ def process_fp8_weight_channel_strategy( weight=weight, weight_scale=weight_scale, input_scale=input_scale ) - return weight, weight_scale, input_scale + return weight.t(), weight_scale, input_scale def process_fp8_weight_block_strategy( weight: torch.Tensor, weight_scale: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """Process weights for block-wise quantization strategy.""" + """Normalize FNUZ if needed and return ``(N, K)`` weight (no transpose).""" from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( normalize_e4m3fn_to_e4m3fnuz, ) diff --git a/vllm/model_executor/layers/quantization/utils/humming/linear.py b/vllm/model_executor/layers/quantization/utils/humming/linear.py index 339ff640b0..0c64645c43 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/linear.py +++ b/vllm/model_executor/layers/quantization/utils/humming/linear.py @@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config( param_dtype=layer.params_dtype, ) + if ( + weight_schema.has_zero_point + and input_schema.a_dtype is not None + and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits + ): + raise ValueError( + "Humming does not support asymmetric (zero_point) weights " + "when weight and activation bit widths are equal " + f"(W{weight_schema.b_dtype.num_bits}" + f"A{input_schema.a_dtype.num_bits}). " + "Use symmetric weights or a different activation bit width." + ) + tensors.update(input_tensors) layer.weight_schema = weight_schema input_schema = check_and_fallback_input_schema( diff --git a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py index af47ea2213..6fa93c0a2b 100644 --- a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py +++ b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum +from typing import Any from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( @@ -11,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kMxfp6E2M3Static, kMxfp6E3M2Dynamic, kMxfp6E3M2Static, + kMxfp8Dynamic, + kMxfp8Static, ) logger = init_logger(__name__) @@ -21,14 +24,19 @@ _WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = { "mxfp4": kMxfp4Static, "mxfp6_e3m2": kMxfp6E3M2Static, "mxfp6_e2m3": kMxfp6E2M3Static, + "mxfp8_e4m3": kMxfp8Static, } _ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = { "mxfp4": kMxfp4Dynamic, "mxfp6_e3m2": kMxfp6E3M2Dynamic, "mxfp6_e2m3": kMxfp6E2M3Dynamic, + "mxfp8_e4m3": kMxfp8Dynamic, } +# Unpacked OCP MX dtypes: one byte per element, no `packed_factor`. +OCP_MX_UNPACKED_DTYPES = {"mxfp8_e4m3"} + OCP_MX_DTYPES = { "mxfp4", "mxfp6_e3m2", @@ -39,6 +47,56 @@ OCP_MX_DTYPES = { } +def ocp_mx_weight_dtype_and_rows( + weight_quant: dict[str, Any] | None, +) -> tuple[str, int] | None: + """Normalize a Quark weight quant config to ``(mx_dtype, scale_block_rows)``. + + Quark spells OCP MX weights two ways. The canonical one is 1-D per-group: + ``qscheme="per_group"``, ``group_size=32``, ``scale_format="e8m0"``. MXFP8 + checkpoints may instead use a 2-D per-block spelling + (``qscheme="per_block"``, ``block_size=[R, 32]``, + ``scale_type="float8_e8m0fnu"``), which carries one scale per ``R`` weight + rows rather than one per row. ``scale_block_rows`` is that ``R``; it is 1 + for the canonical spelling, where the two layouts coincide. + + Returns ``None`` when the config is not an OCP MX weight quantization. + """ + if not isinstance(weight_quant, dict): + return None + + dtype = weight_quant.get("dtype") + if not isinstance(dtype, str): + return None + mx_dtype = dtype.replace("fp", "mxfp") + if mx_dtype not in _WEIGHT_QUANT_KEY_MAP: + return None + + qscheme = weight_quant.get("qscheme") + if qscheme == "per_group": + if weight_quant.get("group_size") != OCP_MX_BLOCK_SIZE: + return None + if weight_quant.get("scale_format") != "e8m0": + return None + return mx_dtype, 1 + + if qscheme == "per_block": + # Only the unpacked dtypes are accepted here: expanding a 2-D scale + # over a sub-byte packed weight has no checkpoint to validate against. + if mx_dtype not in OCP_MX_UNPACKED_DTYPES: + return None + block_size = list(weight_quant.get("block_size") or []) + if len(block_size) != 2 or block_size[1] != OCP_MX_BLOCK_SIZE: + return None + if weight_quant.get("scale_type") != "float8_e8m0fnu": + return None + if weight_quant.get("symmetric") is not True or weight_quant.get("is_dynamic"): + return None + return mx_dtype, block_size[0] + + return None + + class OCP_MX_Scheme(str, Enum): w_mxfp4 = "w_mxfp4" w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4" diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index f3f72adc78..4c0a8a3ff7 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -208,6 +208,9 @@ kNvfp4DynamicGroupScale = ScaleDesc(FP8_DTYPE, False, GroupShape(1, 16)) kNvfp4Dynamic = QuantKey( FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kStaticTensorScale ) +kNvfp4DynamicToken = QuantKey( + FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kDynamicTokenScale +) kNvfp4StaticGroupScale = ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16)) kNvfp4Static = QuantKey( diff --git a/vllm/model_executor/layers/rotary_embedding/common.py b/vllm/model_executor/layers/rotary_embedding/common.py index 24c96c2521..b96ab9cb66 100644 --- a/vllm/model_executor/layers/rotary_embedding/common.py +++ b/vllm/model_executor/layers/rotary_embedding/common.py @@ -237,6 +237,18 @@ class ApplyRotaryEmb(CustomOp): output = self._post_process(output, origin_shape, origin_dtype) return output + def forward_xpu( + self, + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> torch.Tensor: + import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_apply_rotary_emb + + x, cos, sin, origin_shape, origin_dtype = self._pre_process(x, cos, sin) + output = torch.ops.vllm.xpu_apply_rotary_emb(x, cos, sin, self.is_neox_style) + return self._post_process(output, origin_shape, origin_dtype) + def forward_hip( self, x: torch.Tensor, diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 185bfd650c..11882856e0 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -7,6 +7,7 @@ import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op from .base import RotaryEmbeddingBase from .yarn_scaling_rope import YaRNScalingRotaryEmbedding, yarn_get_mscale @@ -19,6 +20,8 @@ def _triton_mrope_forward( cos, sin, num_tokens, + q_token_stride, + k_token_stride, n_qh: tl.constexpr, n_kh: tl.constexpr, hd: tl.constexpr, @@ -38,9 +41,10 @@ def _triton_mrope_forward( # and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2) # instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary pid = tl.program_id(0) - # locate start address - q_ptr = q_ptr + pid * (n_qh * hd) - k_ptr = k_ptr + pid * (n_kh * hd) + # locate start address (int64: num_tokens * row_stride exceeds 2^31 for + # worst-case ViT forwards, e.g. MiniMax M3 packs ~768k patches/item) + q_ptr = q_ptr + pid.to(tl.int64) * q_token_stride + k_ptr = k_ptr + pid.to(tl.int64) * k_token_stride # #################################################################### # get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position @@ -185,6 +189,10 @@ def triton_mrope( adjacent (GPT-J) layout. """ + # The kernel indexes each token row through an explicit row stride and + # requires dense rows. + assert q.dim() == 2 and q.stride(-1) == 1 + assert k.dim() == 2 and k.stride(-1) == 1 n_row, n_q_head_head_dim = q.shape n_q_head = n_q_head_head_dim // head_size n_kv_head = k.shape[1] // head_size @@ -192,10 +200,6 @@ def triton_mrope( pad_n_q_head = triton.next_power_of_2(n_q_head) pad_n_kv_head = triton.next_power_of_2(n_kv_head) - # ensure tensors passed into the kernel are contiguous. - # It will be no-op if they are already contiguous - q = q.contiguous() - k = k.contiguous() cos = cos.contiguous() sin = sin.contiguous() @@ -210,6 +214,8 @@ def triton_mrope( cos, sin, n_row, + q.stride(0), + k.stride(0), n_q_head, n_kv_head, head_size, @@ -240,6 +246,62 @@ def apply_interleaved_rope(x: torch.Tensor, mrope_section: list[int]) -> torch.T return torch.where(is_width, x[2], result) +def _mrope_apply( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + head_size: int, + rotary_dim: int, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + mrope_interleaved: bool, + is_neox_style: bool, +) -> None: + cos_sin = cos_sin_cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + query_out, key_out = triton_mrope( + query, + key, + cos, + sin, + [mrope_section_t, mrope_section_h, mrope_section_w], + head_size, + rotary_dim, + mrope_interleaved, + is_neox_style, + ) + if query_out.data_ptr() != query.data_ptr(): + query.copy_(query_out.view_as(query)) + if key_out.data_ptr() != key.data_ptr(): + key.copy_(key_out.view_as(key)) + + +def _mrope_apply_fake( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + head_size: int, + rotary_dim: int, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + mrope_interleaved: bool, + is_neox_style: bool, +) -> None: + return + + +direct_register_custom_op( + op_name="mrope", + op_func=_mrope_apply, + mutates_args=["query", "key"], + fake_impl=_mrope_apply_fake, +) + + class MRotaryEmbedding(RotaryEmbeddingBase): """Rotary Embedding with Multimodal Sections.""" @@ -426,6 +488,35 @@ class MRotaryEmbedding(RotaryEmbeddingBase): key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) return query, key + def forward_hip( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor | None = None, + offsets: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if positions.ndim == 2: + assert key is not None + assert self.mrope_section + query_shape = query.shape + key_shape = key.shape + cos_sin_cache = self._match_cos_sin_cache_dtype(query) + torch.ops.vllm.mrope( + positions, + query, + key, + cos_sin_cache, + self.head_size, + self.rotary_dim, + self.mrope_section[0], + self.mrope_section[1], + self.mrope_section[2], + self.mrope_interleaved, + self.is_neox_style, + ) + return query.reshape(query_shape), key.reshape(key_shape) + return super().forward_hip(positions, query, key) + def forward_cpu( self, positions: torch.Tensor, diff --git a/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py b/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py new file mode 100644 index 0000000000..6453e8c835 --- /dev/null +++ b/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import async_tensor_h2d + + +# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr +# is an offset slice of the packed H2D buffer, so its 16B alignment varies +# with grid count; without these the kernel recompiles mid-serving. +@triton.jit( + do_not_specialize=["num_tokens", "num_grids"], + do_not_specialize_on_alignment=["grids_ptr"], +) +def _vit_mrope_setup_kernel( + inv_freq_t_ptr, + inv_freq_h_ptr, + inv_freq_w_ptr, + cos_out_ptr, + sin_out_ptr, + cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid + grids_ptr, # (G, 3) int32: t, h, w per grid + num_tokens, + num_grids, + half_t: tl.constexpr, + half_h: tl.constexpr, + half_w: tl.constexpr, + spatial_merge: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one + launch: compute each token's (t, h, w) position arithmetically and the + rotation in fp32 from the per-axis inv_freq. Only each axis's mrope + section columns are written; the rest is left uninitialized (the + consumer's per-axis masked loads read only the section columns).""" + pid = tl.program_id(0) + tok = pid * BLOCK_N + tl.arange(0, BLOCK_N) + tok_mask = tok < num_tokens + + # Grid (segment) index per token; cu_seqlens is nondecreasing. + seg = tl.zeros([BLOCK_N], dtype=tl.int32) + for g in range(num_grids): + end = tl.load(cu_seqlens_ptr + g + 1) + seg += (tok >= end).to(tl.int32) + + seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0) + grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1) + grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1) + + local = tok - seg_start + rem = local % (grid_h * grid_w) + tpos = local // (grid_h * grid_w) + # Within a frame tokens are ordered by spatial-merge block (bh, bw, mh, + # mw), matching the reference ViT position-id permute. + bh = rem // (spatial_merge * grid_w) + r = rem % (spatial_merge * grid_w) + bw = r // (spatial_merge * spatial_merge) + r2 = r % (spatial_merge * spatial_merge) + hpos = bh * spatial_merge + r2 // spatial_merge + wpos = bw * spatial_merge + r2 % spatial_merge + + d = tl.arange(0, BLOCK_D) + half_rot = half_t + half_h + half_w + row = (tok * half_rot)[:, None] + d[None, :] + plane = num_tokens * half_rot + + # Per-axis compute+store to cap live registers; the fp32 trig sequence + # wants few elements per thread (hence small BLOCK_N, many warps). + in_t = d < half_t + inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0) + freq = tpos.to(tl.float32)[:, None] * inv_t[None, :] + mask = tok_mask[:, None] & in_t[None, :] + tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask) + + in_h = (d >= half_t) & (d < half_t + half_h) + inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0) + freq = hpos.to(tl.float32)[:, None] * inv_h[None, :] + mask = tok_mask[:, None] & in_h[None, :] + tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask) + + in_w = (d >= half_t + half_h) & (d < half_rot) + inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0) + freq = wpos.to(tl.float32)[:, None] * inv_w[None, :] + mask = tok_mask[:, None] & in_w[None, :] + tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask) + + +def vit_mrope_setup( + inv_freq_t: torch.Tensor, + inv_freq_h: torch.Tensor, + inv_freq_w: torch.Tensor, + grid_thw: list[list[int]], + spatial_merge_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope. + + One fused kernel: positions are computed arithmetically from the grids + and the rotation comes from the per-axis inv_freq, so there is no + cos/sin cache to size or grow. + + Args: + inv_freq_t: t-axis inverse frequencies, (t_dim // 2,). + inv_freq_h: h-axis inverse frequencies, (h_dim // 2,). + inv_freq_w: w-axis inverse frequencies, (w_dim // 2,). + grid_thw: per-grid [t, h, w] patch counts (host metadata). + spatial_merge_size: spatial merge factor of the ViT. + + """ + device = inv_freq_t.device + cu = [0] + for t, h, w in grid_thw: + cu.append(cu[-1] + t * h * w) + n = cu[-1] + g = len(grid_thw) + # grid_thw is per-request host metadata, so one H2D is unavoidable; + # pack cu_seqlens and grids into a single pinned async copy. + buf = async_tensor_h2d( + cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32 + ) + cu_seqlens = buf[: g + 1] + grids = buf[g + 1 :].view(g, 3) + + half_t = inv_freq_t.numel() + half_h = inv_freq_h.numel() + half_w = inv_freq_w.numel() + half_rot = half_t + half_h + half_w + cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32) + sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32) + _vit_mrope_setup_kernel[(triton.cdiv(n, 32),)]( + inv_freq_t, + inv_freq_h, + inv_freq_w, + cos, + sin, + cu_seqlens, + grids, + n, + g, + half_t=half_t, + half_h=half_h, + half_w=half_w, + spatial_merge=spatial_merge_size, + BLOCK_N=32, + BLOCK_D=triton.next_power_of_2(half_rot), + num_warps=8, + ) + return cos, sin diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index a1b79a59a4..aec4ebdd44 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -346,12 +346,14 @@ def rocm_unquantized_gemm_impl( if use_skinny: # The skinny kernels assume contiguous K elements. A shape-preserving # reshape can retain a transposed activation's non-contiguous strides. - x_view = x.reshape(-1, x.size(-1)).contiguous() - if m > 8 and 0 < n <= 5: + # Note: Only build that view inside the branches that consume it. + if (m == 1 or m > 8) and 0 < n <= 5: + x_view = x.reshape(-1, x.size(-1)).contiguous() cu_count = num_compute_units() out = ops.wvSplitK(weight, x_view, cu_count, bias) return out.reshape(*x.shape[:-1], weight.shape[0]) elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None: + x_view = x.reshape(-1, x.size(-1)).contiguous() out = ops.LLMM1(weight, x_view, 4) return out.reshape(*x.shape[:-1], weight.shape[0]) @@ -513,7 +515,10 @@ def dispatch_cpu_unquantized_gemm( ) ) if remove_weight: - layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) + layer.weight = torch.nn.Parameter( + torch.empty(0, dtype=dtype, device=layer.weight.device), + requires_grad=False, + ) logger.debug_once( "CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)", is_prepacked, @@ -539,7 +544,10 @@ def dispatch_cpu_unquantized_gemm( bias, ) if remove_weight: - layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) + layer.weight = torch.nn.Parameter( + torch.empty(0, dtype=dtype, device=layer.weight.device), + requires_grad=False, + ) logger.debug_once( "CPU unquantized GEMM dispatch: using sgl-kernel weight_packed_linear" ) @@ -554,7 +562,10 @@ def dispatch_cpu_unquantized_gemm( handler = ops.create_onednn_mm(origin_weight.t(), 32) layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(handler, x, bias) if remove_weight: - layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) + layer.weight = torch.nn.Parameter( + torch.empty(0, dtype=dtype, device=layer.weight.device), + requires_grad=False, + ) logger.debug_once("CPU unquantized GEMM dispatch: using oneDNN onednn_mm") return except RuntimeError as e: diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 72cafceb25..be20c41243 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -278,12 +278,12 @@ class VocabParallelEmbedding(PluggableLayer): self.num_embeddings = num_embeddings self.padding_size = padding_size self.org_vocab_size = org_num_embeddings or num_embeddings - num_added_embeddings = num_embeddings - self.org_vocab_size + self.num_added_embeddings = num_embeddings - self.org_vocab_size self.org_vocab_size_padded = pad_vocab_size( self.org_vocab_size, self.padding_size ) self.num_embeddings_padded = pad_vocab_size( - self.org_vocab_size_padded + num_added_embeddings, self.padding_size + self.org_vocab_size_padded + self.num_added_embeddings, self.padding_size ) assert self.org_vocab_size_padded <= self.num_embeddings_padded @@ -296,6 +296,18 @@ class VocabParallelEmbedding(PluggableLayer): self.tp_size, ) self.embedding_dim = embedding_dim + # Divide the weight matrix along the vocabulary dimension. + self.num_embeddings_per_partition = divide( + self.num_embeddings_padded, self.tp_size + ) + + # Quantization methods share the same weight factory as linear layers, + # so setup standard linear metadata. + self.input_size = self.input_size_per_partition = embedding_dim + self.output_size = self.num_embeddings_padded + self.output_size_per_partition = self.num_embeddings_per_partition + self.output_partition_sizes = [self.output_size_per_partition] + self.prefix = prefix # Avoid overriding a preselected model-specific method with generic # config-based dispatch. @@ -322,11 +334,6 @@ class VocabParallelEmbedding(PluggableLayer): if params_dtype is None: params_dtype = torch.get_default_dtype() self.params_dtype = params_dtype - # Divide the weight matrix along the vocabulary dimension. - self.num_added_embeddings = self.num_embeddings - self.org_vocab_size - self.num_embeddings_per_partition = divide( - self.num_embeddings_padded, self.tp_size - ) assert ( self.shard_indices.num_elements_padded == self.num_embeddings_per_partition ) @@ -349,10 +356,10 @@ class VocabParallelEmbedding(PluggableLayer): self.quant_method.create_weights( self, - self.embedding_dim, - [self.num_embeddings_per_partition], - self.embedding_dim, - self.num_embeddings_padded, + self.input_size_per_partition, + self.output_partition_sizes, + self.input_size, + self.output_size, params_dtype=params_dtype, weight_loader=self.weight_loader, ) @@ -600,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding): *, disable_tp: bool = False, ): + self.has_bias = bias super().__init__( num_embeddings, embedding_dim, diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index 039a577f80..c3e09bccd4 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -1,11 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + import dataclasses import glob import os import time from collections.abc import Generator, Iterable -from typing import cast +from typing import TYPE_CHECKING, cast import torch from torch import nn @@ -25,6 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import ( fastsafetensors_weights_iterator, filter_duplicate_safetensors_files, filter_files_not_needed_for_inference, + filter_mm_encoder_only_safetensors_files, get_quant_config, instanttensor_weights_iterator, maybe_download_from_modelscope, @@ -32,11 +35,15 @@ from vllm.model_executor.model_loader.weight_utils import ( multi_thread_safetensors_weights_iterator, np_cache_weights_iterator, pt_weights_iterator, + resolve_mm_encoder_only_lm_prefixes, safetensors_weights_iterator, ) from vllm.tracing import instrument from vllm.transformers_utils.repo_utils import list_filtered_repo_files +if TYPE_CHECKING: + from vllm.model_executor.models.utils import WeightsMapper + logger = init_logger(__name__) @@ -74,6 +81,9 @@ class DefaultModelLoader(BaseModelLoader): def __init__(self, load_config: LoadConfig): super().__init__(load_config) self.local_expert_ids: set[int] | None = None + # Set in load_weights when --mm-encoder-only; used to drop LM-only shards. + self._encoder_only_lm_prefixes: tuple[str, ...] | None = None + self._encoder_only_weights_mapper: WeightsMapper | None = None extra_config = load_config.model_loader_extra_config if not isinstance(extra_config, dict): @@ -132,7 +142,7 @@ class DefaultModelLoader(BaseModelLoader): revision: str | None, fall_back_to_pt: bool, allow_patterns_overrides: list[str] | None, - ) -> tuple[str, list[str], bool]: + ) -> tuple[str, list[str], bool, str]: """Prepare weights for the model. If the model is not local, it will be downloaded.""" @@ -239,20 +249,40 @@ class DefaultModelLoader(BaseModelLoader): f"Cannot find any model weights with `{model_name_or_path}`" ) - return hf_folder, hf_weights_files, use_safetensors + return hf_folder, hf_weights_files, use_safetensors, index_file def _get_weights_iterator( - self, source: "Source" + self, source: Source ) -> Generator[tuple[str, torch.Tensor], None, None]: """Get an iterator for the model weights based on the load format.""" extra_config = self.load_config.model_loader_extra_config - hf_folder, hf_weights_files, use_safetensors = self._prepare_weights( - source.model_or_path, - source.subfolder, - source.revision, - source.fall_back_to_pt, - source.allow_patterns_overrides, + hf_folder, hf_weights_files, use_safetensors, index_file = ( + self._prepare_weights( + source.model_or_path, + source.subfolder, + source.revision, + source.fall_back_to_pt, + source.allow_patterns_overrides, + ) ) + if ( + self._encoder_only_lm_prefixes is not None + and use_safetensors + and hf_weights_files + ): + hf_weights_files = filter_mm_encoder_only_safetensors_files( + hf_weights_files, + hf_folder, + index_file, + self._encoder_only_lm_prefixes, + weights_mapper=self._encoder_only_weights_mapper, + ) + if not hf_weights_files: + raise RuntimeError( + "mm-encoder-only shard filter removed all weight files for " + f"`{source.model_or_path}`; check language_model prefixes " + f"{self._encoder_only_lm_prefixes}" + ) if self.load_config.load_format == "npcache": # Currently np_cache only support *.bin checkpoints assert use_safetensors is False @@ -348,6 +378,29 @@ class DefaultModelLoader(BaseModelLoader): allow_patterns_overrides=None, ) + def _init_mm_encoder_only_weight_filter( + self, model: nn.Module, model_config: ModelConfig + ) -> None: + """Skip pure language-model safetensors shards under --mm-encoder-only.""" + mm_config = model_config.multimodal_config + if mm_config is None or not mm_config.mm_encoder_only: + self._encoder_only_lm_prefixes = None + self._encoder_only_weights_mapper = None + return + + # Derive from _language_model_names; fail-closed on shared HF roots + # (Molmo/Phi-4-MM/Muse). Qwen nested/flat keys classified via mapper. + weights_mapper = cast( + "WeightsMapper | None", getattr(model, "hf_to_vllm_mapper", None) + ) + self._encoder_only_lm_prefixes = resolve_mm_encoder_only_lm_prefixes( + getattr(model, "_language_model_names", None), + weights_mapper=weights_mapper, + ) + self._encoder_only_weights_mapper = ( + weights_mapper if self._encoder_only_lm_prefixes is not None else None + ) + def _init_ep_weight_filter(self, model_config: ModelConfig) -> None: """Compute local expert ids for EP weight filtering. @@ -423,6 +476,7 @@ class DefaultModelLoader(BaseModelLoader): self.load_config.safetensors_load_strategy = "torchao" self._init_ep_weight_filter(model_config) + self._init_mm_encoder_only_weight_filter(model, model_config) loaded_weights = model.load_weights(self.get_all_weights(model_config, model)) diff --git a/vllm/model_executor/model_loader/runai_streamer_loader.py b/vllm/model_executor/model_loader/runai_streamer_loader.py index e07330ae57..2f18a2e3ac 100644 --- a/vllm/model_executor/model_loader/runai_streamer_loader.py +++ b/vllm/model_executor/model_loader/runai_streamer_loader.py @@ -15,6 +15,7 @@ from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf, runai_safetensors_weights_iterator, ) +from vllm.transformers_utils.repo_utils import resolve_revision from vllm.transformers_utils.runai_utils import is_runai_obj_uri, list_safetensors @@ -87,6 +88,11 @@ class RunaiModelStreamerLoader(BaseModelLoader): safetensors_pattern = "*.safetensors" index_file = SAFE_WEIGHTS_INDEX_NAME + if not is_local and not is_object_storage_path: + # `model_weights` can point to another repo than the one `revision` was + # resolved for, which does not pin this one. + revision = resolve_revision(model_name_or_path, revision) + hf_folder = ( model_name_or_path if (is_local or is_object_storage_path) diff --git a/vllm/model_executor/model_loader/tensorizer.py b/vllm/model_executor/model_loader/tensorizer.py index 3f865c3f39..bb48c5dc08 100644 --- a/vllm/model_executor/model_loader/tensorizer.py +++ b/vllm/model_executor/model_loader/tensorizer.py @@ -18,7 +18,7 @@ import regex as re import torch from torch import nn from torch.utils._python_dispatch import TorchDispatchMode -from transformers import PretrainedConfig +from transformers import PreTrainedConfig import vllm.envs as envs from vllm.config import ModelConfig, ParallelConfig, VllmConfig, set_current_vllm_config @@ -173,7 +173,7 @@ class TensorizerConfig(MutableMapping): deserialization_kwargs: dict[str, Any] | None = None _extra_serialization_attrs: dict[str, Any] | None = field(init=False, default=None) model_class: type[torch.nn.Module] | None = field(init=False, default=None) - hf_config: PretrainedConfig | None = field(init=False, default=None) + hf_config: PreTrainedConfig | None = field(init=False, default=None) dtype: str | torch.dtype | None = field(init=False, default=None) _is_sharded: bool = field(init=False, default=False) _fields: ClassVar[tuple[str, ...]] diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index 1eeff81f65..561f693ae7 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -9,7 +9,7 @@ reloading from disk. Launch one daemon per TP rank with a single command: - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 4 Engines then load from the daemons with: @@ -17,8 +17,8 @@ Engines then load from the daemons with: vllm serve /path/to/model --tensor-parallel-size 4 \\ --load-format ipc_cache -Only tensor and expert parallelism are supported; pipeline and data parallelism -are rejected at launch. +Tensor, expert and data parallelism are supported; pipeline parallelism is +rejected at launch. For multi-node tensor parallelism, run one launcher per node with a shared rendezvous so the global TP group forms across nodes (CUDA IPC handles are @@ -27,12 +27,12 @@ node-local, so each node serves only its local GPUs' shards). Reuse the same ``--weight-cache-master-port`` distinct from the engine's ``--master-port``: # node 0 (8 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 16 \\ --nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \\ --weight-cache-master-port 29600 # node 1 (8 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 16 \\ --nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \\ --weight-cache-master-port 29600 @@ -42,6 +42,32 @@ The global TP rank of local GPU ``i`` on node ``r`` is assignment, so each engine worker maps its shard from the daemon on its own node. +For data parallelism (e.g. a TP1 x DP16 x EP decode fleet) run one launcher per +node with the engine's DP placement flags. Local GPU ``i`` serves DP rank +``start_rank + i // tp_size`` and TP rank ``i % tp_size``, and all +``dp_size * tp_size`` daemons form one world group on +``--data-parallel-address``/``--weight-cache-master-port`` so the expert +shards are laid out exactly as in the engine: + + # node r (4 local GPUs) + vllm preload \\ + --model /path/to/model --tensor-parallel-size 1 --enable-expert-parallel \\ + --data-parallel-size 16 --data-parallel-size-local 4 \\ + --data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \\ + --weight-cache-master-port 29600 + +Data parallelism also combines with multi-node tensor parallelism: pass both +flag sets (``--nnodes``/``--node-rank``/``--master-addr`` and the DP flags with +a reachable ``--data-parallel-address``). Each node then serves a contiguous +block of the ``dp_size * tp_size`` global ranks, e.g. TP8 x DP2 on 4 nodes: + + # node r (4 local GPUs) + vllm preload \\ + --model /path/to/model --tensor-parallel-size 8 --enable-expert-parallel \\ + --nnodes 4 --node-rank r --master-addr 10.0.0.1 \\ + --data-parallel-size 2 --data-parallel-address 10.0.0.1 \\ + --weight-cache-master-port 29600 + With MTP, EAGLE or EAGLE3 speculative decoding the launcher additionally starts a draft daemon group that caches the draft model. It uses its own cache key, Unix sockets (``*_draft.sock``) and rendezvous port @@ -54,25 +80,21 @@ import contextlib import fcntl import multiprocessing import os -import queue -import signal import socket -import sys from collections.abc import Callable -from itertools import product import torch from vllm.config import ( ParallelConfig, VllmConfig, + replace, set_current_vllm_config, ) from vllm.distributed import ( ensure_model_parallel_initialized, init_distributed_environment, ) -from vllm.engine.arg_utils import EngineArgs from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model_loader from vllm.model_executor.model_loader.utils import process_weights_after_loading @@ -80,7 +102,6 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( TensorEntry, WeightCacheKey, WeightCacheUnavailableError, - check_ipc_platform_support, check_ipc_quant_support, ensure_private_socket_dir, get_current_device_uuid, @@ -95,9 +116,8 @@ from vllm.model_executor.model_loader.weight_cache.utils import ( is_draft_model_cacheable, ) from vllm.platforms import current_platform -from vllm.utils.argparse_utils import FlexibleArgumentParser -from vllm.utils.network_utils import get_distributed_init_method, get_open_port from vllm.utils.torch_utils import set_default_torch_dtype +from vllm.v1.worker.workspace import init_workspace_manager logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon") @@ -158,7 +178,25 @@ class WeightCacheDaemon: distributed_init_method: str, socket_dir: str | None = None, is_draft: bool = False, + dp_rank: int = 0, ): + parallel_config = vllm_config.parallel_config + if parallel_config.data_parallel_size > 1: + # The engine sets these on each DP rank's workers; the MoE/EP + # layers read them to place their expert shards. + tp_size = parallel_config.tensor_parallel_size + if parallel_config.nnodes > 1: + world_size = parallel_config.data_parallel_size * tp_size + local_world = world_size // parallel_config.nnodes + start_dp_rank = parallel_config.node_rank * local_world // tp_size + else: + start_dp_rank = parallel_config.data_parallel_rank + parallel_config = replace( + parallel_config, + data_parallel_rank=dp_rank, + data_parallel_rank_local=dp_rank - start_dp_rank, + ) + vllm_config = replace(vllm_config, parallel_config=parallel_config) self.vllm_config = vllm_config # A draft daemon builds the draft with the target's VllmConfig, like # the engine does; only the ModelConfig differs. @@ -169,6 +207,11 @@ class WeightCacheDaemon: else: self.model_config = vllm_config.model_config self.tp_rank = tp_rank + self.dp_rank = dp_rank + self.tp_size = parallel_config.tensor_parallel_size + self.dp_size = parallel_config.data_parallel_size + self.global_rank = dp_rank * self.tp_size + tp_rank + self.world_size = self.dp_size * self.tp_size self.local_rank = local_rank self.distributed_init_method = distributed_init_method self.socket_dir = socket_dir @@ -179,28 +222,32 @@ class WeightCacheDaemon: # mutate hf_config.quantization_config. self.cache_config = WeightCacheKey.from_model_config( self.model_config, - tp_size=vllm_config.parallel_config.tensor_parallel_size, + tp_size=self.tp_size, tp_rank=tp_rank, + dp_size=self.dp_size, + dp_rank=dp_rank, is_draft=is_draft, ) def load_model(self) -> None: - tp_size = self.cache_config.tp_size torch.accelerator.set_device_index(self.local_rank) + # Outside the config context so the daemon keeps its own rendezvous + # instead of the engine's; model parallel groups (TP/DP/EP) are then + # carved out of this world group like in the engine. init_distributed_environment( - world_size=tp_size, - rank=self.tp_rank, + world_size=self.world_size, + rank=self.global_rank, distributed_init_method=self.distributed_init_method, local_rank=self.local_rank, backend=current_platform.dist_backend, ) with set_current_vllm_config(self.vllm_config): - ensure_model_parallel_initialized(tp_size, 1) + ensure_model_parallel_initialized(self.tp_size, 1) self.model = self.get_model() logger.info( "Weight cache %s daemon rank %d loaded model", self.role, - self.tp_rank, + self.global_rank, ) def get_model(self) -> torch.nn.Module: @@ -219,6 +266,9 @@ class WeightCacheDaemon: target_device = torch.device( device_config.device if load_config.device is None else load_config.device ) + # Attention backends may take workspace at construction, like in the + # engine's worker init. + init_workspace_manager(target_device) with set_default_torch_dtype(model_config.dtype): with target_device: model = loader.create_model(vllm_config, model_config) @@ -255,7 +305,7 @@ class WeightCacheDaemon: logger.info( "Weight cache %s daemon rank %d serving on %s", self.role, - self.tp_rank, + self.global_rank, socket_path, ) if ready_callback is not None: @@ -347,7 +397,7 @@ class WeightCacheDaemon: logger.info_once( "Weight cache %s daemon rank %d sent %d tensors (+%d aliases) to engine", self.role, - self.tp_rank, + self.global_rank, len(entries), len(aliases), ) @@ -358,7 +408,7 @@ class WeightCacheDaemon: logger.info( "Weight cache %s daemon rank %d released cached weights", self.role, - self.tp_rank, + self.global_rank, ) send_msg(conn, {"status": "ok"}) @@ -371,6 +421,7 @@ def _run_daemon( socket_dir: str | None, ready_queue: "multiprocessing.Queue[tuple[str, int]]", is_draft: bool = False, + dp_rank: int = 0, ) -> None: daemon = WeightCacheDaemon( vllm_config, @@ -379,9 +430,34 @@ def _run_daemon( distributed_init_method, socket_dir, is_draft, + dp_rank, ) daemon.load_model() - daemon.serve_forever(ready_callback=lambda: ready_queue.put((daemon.role, tp_rank))) + daemon.serve_forever( + ready_callback=lambda: ready_queue.put((daemon.role, daemon.global_rank)) + ) + + +def plan_local_ranks(parallel_config: ParallelConfig) -> list[tuple[int, int, int]]: + """``(local_rank, dp_rank, tp_rank)`` for every GPU this launcher serves. + + Global ranks enumerate DP then TP: ``global = dp_rank * tp_size + tp_rank``. + With ``--nnodes`` the engine hands each node a contiguous block of global + ranks, so node ``r`` serves ``node_rank * local + i``; without it a + launcher's block starts at ``--data-parallel-start-rank * tp_size``. + """ + tp_size = parallel_config.tensor_parallel_size + world_size = parallel_config.data_parallel_size * tp_size + if parallel_config.nnodes > 1: + local_world_size = world_size // parallel_config.nnodes + base = parallel_config.node_rank * local_world_size + else: + local_world_size = parallel_config.data_parallel_size_local * tp_size + base = parallel_config.data_parallel_rank * tp_size + return [ + (i, (base + i) // tp_size, (base + i) % tp_size) + for i in range(local_world_size) + ] def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None: @@ -393,169 +469,30 @@ def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None: def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None: - """Reject parallelism modes other than tensor/expert parallelism.""" - unsupported = { - "pipeline parallelism": parallel_config.pipeline_parallel_size > 1, - "data parallelism": parallel_config.data_parallel_size > 1, - } - for name, enabled in unsupported.items(): - if enabled: - raise ValueError( - f"The weight cache daemon only supports tensor and expert " - f"parallelism; {name} is not supported" - ) - - -def main() -> None: - parser = FlexibleArgumentParser( - description="Launch weight cache daemons (one per TP rank)." - ) - EngineArgs.add_cli_args(parser) - parser.add_argument( - "--weight-cache-socket-dir", - type=str, - default=None, - help="Directory for the daemon Unix sockets (default: tempdir).", - ) - parser.add_argument( - "--weight-cache-master-port", - type=int, - default=None, - help="Rendezvous port for the daemon's own TP group. Must differ from " - "the engine's --master-port (the daemon holds its group open while " - "serving) and match across nodes. Required when --nnodes > 1; defaults " - "to a free port for single-node.", - ) - parser.add_argument( - "--weight-cache-draft-master-port", - type=int, - default=None, - help="Rendezvous port for the MTP draft daemon group. Defaults to " - "--weight-cache-master-port + 1 for multi-node, or a free port for " - "single-node.", - ) - args = parser.parse_args() - engine_args = EngineArgs.from_cli_args(args) - vllm_config = engine_args.create_engine_config() - if vllm_config.load_config.load_format == "ipc_cache": + """Reject pipeline parallelism and placements the daemon cannot map.""" + if parallel_config.pipeline_parallel_size > 1: raise ValueError( - "The weight cache daemon itself must load from disk; use the " - "default --load-format" + "The weight cache daemon only supports tensor, expert and data " + "parallelism; pipeline parallelism is not supported" ) - # Config-only so it can fail before any model loading; the quant method - # check needs the created model and runs in get_daemon_model. - check_ipc_platform_support() - parallel_config = vllm_config.parallel_config - _reject_unsupported_parallelism(parallel_config) + dp_size = parallel_config.data_parallel_size tp_size = parallel_config.tensor_parallel_size - - nnodes = parallel_config.nnodes - node_rank = parallel_config.node_rank - local_world_size = tp_size // nnodes - - # The daemon forms its own TP group and holds it open while serving, so it - # needs a rendezvous port distinct from the engine's --master-port. All - # nodes must agree on it; single-node can auto-pick a free port. The master - # address is reused from the engine's --master-addr (node_rank 0). - master_addr = parallel_config.master_addr - if nnodes > 1 and args.weight_cache_master_port is None: - raise ValueError("--weight-cache-master-port is required when --nnodes > 1") - master_port = args.weight_cache_master_port or get_open_port() - distributed_init_method = get_distributed_init_method(master_addr, master_port) - - # (is_draft, vllm_config, rendezvous) per daemon group. - groups: list[tuple[bool, VllmConfig, str]] = [ - (False, vllm_config, distributed_init_method) - ] - draft_vllm_config = get_draft_daemon_config(vllm_config) - if draft_vllm_config is not None: - draft_master_port = args.weight_cache_draft_master_port or ( - master_port + 1 if nnodes > 1 else get_open_port() - ) - if draft_master_port == master_port: + if parallel_config.nnodes > 1: + world_size = dp_size * tp_size + if world_size % parallel_config.nnodes != 0: raise ValueError( - "--weight-cache-draft-master-port must differ from " - "--weight-cache-master-port" + f"--nnodes ({parallel_config.nnodes}) must evenly divide the " + f"daemon world size ({world_size} = data-parallel-size " + f"{dp_size} x tensor-parallel-size {tp_size})" ) - groups.append( - ( - True, - draft_vllm_config, - get_distributed_init_method(master_addr, draft_master_port), - ) - ) - - ctx = multiprocessing.get_context("spawn") - ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue() - # Global TP rank of local GPU i on this node; local index == device index. - global_ranks = [ - node_rank * local_world_size + local_rank - for local_rank in range(local_world_size) - ] - expected_ready = { - (format_daemon_role(is_draft), global_rank) - for (is_draft, _, _), global_rank in product(groups, global_ranks) - } - procs = [ - ctx.Process( - target=_run_daemon, - args=( - global_rank, - local_rank, - config, - init_method, - args.weight_cache_socket_dir, - ready_queue, - is_draft, - ), - name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-{global_rank}", - ) - for (is_draft, config, init_method), (local_rank, global_rank) in product( - groups, enumerate(global_ranks) - ) - ] - for proc in procs: - proc.start() - - def _shutdown(signum, frame): - for proc in procs: - proc.terminate() - - signal.signal(signal.SIGINT, _shutdown) - signal.signal(signal.SIGTERM, _shutdown) - - ready: set[tuple[str, int]] = set() - while len(ready) < len(expected_ready): - try: - ready.add(ready_queue.get(timeout=1.0)) - except queue.Empty: - dead = [p for p in procs if p.exitcode is not None] - if dead: - logger.error( - "Weight cache daemon rank(s) exited during startup " - "(exitcodes=%s); shutting down.", - [p.exitcode for p in dead], - ) - for proc in procs: - proc.terminate() - for proc in procs: - proc.join() - sys.exit(max((p.exitcode or 0) for p in procs)) - socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir" - logger.info_once( - "===== Weight cache daemon READY: node %d/%d serving %d local rank(s) " - "x %d role(s) in %s =====", - node_rank, - nnodes, - local_world_size, - len(groups), - socket_dir_msg, + return + if dp_size == 1: + return + end_rank = ( + parallel_config.data_parallel_rank + parallel_config.data_parallel_size_local ) - - for proc in procs: - proc.join() - sys.exit(max(proc.exitcode or 0 for proc in procs)) - - -if __name__ == "__main__": - main() + if end_rank > dp_size: + raise ValueError( + "--data-parallel-start-rank + --data-parallel-size-local " + f"({end_rank}) exceeds --data-parallel-size ({dp_size})" + ) diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py index e22fc2b029..508e95c83d 100644 --- a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -5,6 +5,7 @@ daemon via CUDA IPC instead of loading from disk.""" import dataclasses import socket +import time from copy import copy import torch @@ -13,6 +14,7 @@ import torch.nn as nn from vllm.config import ModelConfig, VllmConfig from vllm.config.load import LoadConfig from vllm.distributed import ( + get_dp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) @@ -45,6 +47,7 @@ logger = init_logger(__name__) _CONNECT_TIMEOUT_S = 5.0 _STATE_TIMEOUT_S = 300.0 +_STARTUP_RETRY_INTERVAL_S = 0.5 class IpcModelLoader(BaseModelLoader): @@ -285,14 +288,46 @@ class IpcModelLoader(BaseModelLoader): _register(alias_name, obj, isinstance(obj, nn.Parameter)) def _fetch_entries(self, model_config: ModelConfig) -> WeightCacheState: + dp_group = get_dp_group() cache_config = WeightCacheKey.from_model_config( model_config, tp_size=get_tensor_model_parallel_world_size(), tp_rank=get_tensor_model_parallel_rank(), + dp_size=dp_group.world_size, + dp_rank=dp_group.rank_in_group, is_draft=self.is_draft, ) + if not self.fallback: + return self._request_state_with_startup_wait(cache_config) return self._request_state(cache_config) + def _request_state_with_startup_wait( + self, cache_config: WeightCacheKey + ) -> WeightCacheState: + deadline = time.monotonic() + self.state_timeout_s + waiting_logged = False + while True: + try: + return self._request_state(cache_config) + except WeightCacheUnavailableError as e: + if time.monotonic() >= deadline: + raise WeightCacheUnavailableError( + "Weight cache daemon did not become ready within " + f"{self.state_timeout_s:.1f}s" + ) from e + if not waiting_logged: + logger.info( + "Waiting up to %.1fs for the weight cache daemon to start", + self.state_timeout_s, + ) + waiting_logged = True + time.sleep( + max( + 0.0, + min(_STARTUP_RETRY_INTERVAL_S, deadline - time.monotonic()), + ) + ) + def _request_state(self, cache_config: WeightCacheKey) -> WeightCacheState: with self._connect(self.state_timeout_s) as conn: send_msg(conn, {"cmd": "get_state", "cache_config": cache_config}) diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index 3803030cd4..6b5bbc69e5 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -291,6 +291,8 @@ class WeightCacheKey: vllm_version: str is_draft: bool = False """Daemon group the weights come from; False is the target model.""" + dp_size: int = 1 + dp_rank: int = 0 @classmethod def from_model_config( @@ -300,6 +302,8 @@ class WeightCacheKey: tp_rank: int, *, is_draft: bool = False, + dp_size: int = 1, + dp_rank: int = 0, ) -> "WeightCacheKey": """Build the fingerprint for a model configuration. @@ -327,6 +331,8 @@ class WeightCacheKey: revision=model_config.revision, vllm_version=vllm.version.__version__, is_draft=is_draft, + dp_size=dp_size, + dp_rank=dp_rank, ) def mismatched_fields(self, other: "WeightCacheKey") -> list[str]: diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index c781a8bf87..12d557fa1c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -12,18 +12,17 @@ import os import tempfile import threading import time -from collections import defaultdict from collections.abc import Callable, Generator, Iterable -from contextlib import contextmanager +from contextlib import contextmanager, suppress from pathlib import Path -from typing import IO, Any +from typing import IO, TYPE_CHECKING, Any import filelock import huggingface_hub.constants import numpy as np import regex as re import torch -from safetensors.torch import load, load_file, safe_open, save_file +from safetensors.torch import load, load_file, safe_open from tqdm.auto import tqdm from transformers.utils import SAFE_WEIGHTS_INDEX_NAME @@ -62,6 +61,9 @@ except ImportError: from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least +if TYPE_CHECKING: + from vllm.model_executor.models.utils import WeightsMapper + logger = init_logger(__name__) # use system-level temp directory for file locks, so that multiple users @@ -186,56 +188,6 @@ def maybe_download_from_modelscope( return None -def _shared_pointers(tensors): - ptrs = defaultdict(list) - for k, v in tensors.items(): - ptrs[v.data_ptr()].append(k) - failing = [] - for _, names in ptrs.items(): - if len(names) > 1: - failing.append(names) - return failing - - -def convert_bin_to_safetensor_file( - pt_filename: str, - sf_filename: str, -) -> None: - loaded = torch.load(pt_filename, map_location="cpu", weights_only=True) - if "state_dict" in loaded: - loaded = loaded["state_dict"] - shared = _shared_pointers(loaded) - for shared_weights in shared: - for name in shared_weights[1:]: - loaded.pop(name) - - # For tensors to be contiguous - loaded = {k: v.contiguous() for k, v in loaded.items()} - - dirname = os.path.dirname(sf_filename) - os.makedirs(dirname, exist_ok=True) - save_file(loaded, sf_filename, metadata={"format": "pt"}) - - # check file size - sf_size = os.stat(sf_filename).st_size - pt_size = os.stat(pt_filename).st_size - if (sf_size - pt_size) / pt_size > 0.01: - raise RuntimeError( - f"""The file size different is more than 1%: - - {sf_filename}: {sf_size} - - {pt_filename}: {pt_size} - """ - ) - - # check if the tensors are the same - reloaded = load_file(sf_filename) - for k in loaded: - pt_tensor = loaded[k] - sf_tensor = reloaded[k] - if not torch.equal(pt_tensor, sf_tensor): - raise RuntimeError(f"The output tensors do not match for key {k}") - - # TODO(woosuk): Move this to other place. def get_quant_config( model_config: ModelConfig, load_config: LoadConfig @@ -243,7 +195,11 @@ def get_quant_config( if model_config.quantization is None: raise ValueError("Model quantization method is not specified in the config.") quant_cls = get_quantization_config(model_config.quantization) - from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs + from vllm.config.quantization import ( + _ONLINE_SHORTHANDS, + QuantizationConfigArgs, + resolve_quantization_config, + ) from vllm.model_executor.layers.quantization.online.base import ( OnlineQuantizationConfig, ) @@ -383,6 +339,14 @@ def get_quant_config( # If the quantization config is not found, use the default config. if not possible_config_filenames: + if model_config.quantization == "fp8": + logger.warning_once( + "--quantization fp8 is deprecated for online quantization; " + "use --quantization fp8_per_tensor instead." + ) + fp8_args = resolve_quantization_config("fp8_per_tensor", online_args) + assert fp8_args is not None + return OnlineQuantizationConfig(args=fp8_args) if model_config.quantization in _ONLINE_SHORTHANDS: args = online_args or _ONLINE_SHORTHANDS[model_config.quantization] assert isinstance(args, QuantizationConfigArgs) @@ -656,6 +620,171 @@ def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[s return hf_weights_files +def _normalize_module_prefixes( + language_model_names: Iterable[str] | None, +) -> tuple[str, ...]: + names = tuple(language_model_names or ()) + return tuple(n if n.endswith(".") else f"{n}." for n in names) + + +def _mapped_weight_name(weights_mapper: "WeightsMapper | None", key: str) -> str | None: + """Apply ``WeightsMapper.map_name`` when present; else identity.""" + if weights_mapper is None: + return key + return weights_mapper.map_name(key) + + +def _hf_prefix_maps_to_lm( + new_name: str | None, lm_module_prefixes: tuple[str, ...] +) -> bool: + if not new_name: + return False + new_p = new_name if new_name.endswith(".") else f"{new_name}." + return any(new_p.startswith(p) or p.startswith(new_p) for p in lm_module_prefixes) + + +def _shared_hf_root_module_prefixes( + lm_module_prefixes: tuple[str, ...], + weights_mapper: "WeightsMapper | None", +) -> tuple[str, ...]: + """Module prefixes that are also an HF root for non-LM weights. + + E.g. Molmo/Phi-4-MM/Muse mark LM as ``model`` while vision HF keys stay + under ``model.vision_*`` / ``model.embed_tokens_extend.*``. Detected via + ``WeightsMapper.orig_to_new_prefix`` (no hard-coded name list). + """ + if weights_mapper is None: + return () + mapping = weights_mapper.orig_to_new_prefix + if not mapping: + return () + + non_lm_origs: list[str] = [] + for orig, new in mapping.items(): + orig_p = orig if orig.endswith(".") else f"{orig}." + if not _hf_prefix_maps_to_lm(new, lm_module_prefixes): + non_lm_origs.append(orig_p) + + if not non_lm_origs: + return () + + shared = [ + p + for p in lm_module_prefixes + if any(orig.startswith(p) for orig in non_lm_origs) + ] + return tuple(shared) + + +def resolve_mm_encoder_only_lm_prefixes( + language_model_names: Iterable[str] | None, + *, + weights_mapper: "WeightsMapper | None" = None, +) -> tuple[str, ...] | None: + """Resolve vLLM LM *module* prefixes for ``--mm-encoder-only`` shard skip. + + Prefixes come from ``_language_model_names``. Classification of HF index + keys is done later via optional ``weights_mapper`` (see + ``filter_mm_encoder_only_safetensors_files``), so this helper does **not** + hard-code Qwen/HF nest lists. + + Returns ``None`` (leave the safetensors file list unchanged) when + ``_language_model_names`` is empty/missing, or when a module prefix is a + shared HF checkpoint root for both LM and non-LM weights — fail-closed so + Molmo / Phi-4-MM / Muse cannot under-load the encoder. + """ + prefixes = _normalize_module_prefixes(language_model_names) + if not prefixes: + return None + shared = _shared_hf_root_module_prefixes(prefixes, weights_mapper) + if shared: + logger.warning_once( + "mm-encoder-only whole-shard filter not applied: LM module " + "prefix(es) %s share an HF checkpoint root with non-LM weights " + "(via WeightsMapper). Encoder-only load keeps the full " + "safetensors file list; finer HF prefixes or key-level " + "filtering can enable skip later.", + shared, + ) + return None + return prefixes + + +def filter_mm_encoder_only_safetensors_files( + hf_weights_files: list[str], + hf_folder: str, + index_file: str, + language_model_prefixes: Iterable[str], + *, + weights_mapper: "WeightsMapper | None" = None, +) -> list[str]: + """Drop safetensors shards that only contain language-model weights. + + Used with ``--mm-encoder-only`` so Encoder-only EPD instances avoid reading + pure LM shards from disk/DRAM. + + Each HF index key is classified by mapping through ``weights_mapper`` (when + provided) and testing the *vLLM* name against ``language_model_prefixes``. + Without a mapper, HF names are compared directly (identity checkpoint + layout, e.g. Kimi ``language_model.*``). + + A shard is kept if it contains any non-LM key. Without an index file, + returns ``hf_weights_files`` unchanged. + """ + prefixes = tuple(language_model_prefixes) + if not prefixes: + return hf_weights_files + + index_path = os.path.join(hf_folder, index_file) + if not os.path.isfile(index_path): + logger.warning_once( + "mm-encoder-only shard filter skipped: index file %s not found", + index_path, + ) + return hf_weights_files + + with open(index_path) as f: + weight_map: dict[str, str] = json.load(f)["weight_map"] + + keys_by_file: dict[str, list[str]] = {} + for weight_name, weight_file in weight_map.items(): + keys_by_file.setdefault(weight_file, []).append(weight_name) + + def _is_lm_key(key: str) -> bool: + mapped = _mapped_weight_name(weights_mapper, key) + if mapped is None: + return False + return any(mapped.startswith(p) for p in prefixes) + + def _is_lm_only(weight_file: str) -> bool: + keys = keys_by_file.get(weight_file) + if not keys: + # Not listed in the index (e.g. extra matched glob) — keep. + return False + return all(_is_lm_key(key) for key in keys) + + kept: list[str] = [] + skipped = 0 + for path in hf_weights_files: + rel = os.path.relpath(path, hf_folder) + # Index weight_map values are usually basenames; tolerate either form. + basename = os.path.basename(path) + if _is_lm_only(rel) or _is_lm_only(basename): + skipped += 1 + continue + kept.append(path) + + if skipped: + logger.info_once( + "mm-encoder-only: skipped %d/%d safetensors shard(s) that only " + "contain language-model weights (prefixes=%s)", + skipped, + skipped + len(kept), + prefixes, + ) + return kept + + # explicitly use pure text format, with a newline at the end # this makes it impossible to see the animation in the progress bar # but will avoid messing up with ray or multiprocessing, which wraps @@ -715,6 +844,40 @@ def np_cache_weights_iterator( yield name, torch.from_numpy(param) +def drop_checkpoint_cache( + model_name_or_path: str, + revision: str | None = None, + cache_dir: str | None = None, +) -> None: + checkpoint_dir: Path | None = None + if os.path.isdir(model_name_or_path): + checkpoint_dir = Path(model_name_or_path) + else: + with suppress(OSError, ValueError): + checkpoint_dir = Path( + hf_api().snapshot_download( + model_name_or_path, + revision=revision, + cache_dir=cache_dir, + local_files_only=True, + ) + ) + if checkpoint_dir is None: + logger.info_once( + "No local checkpoint found for %s; skipping page-cache eviction", + model_name_or_path, + ) + return + for path in checkpoint_dir.glob("*.safetensors"): + try: + with path.open("rb") as file: + os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_DONTNEED) + except OSError as exc: + logger.warning( + "Could not release checkpoint page cache for %s: %s", path, exc + ) + + def _get_checkpoints_size_bytes(files: list[str]) -> int: """Return the total size of the checkpoint files in bytes.""" if not files: @@ -1313,29 +1476,6 @@ def composed_weight_loader( return composed_loader -def initialize_dummy_weights( - model: torch.nn.Module, - model_config: ModelConfig, - low: float = -1e-3, - high: float = 1e-3, - seed: int = 1234, -) -> None: - """Initialize model weights with random values. - - The model weights must be randomly initialized for accurate performance - measurements. Additionally, the model weights should not cause NaNs in the - forward pass. We empirically found that initializing the weights with - values between -1e-3 and 1e-3 works well for most models. - - We use per-parameter random seed, so that dummy weights are consistent, - even if the model is partitioned across multiple devices. When the seed - is fixed, the random values generated by this function only depends on - the parameter's number of elements and its data type. - """ - for param in model.state_dict().values(): - initialize_single_dummy_weight(param, low, high, seed) - - @torch.no_grad() def initialize_single_dummy_weight( param: torch.Tensor, diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index 9981f5904f..da38035db9 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -44,6 +44,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.utils import ( @@ -128,11 +129,9 @@ class AXK1MoE(nn.Module): "Only silu is supported for now." ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.n_routed_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) if config.topk_method == "noaux_tc": diff --git a/vllm/model_executor/models/adapters.py b/vllm/model_executor/models/adapters.py index 4139460762..cdbf628e61 100644 --- a/vllm/model_executor/models/adapters.py +++ b/vllm/model_executor/models/adapters.py @@ -284,7 +284,7 @@ def as_embedding_model(cls: type[_T]) -> type[_T]: def _resolve_num_labels(hf_config: Any, text_config: Any) -> int: """Resolve the label count for a sequence classification head. - ``PretrainedConfig.num_labels`` is derived from ``id2label``, which always + ``PreTrainedConfig.num_labels`` is derived from ``id2label``, which always carries a default of two entries. Composite configs (such as multimodal checkpoints) declare their label space on the top-level config, so reading ``num_labels`` from ``get_text_config()`` silently returns that default and @@ -297,9 +297,9 @@ def _resolve_num_labels(hf_config: Any, text_config: Any) -> int: if text_config is hf_config: return hf_config.num_labels - from transformers import PretrainedConfig + from transformers import PreTrainedConfig - if hf_config.num_labels != PretrainedConfig().num_labels: + if hf_config.num_labels != PreTrainedConfig().num_labels: return hf_config.num_labels return text_config.num_labels diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index 808775e35f..42643cb8c6 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -19,6 +19,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, MoERunner, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -82,11 +83,12 @@ class AfmoeMoE(nn.Module): ) # Router gate - self.gate = nn.Linear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - dtype=torch.float32, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", ) self.expert_bias = nn.Parameter( torch.empty(config.num_experts, dtype=torch.float32) @@ -140,7 +142,7 @@ class AfmoeMoE(nn.Module): num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) - router_logits = self.gate(hidden_states.to(dtype=torch.float32)) + router_logits, _ = self.gate(hidden_states) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 4be996d278..6308813ee2 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -22,7 +22,7 @@ from typing import Annotated, Any, Literal, TypeAlias import torch import torch.nn as nn -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from transformers.models.audioflamingo3 import ( AudioFlamingo3Config, AudioFlamingo3Processor, @@ -118,7 +118,7 @@ AudioFlamingo3Inputs: TypeAlias = ( class AudioFlamingo3Encoder(Qwen2AudioEncoder): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, ): super().__init__(config) self.avg_pooler = nn.AvgPool1d(kernel_size=2, stride=2) @@ -161,7 +161,7 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder): class AudioFlamingo3MultiModalProjector(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.linear_1 = nn.Linear( config.audio_config.hidden_size, @@ -197,6 +197,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo): return AudioFlamingo3MultiModalDataParser( target_sr=feature_extractor.sampling_rate, expected_hidden_size=self._get_expected_hidden_size(), + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_supported_mm_limits(self) -> Mapping[str, int | None]: @@ -237,6 +238,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]): if chunk_counts is not None: return dict( audio_embeds=MultiModalFieldConfig.batched("audio"), + audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), input_features=MultiModalFieldConfig.flat_from_sizes( "audio", chunk_counts, dim=0 ), @@ -247,6 +249,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]): ) return dict( audio_embeds=MultiModalFieldConfig.batched("audio"), + audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), input_features=MultiModalFieldConfig.batched("audio"), feature_attention_mask=MultiModalFieldConfig.batched("audio"), chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), @@ -354,11 +357,15 @@ def _count_audio_tokens_from_mask( class AudioFlamingo3MultiModalDataParser(MultiModalDataParser): + embedding_fields = { + "audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"}, + } + def _parse_audio_data( self, data: dict[str, torch.Tensor] | ModalityData[Any], ) -> ModalityDataItems[Any, Any] | None: - if isinstance(data, dict): + if isinstance(data, dict) and "audio_num_tokens" not in data: return DictEmbeddingItems( data, modality="audio", @@ -442,7 +449,11 @@ class AudioFlamingo3MultiModalProcessor( chunk_counts = out_mm_data.get("chunk_counts") def get_replacement_audioflamingo3(item_idx: int): - if feature_attention_mask is not None: + if "audio_num_tokens" in out_mm_data: + counts = out_mm_data["audio_num_tokens"] + assert isinstance(counts, torch.Tensor) + num_features = int(counts[item_idx]) + elif feature_attention_mask is not None: num_features = _count_audio_tokens_from_mask( feature_attention_mask, chunk_counts, diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index 33749818b1..0a3b9c8e47 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -30,7 +30,7 @@ from itertools import islice import torch import torch.nn.functional as F from torch import nn -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -41,7 +41,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -71,7 +74,7 @@ from .utils import ( class BailingAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, @@ -180,7 +183,7 @@ class BailingMLP(nn.Module): def __init__( self, intermediate_size: int, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, prefix: str = "", @@ -214,7 +217,7 @@ class BailingMoE(nn.Module): def __init__( self, intermediate_size: int, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, reduce_results: bool | None = True, prefix: str = "", @@ -243,11 +246,12 @@ class BailingMoE(nn.Module): else: self.router_dtype = torch.bfloat16 - self.gate = nn.Linear( + self.gate = GateLinear( self.hidden_size, self.num_experts, - bias=False, - dtype=self.router_dtype, + out_dtype=self.router_dtype, + params_dtype=self.router_dtype, + prefix=f"{prefix}.gate", ) if getattr(config, "moe_router_enable_expert_bias", False): @@ -312,7 +316,7 @@ class BailingMoE(nn.Module): hidden_states = hidden_states.view(-1, hidden_size) # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states.to(self.router_dtype)) + router_logits, _ = self.gate(hidden_states) router_logits = router_logits.to(hidden_states.dtype) final_hidden_states = self.experts( @@ -324,7 +328,7 @@ class BailingMoE(nn.Module): class BailingMoeBlock(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index 58a06c5b2f..aea9559e6c 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -5,8 +5,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -import torch.nn.functional as F -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -19,6 +18,7 @@ from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -79,7 +79,7 @@ class BailingMoeV25MLAAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "attention", @@ -229,43 +229,12 @@ class BailingMoeV25MLAAttention(nn.Module): return self.mla_attn(positions, hidden_states) -class BailingMoEGate(nn.Module): - def __init__( - self, - config: PretrainedConfig, - params_dtype: torch.dtype | None = None, - prefix: str = "", - ): - super().__init__() - if params_dtype is None: - params_dtype = torch.get_default_dtype() - self.params_dtype = params_dtype - self.weight = nn.Parameter( - torch.empty( - (config.num_experts, config.hidden_size), - dtype=self.params_dtype, - ), - ) - if getattr(config, "moe_router_enable_expert_bias", False): - self.expert_bias = nn.Parameter( - torch.empty((config.num_experts,), dtype=torch.float32), - ) - else: - self.expert_bias = None - - def forward(self, hidden_states): - logits = F.linear(hidden_states.to(self.weight.dtype), self.weight, None).to( - hidden_states.dtype - ) - return logits - - class BailingMoeV25(nn.Module): """Bailing MoE v2.5 - standalone implementation for linear attention model.""" def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "", @@ -296,11 +265,19 @@ class BailingMoeV25(nn.Module): self.router_dtype = torch.bfloat16 # Gate for routing - self.gate = BailingMoEGate( - config=config, + self.gate = GateLinear( + self.hidden_size, + self.num_experts, + out_dtype=self.router_dtype, params_dtype=self.router_dtype, prefix=f"{prefix}.gate", ) + if getattr(config, "moe_router_enable_expert_bias", False): + self.gate.expert_bias = nn.Parameter( + torch.empty((self.num_experts,), dtype=torch.float32), + ) + else: + self.gate.expert_bias = None correction_bias = ( self.gate.expert_bias if self.gate.expert_bias is not None else None ) @@ -357,7 +334,7 @@ class BailingMoeV25(nn.Module): hidden_states = hidden_states.contiguous().view(-1, hidden_size) # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states.to(self.router_dtype)) + router_logits, _ = self.gate(hidden_states) router_logits = router_logits.to(hidden_states.dtype) final_hidden_states = self.experts( @@ -372,7 +349,7 @@ class BailingMoeV25DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = "layer", layer_id: int = 0, diff --git a/vllm/model_executor/models/bailing_moe_mtp.py b/vllm/model_executor/models/bailing_moe_mtp.py index a403244a17..32cc168102 100644 --- a/vllm/model_executor/models/bailing_moe_mtp.py +++ b/vllm/model_executor/models/bailing_moe_mtp.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -36,7 +36,7 @@ from vllm.sequence import IntermediateTensors from .utils import PPMissingLayer, is_pp_missing_parameter, maybe_prefix -def _get_draft_hf_config(vllm_config: VllmConfig) -> PretrainedConfig: +def _get_draft_hf_config(vllm_config: VllmConfig) -> PreTrainedConfig: speculative_config = vllm_config.speculative_config if speculative_config is not None: draft_model_config = speculative_config.draft_model_config @@ -48,7 +48,7 @@ def _get_draft_hf_config(vllm_config: VllmConfig) -> PretrainedConfig: class BailingMTPSharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, vllm_config: VllmConfig, ) -> None: diff --git a/vllm/model_executor/models/bailing_moe_v3.py b/vllm/model_executor/models/bailing_moe_v3.py index 710a37268e..9f5c078fac 100644 --- a/vllm/model_executor/models/bailing_moe_v3.py +++ b/vllm/model_executor/models/bailing_moe_v3.py @@ -16,9 +16,8 @@ from typing import TypeGuard import regex as re import torch import torch.nn as nn -import torch.nn.functional as F from einops import rearrange -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config @@ -32,6 +31,7 @@ from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -151,7 +151,7 @@ def _get_kda_state_shape_for_config( return shapes -def _build_rope_parameters(config: PretrainedConfig) -> dict | None: +def _build_rope_parameters(config: PreTrainedConfig) -> dict | None: rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {} if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"): rope_parameters["rope_theta"] = config.rope_theta @@ -172,7 +172,7 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None: def _build_mla_rotary_embedding( - config: PretrainedConfig, + config: PreTrainedConfig, head_size: int, ) -> nn.Module: rope_parameters = _build_rope_parameters(config) @@ -223,7 +223,7 @@ def _is_block_fp8_config( def _configure_ling_fp8_quant_config( quant_config: QuantizationConfig | None, - config: PretrainedConfig, + config: PreTrainedConfig, ) -> None: if not _is_block_fp8_config(quant_config): return @@ -372,7 +372,7 @@ def _pad_block_fp8_mlp_checkpoint_tensor( def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor( quant_config: QuantizationConfig | None, - config: PretrainedConfig, + config: PreTrainedConfig, name: str, loaded_weight: torch.Tensor, ) -> torch.Tensor: @@ -410,7 +410,7 @@ class BailingMoeV3MLP(nn.Module): def __init__( self, intermediate_size: int, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, prefix: str = "", @@ -446,7 +446,7 @@ class BailingMoeV3MLP(nn.Module): class BailingMoeV3MLAAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "attention", @@ -625,7 +625,7 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "attention", @@ -1118,32 +1118,10 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase): core_attn_out[0, :num_actual_tokens] = out[0, :num_actual_tokens] -class BailingMoeV3Gate(nn.Module): - def __init__( - self, - config: PretrainedConfig, - params_dtype: torch.dtype | None = None, - ) -> None: - super().__init__() - if params_dtype is None: - params_dtype = torch.float32 - self.weight = nn.Parameter( - torch.empty((config.num_experts, config.hidden_size), dtype=params_dtype) - ) - self.expert_bias = nn.Parameter( - torch.empty((config.num_experts,), dtype=torch.float32) - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - return F.linear(hidden_states.to(self.weight.dtype), self.weight).to( - hidden_states.dtype - ) - - class BailingMoeV3MoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "", @@ -1153,7 +1131,16 @@ class BailingMoeV3MoE(nn.Module): self.top_k = config.num_experts_per_tok self.hidden_size = config.hidden_size self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) - self.gate = BailingMoeV3Gate(config) + self.gate = GateLinear( + config.hidden_size, + config.num_experts, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", + ) + self.gate.expert_bias = nn.Parameter( + torch.empty((config.num_experts,), dtype=torch.float32) + ) self.expert_swiglu_limit = _get_layer_swiglu_limit( getattr(config, "expert_swiglu_limit_list", None), layer_id ) @@ -1196,7 +1183,7 @@ class BailingMoeV3MoE(nn.Module): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: num_tokens, hidden_size = hidden_states.shape hidden_states = hidden_states.contiguous().view(-1, hidden_size) - router_logits = self.gate(hidden_states.to(torch.float32)) + router_logits, _ = self.gate(hidden_states) hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits ) @@ -1206,7 +1193,7 @@ class BailingMoeV3MoE(nn.Module): class BailingMoeV3DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, layer_id: int = 0, prefix: str = "layer", diff --git a/vllm/model_executor/models/bailing_moe_v3_mtp.py b/vllm/model_executor/models/bailing_moe_v3_mtp.py index 2b4c3c8d78..3d5b3556c2 100644 --- a/vllm/model_executor/models/bailing_moe_v3_mtp.py +++ b/vllm/model_executor/models/bailing_moe_v3_mtp.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -47,7 +47,7 @@ from vllm.model_executor.models.utils import ( from vllm.sequence import IntermediateTensors -def _get_draft_hf_config(vllm_config: VllmConfig) -> PretrainedConfig: +def _get_draft_hf_config(vllm_config: VllmConfig) -> PreTrainedConfig: speculative_config = vllm_config.speculative_config if speculative_config is not None: draft_model_config = speculative_config.draft_model_config @@ -59,7 +59,7 @@ def _get_draft_hf_config(vllm_config: VllmConfig) -> PretrainedConfig: class BailingMoeV3MTPSharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, vllm_config: VllmConfig, ) -> None: diff --git a/vllm/model_executor/models/bert.py b/vllm/model_executor/models/bert.py index 85ef42675b..7348ec9383 100644 --- a/vllm/model_executor/models/bert.py +++ b/vllm/model_executor/models/bert.py @@ -25,7 +25,6 @@ from vllm.model_executor.layers.pooler import ( Pooler, PoolingParamsUpdate, ) -from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation from vllm.model_executor.layers.pooler.seqwise import ( EmbeddingPoolerHead, SequencePooler, @@ -116,11 +115,11 @@ class BertPooler(SequencePooler): ) self.act_fn = nn.Tanh() + # Keep the model's head layers even when output activation is disabled. # Use lambdas so that weights are not registered under `self.head` self.head = EmbeddingPoolerHead( head_dtype=head_dtype, - projector=lambda x: self.dense(x), - activation=LambdaPoolerActivation(self.act_fn), + projector=lambda x: self.act_fn(self.dense(x)), ) diff --git a/vllm/model_executor/models/bert_with_rope.py b/vllm/model_executor/models/bert_with_rope.py index ba6b20ec5f..cd35792b28 100644 --- a/vllm/model_executor/models/bert_with_rope.py +++ b/vllm/model_executor/models/bert_with_rope.py @@ -4,7 +4,7 @@ from collections.abc import Iterable import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -18,7 +18,11 @@ from vllm.model_executor.layers.activation import get_act_and_mul_fn, get_act_fn from vllm.model_executor.layers.attention import ( EncoderOnlyAttention, ) -from vllm.model_executor.layers.fused_moe import activation_without_mul, fused_topk +from vllm.model_executor.layers.fused_moe import ( + GateLinear, + activation_without_mul, + fused_topk, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, @@ -46,7 +50,7 @@ from .interfaces_base import default_pooling_type class BertWithRopeEmbedding(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() if config.position_embedding_type not in ["rope", "rotary"]: raise ValueError( @@ -254,9 +258,7 @@ class NomicMoE(nn.Module): params_dtype = torch.get_default_dtype() self.params_dtype = params_dtype - self.router = ReplicatedLinear( - self.hidden_size, self.num_total_experts, bias=False - ) + self.router = GateLinear(self.hidden_size, self.num_total_experts) self.w1 = nn.Parameter( torch.empty( self.num_total_experts, @@ -345,7 +347,7 @@ class NomicMoE(nn.Module): class BertWithRopeBlock(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, moe: bool = False, diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index 98e649dec6..7c17edc2a1 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -16,11 +16,13 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -275,12 +277,10 @@ class Cohere2Moe(nn.Module): else: self.custom_routing_function = None - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, params_dtype=params_dtype, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index 4fdaff4444..bc4f78ed2c 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -8,10 +8,10 @@ from typing import Annotated, Literal import torch from torch import nn -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from transformers.models.cohere2_vision import Cohere2VisionConfig -from transformers.models.cohere2_vision.image_processing_cohere2_vision_fast import ( # noqa: E501 - Cohere2VisionImageProcessorFast, +from transformers.models.cohere2_vision.image_processing_cohere2_vision import ( # noqa: E501 + Cohere2VisionImageProcessor, ) from transformers.models.cohere2_vision.processing_cohere2_vision import ( Cohere2VisionProcessor, @@ -178,7 +178,7 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo): """Calculate the number of image patches for a given image. Uses the HF processor to determine the actual number of patches. """ - image_processor: Cohere2VisionImageProcessorFast = processor.image_processor + image_processor: Cohere2VisionImageProcessor = processor.image_processor return image_processor.get_number_of_image_patches( image_height, @@ -415,7 +415,7 @@ class Cohere2VisionForConditionalGeneration( ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig | None + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index c8a196ffdc..298ccda952 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar import torch import torch.nn.functional as F from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig @@ -1861,7 +1861,7 @@ class CohereASRModel(nn.Module): class CohereASRProcessingInfo(BaseProcessingInfo): - def get_hf_config(self) -> PretrainedConfig: + def get_hf_config(self) -> PreTrainedConfig: return self.ctx.get_hf_config() def get_default_tok_params(self) -> TokenizeParams: diff --git a/vllm/model_executor/models/cohere_compass.py b/vllm/model_executor/models/cohere_compass.py index 9e8adabb80..8593bedfc0 100644 --- a/vllm/model_executor/models/cohere_compass.py +++ b/vllm/model_executor/models/cohere_compass.py @@ -1161,7 +1161,6 @@ class CohereCompassProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> CohereCompassProcessor: return self.ctx.get_hf_processor( CohereCompassProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index 3f04c86b96..0e92363001 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -61,7 +61,6 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo): # correctly for the Qwen3-VL backbone. return self.ctx.get_hf_processor( Qwen3VLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) diff --git a/vllm/model_executor/models/colqwen3_5.py b/vllm/model_executor/models/colqwen3_5.py index aa8a5d0c81..7eddbe983e 100644 --- a/vllm/model_executor/models/colqwen3_5.py +++ b/vllm/model_executor/models/colqwen3_5.py @@ -58,7 +58,6 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor: return self.ctx.get_hf_processor( Qwen3VLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 8288eab236..3c87b06167 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -6,7 +6,7 @@ from vllm.logger import init_logger from vllm.utils.math_utils import round_up if TYPE_CHECKING: - from transformers import PretrainedConfig + from transformers import PreTrainedConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.config.cache import MambaDType @@ -212,12 +212,15 @@ class Gemma4Config(VerifyAndUpdateConfig): """Configure attention for heterogeneous head dimensions. Gemma4 uses different head dimensions for sliding window vs full attention - layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes - mixed backend selection and numerical divergence. + layers. The default FA3 on Hopper cannot handle head_dim > 256. - When FA4 is available we force it for ALL layers, giving a uniform kernel path - and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back - to Triton. + On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic + FlashAttention selector upgrade larger head dimensions to FA4. + The multimodal-prefix composite routes image masks to Triton and causal + requests to this per-layer FA3/FA4 selection. For other configurations, + force FA4 for all layers to avoid the mixed + FA3+FA4 penalty. + When FA4 is not available, fall back to Triton. """ model_config = vllm_config.model_config arch_config = model_config.model_arch_config @@ -230,6 +233,7 @@ class Gemma4Config(VerifyAndUpdateConfig): if len(set(head_dims.values())) <= 1: return + from vllm.platforms import current_platform from vllm.v1.attention.backends.fa_utils import is_fa_version_supported from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -241,12 +245,22 @@ class Gemma4Config(VerifyAndUpdateConfig): and vllm_config.attention_config.backend in (None, AttentionBackendEnum.FLASH_ATTN) ): - vllm_config.attention_config.flash_attn_version = 4 - logger.info( - "Gemma4 model has heterogeneous head dimensions %s. Using FA4 for " - "all layers to avoid mixed FA3/FA4 penalty.", - head_dims, - ) + use_per_layer_fa = current_platform.is_device_capability_family( + 90 + ) and vllm_config.cache_config.cache_dtype.startswith("fp8") + if use_per_layer_fa: + logger.info( + "Gemma4 model has heterogeneous head dimensions %s. Using " + "per-layer FA3/FA4 selection for FP8 KV cache on SM90.", + head_dims, + ) + else: + vllm_config.attention_config.flash_attn_version = 4 + logger.info( + "Gemma4 model has heterogeneous head dimensions %s. Using FA4 " + "for all layers to avoid mixed FA3/FA4 penalty.", + head_dims, + ) elif vllm_config.attention_config.backend is None: vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN logger.info( @@ -280,11 +294,14 @@ class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig): "FLASH_ATTN or TRITON_ATTN instead." ) if attention_config.backend is None and not attention_config.use_non_causal: + # Only the sm_100 default backend order reads this flag (it puts + # FlashInfer first for causal models). Elsewhere Gemma4Config above + # keeps FlashInfer out: FA4 on every layer, else TRITON_ATTN. attention_config.use_non_causal = True logger.info( - "DiffusionGemma uses mixed causal/bidirectional attention " - "within a batch; setting use_non_causal=True to exclude " - "FlashInfer from auto-selection." + "DiffusionGemma mixes causal and bidirectional attention in " + "one batch; setting use_non_causal=True so the default backend " + "order prefers FlashAttention over FlashInfer." ) # Auto-create DiffusionConfig from HF config if not provided. @@ -529,6 +546,7 @@ class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig): def verify_and_update_model_config(model_config: "ModelConfig") -> None: config = model_config.hf_config config.num_labels = 1 + config.get_text_config().num_labels = 1 pooler_config = model_config.pooler_config assert pooler_config is not None if pooler_config.logit_mean is None: @@ -663,7 +681,7 @@ class NemotronHForCausalLMConfig(VerifyAndUpdateConfig): @classmethod def update_mamba_ssm_cache_dtype( - cls, *, cache_config: "CacheConfig", hf_config: "PretrainedConfig" + cls, *, cache_config: "CacheConfig", hf_config: "PreTrainedConfig" ) -> None: """Update mamba_ssm_cache_dtype for NemotronH models when set to 'auto' (or not explicitly set), to the value specified in the HF config, or to diff --git a/vllm/model_executor/models/dbrx.py b/vllm/model_executor/models/dbrx.py index 0f5ffeff03..bfea997698 100644 --- a/vllm/model_executor/models/dbrx.py +++ b/vllm/model_executor/models/dbrx.py @@ -18,11 +18,11 @@ from vllm.distributed import ( from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, RoutedExperts, ) from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -62,12 +62,10 @@ class DbrxRouter(nn.Module): self.tp_size = get_tensor_model_parallel_world_size() self.num_total_experts = config.ffn_config.moe_num_experts self.d_model = config.d_model - self.layer = ReplicatedLinear( + self.layer = GateLinear( self.d_model, self.num_total_experts, - bias=False, params_dtype=params_dtype, - quant_config=None, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 36f0a240bc..5adc732f42 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -5,7 +5,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -51,18 +51,23 @@ from .utils import ( class SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, + *, + defer_lm_head: bool = False, ) -> None: super().__init__() self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "head"), - ) + if defer_lm_head: + self.head = None + else: + self.head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "head"), + ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.norm(hidden_states) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index ed702400be..d5728e0c33 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -665,6 +665,11 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase): def forward(self): ... + @property + def uses_shuffled_layout(self) -> bool: + """Whether this cache's reader expects the shuffled value layout.""" + return False + def get_attn_backend(self) -> type[AttentionBackend]: return DeepseekV32IndexerBackend diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index aef9d81625..1ae1cc7116 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -16,12 +16,13 @@ via Gemma4MultimodalEmbedder. from __future__ import annotations -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from types import SimpleNamespace from typing import Any import numpy as np import torch +import torch._dynamo from torch import nn from torch.nn import functional as F from transformers import AutoModel @@ -36,6 +37,10 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, ) +from vllm.model_executor.models.diffusion_gemma_sampler import ( + sample_row_stats, + sample_row_stats_reference, +) from vllm.model_executor.models.gemma4 import Gemma4Model from vllm.model_executor.models.gemma4_mm import ( Gemma4DummyInputsBuilder, @@ -111,13 +116,13 @@ class DiffusionGemmaProcessingInfo(Gemma4ProcessingInfo): """Processing info for DiffusionGemma. Overrides ``get_hf_config`` to accept ``DiffusionGemmaConfig`` - (which inherits from ``PretrainedConfig``, not ``Gemma4Config``). + (which inherits from ``PreTrainedConfig``, not ``Gemma4Config``). Supports image and video modalities. """ def get_hf_config(self): # DiffusionGemmaConfig doesn't inherit from Gemma4Config, so we - # accept any PretrainedConfig here. + # accept any PreTrainedConfig here. return self.ctx.get_hf_config() def get_supported_mm_limits(self) -> Mapping[str, int | None]: @@ -249,6 +254,8 @@ class DiffusionGemmaForConditionalGeneration( self.lm_head = ParallelLMHead( num_embeddings=text_config.vocab_size, embedding_dim=text_config.hidden_size, + quant_config=vllm_config.quant_config, + prefix=maybe_prefix(prefix, "lm_head"), ) if text_config.tie_word_embeddings: @@ -338,6 +345,17 @@ class DiffusionGemmaForConditionalGeneration( ) def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + states = getattr(self, "diffusion_states", None) + allowed = states.step_allowed if states is not None else None + if allowed is not None: + # One column per allowed id: a [rows, K] GEMM over the K gathered + # rows of the tied embedding. + logits = torch.nn.functional.linear( + hidden_states, self.lm_head.weight[allowed] + ).float() + if self.final_logit_softcapping is not None: + logits = _softcap_logits(logits, self.final_logit_softcapping) + return logits logits = self.logits_processor(self.lm_head, hidden_states) if logits is not None and self.final_logit_softcapping is not None: logits = _softcap_logits(logits, self.final_logit_softcapping) @@ -371,10 +389,89 @@ def _compute_num_rejected( return torch.where(is_denoise, query_lens, num_rejected) -@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) -def _compiled_sample_step( - # Logits from the model [num_decode * CL, vocab] +def _concat_logprob_stashes( + parts: list[LogprobsTensors], cu_num_generated_tokens: list[int] +) -> LogprobsTensors: + """Join the logprobs stashed for the requests committing this step. + + Each stash is as wide as the widest logprobs request in the batch at the + step that request converged, so stashes from different steps can differ + in width. Pad the narrow ones the way compute_topk_scores pads a mixed + batch: token id 0 at -inf, which the output processor never reports. + """ + width = max(p.logprob_token_ids.shape[1] for p in parts) + + def pad(t: torch.Tensor, value: float) -> torch.Tensor: + return F.pad(t, (0, width - t.shape[1]), value=value) + + return LogprobsTensors( + logprob_token_ids=torch.cat([pad(p.logprob_token_ids, 0) for p in parts]), + logprobs=torch.cat([pad(p.logprobs, float("-inf")) for p in parts]), + selected_token_ranks=torch.cat([p.selected_token_ranks for p in parts]), + cu_num_generated_tokens=cu_num_generated_tokens, + ) + + +_MASKED_LOGIT = -1e20 + + +def _mask_rows_to_allowed( logits: torch.Tensor, + row_starts: Sequence[int] | np.ndarray, + row_lens: Sequence[int] | np.ndarray, + allowed_per_row: Sequence[torch.Tensor | None], +) -> torch.Tensor: + """Mask every column outside a request's allowed ids on that request's + rows. Request i owns rows [row_starts[i], +row_lens[i]); None leaves its + rows alone. Returns a copy when any row is masked, so the caller's tensor + (possibly the runner's) is never written. + + Softmax over a masked row equals the K-space distribution the shared + fast path computes, so both paths give the same reads. The mask value is + a large finite negative rather than -inf: the entropy is probs times + log-probs, and 0 * -inf is NaN, while 0 * -1e20 is 0. It stays finite + after the greedy temperature clamp (1e-10) scales it by 1e10. + """ + out: torch.Tensor | None = None + keep = torch.zeros(logits.shape[-1], dtype=torch.bool, device=logits.device) + for start, n, allowed in zip(row_starts, row_lens, allowed_per_row): + if allowed is None: + continue + if out is None: + out = logits.clone() + keep.zero_() + keep[allowed] = True + out[int(start) : int(start) + int(n)].masked_fill_(~keep, _MASKED_LOGIT) + return logits if out is None else out + + +# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas, +# on compute_sc and on sizes that coincide with the state buffers. That set +# is small but passes Dynamo's default of 8, after which every step would run +# eager: about twice as slow for a self-conditioned step. +@torch._dynamo.config.patch(recompile_limit=64) +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) +def _denoise_temperature( + step_tensor: torch.Tensor, + slots: torch.Tensor, + max_denoising_steps: float, + t_min: float, + t_max: float, +) -> torch.Tensor: + """The schedule's temperature for each slot at its current step.""" + steps_f = step_tensor[slots].float() + remaining = (max_denoising_steps - steps_f).clamp(min=1.0) + return t_min + (t_max - t_min) * (remaining / max_denoising_steps) + + +def _compiled_sample_step( + # Per-position statistics of the temperature-scaled logits, from + # sample_row_stats: [num_decode, CL] each, and the softmax + # [num_decode, CL, vocab] in the embedding dtype when compute_sc. + new_tokens: torch.Tensor, + argmax_tokens: torch.Tensor, + token_entropy: torch.Tensor, + probs: torch.Tensor | None, # Request mapping decode_slots: torch.Tensor, # [num_decode] int64 → slot indices decode_idx: torch.Tensor, # [num_decode] int64 → position in num_reqs @@ -391,14 +488,15 @@ def _compiled_sample_step( normalizer: torch.Tensor, history: torch.Tensor, # [max_num_reqs, ST, CL] history_len_tensor: torch.Tensor, # [max_num_reqs] + max_steps_tensor: torch.Tensor, # [max_num_reqs] int32, per-slot step cap + pin_mask: torch.Tensor, # [max_num_reqs, CL] bool, positions held at the seed + seed_canvas: torch.Tensor, # [max_num_reqs, CL] + read_only: torch.Tensor, # [max_num_reqs] bool, emit without a commit forward # Output tensors (modified in-place) sampled: torch.Tensor, # [num_reqs, CL] num_sampled: torch.Tensor, # [num_reqs] draft_tokens: torch.Tensor, # [max_num_reqs, >=CL] # Scalar config - max_denoising_steps: float, - t_min: float, - t_max: float, confidence_threshold: float, vocab_size: int, CL: int, @@ -412,39 +510,16 @@ def _compiled_sample_step( sc_vocab_end: int, tp_size: int, tp_group_name: str, -) -> torch.Tensor: - """Compiled decode step: temperature → Gumbel sample → probs/confidence → - accept/renoise → convergence, all as vectorized PyTorch ops. - - Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the - caller can compute logprobs outside the compiled region.""" + compute_sc: bool = True, +) -> None: + """Compiled decode step: confidence → accept/renoise → convergence, as + vectorized PyTorch ops over [num_decode, CL] tensors. The per-position + statistics (argmax, Gumbel-max sample, entropy, softmax) come from one + pass over the logits in ``sample_row_stats``.""" num_decode = decode_slots.shape[0] device = decode_slots.device - # ---- Phase 1: Temperature schedule ---- - steps_f = step_tensor[decode_slots].float() - remaining = (max_denoising_steps - steps_f).clamp(min=1.0) - temp = t_min + (t_max - t_min) * (remaining / max_denoising_steps) - - # ---- Phase 2: Temperature scaling + Gumbel-max sampling ---- - logits_3d = logits.reshape(num_decode, CL, -1).float() - scaled = logits_3d / temp[:, None, None].clamp(min=1e-10) - - # Gumbel-max trick: argmax(logits/T + Gumbel) ~ sample from softmax(logits/T) - u = torch.rand_like(scaled).clamp(min=1e-20) - gumbel = -torch.log(-torch.log(u)) - # Zero noise when temp==0 (greedy) - noisy = scaled + gumbel * (temp[:, None, None] > 0).float() - new_tokens = noisy.view(-1, noisy.shape[-1]).argmax(dim=-1).view(num_decode, CL) - argmax_tokens = ( - scaled.view(-1, scaled.shape[-1]).argmax(dim=-1).view(num_decode, CL) - ) - - # ---- Phase 3: Probs, self-conditioning, confidence ---- - log_probs = scaled.log_softmax(dim=-1) - probs = log_probs.exp() - - token_entropy = -(probs * log_probs).sum(dim=-1) # [num_decode, CL] + # ---- Phase 3: Confidence ---- # A canvas truncated near max_model_len is zero-padded up to CL by the # caller; those padded rows are uniform (max entropy, argmax 0), so they # never trigger early convergence and are stable, and only the real @@ -474,12 +549,19 @@ def _compiled_sample_step( step_tensor[decode_slots] = new_step_val # Random tokens for renoise / canvas reinit + # Renoise draws from the whole vocabulary even for a constrained read. + # The model was trained on random-vocabulary noise. Noise drawn from the + # allowed set looks like garbled text, and later steps denoise it into + # more garbled text. random_tokens = torch.randint( 0, vocab_size, (num_decode, CL), device=device, dtype=canvas.dtype ) - # Compute denoise canvas (accept/renoise) + # Compute denoise canvas (accept/renoise). Pinned positions keep the seed. denoise_canvas = torch.where(eb_mask, new_tokens, random_tokens) + denoise_canvas = torch.where( + pin_mask[decode_slots], seed_canvas[decode_slots], denoise_canvas + ) # Canvas: commit → random reinit, denoise → accept/renoise result canvas[decode_slots] = torch.where( @@ -504,16 +586,6 @@ def _compiled_sample_step( new_hist_len = torch.where(is_denoise, hist_len + 1, hist_len.new_zeros(num_decode)) history_len_tensor[decode_slots] = new_hist_len - # Sampled output: commit → emit argmax_canvas, denoise → 0 (pre-zeroed) - sampled[decode_idx] = argmax_canvas[decode_slots].to( - sampled.dtype - ) * is_commit.unsqueeze(1).to(sampled.dtype) - # Commit only the real canvas length (== CL except for a canvas truncated - # near max_model_len); the padded tail positions are never emitted. - num_sampled[decode_idx] = is_commit.to(num_sampled.dtype) * valid_canvas_len.to( - num_sampled.dtype - ) - # ---- Phase 6: Stability + convergence ---- ref = history[decode_slots, 0] mismatch = torch.zeros(num_decode, device=device, dtype=torch.int32) @@ -523,13 +595,17 @@ def _compiled_sample_step( step_after = step_tensor[decode_slots] converged = (stable & confident_tensor[decode_slots] & (new_hist_len >= ST)) | ( - step_after >= max_denoising_steps + step_after >= max_steps_tensor[decode_slots] ) # Commit done → denoise next (False); denoise converged → commit next (True) is_encoder_phase[decode_slots] = torch.where( is_commit, is_commit.new_zeros(num_decode), converged ) + emit = is_commit | (is_denoise & converged & read_only[decode_slots]) + sampled[decode_idx] = argmax_canvas[decode_slots].to(sampled.dtype) * emit[:, None] + num_sampled[decode_idx] = (emit * valid_canvas_len).to(num_sampled.dtype) + # SC soft embedding: store ``probs @ embed_weight`` (the value the next step's # self-conditioning MLP consumes) only for slots that will denoise next — i.e. # this step denoised AND it isn't about to commit (is_encoder_phase now False). @@ -537,30 +613,46 @@ def _compiled_sample_step( # sc_embeds directly. Storing the [.., hidden] soft embed instead of the full # [.., vocab] probs avoids a giant persistent buffer. sc_keep = (is_denoise & ~is_encoder_phase[decode_slots])[:, None, None] - # Self-conditioning soft embed = probs @ embed_tokens.weight. Under tensor - # parallelism the embedding is vocab-sharded ([vocab/tp, hidden]) while - # probs spans the full vocab, so each rank multiplies its local vocab slice - # [sc_vocab_start, sc_vocab_end) and the partials are summed across ranks. - local_probs = probs[..., sc_vocab_start:sc_vocab_end].to(embed_weight.dtype) - soft_embeds = torch.matmul( - local_probs, embed_weight[: sc_vocab_end - sc_vocab_start] - ) - if tp_size > 1: - soft_embeds = torch.ops.vllm.all_reduce(soft_embeds, group_name=tp_group_name) - soft_embeds = soft_embeds * normalizer - sc_embeds[decode_slots] = (soft_embeds * sc_keep).to(sc_embeds.dtype) + if compute_sc: + assert probs is not None + # Self-conditioning soft embed = probs @ embed_tokens.weight. Under + # tensor parallelism the embedding is vocab-sharded ([vocab/tp, + # hidden]) while probs spans the full vocab, so each rank multiplies + # its local vocab slice [sc_vocab_start, sc_vocab_end) and the + # partials are summed across ranks. + local_probs = probs[..., sc_vocab_start:sc_vocab_end].to(embed_weight.dtype) + soft_embeds = torch.matmul( + local_probs, embed_weight[: sc_vocab_end - sc_vocab_start] + ) + if tp_size > 1: + soft_embeds = torch.ops.vllm.all_reduce( + soft_embeds, group_name=tp_group_name + ) + soft_embeds = soft_embeds * normalizer + # A pinned position holds its seed token as input. Zero its soft embed + # too, or the model's own prediction there reaches the next step + # through self-conditioning. The buffer is fp32 while the embedding is + # in the model dtype: compiled code casts on the store but eager does + # not, and the step runs eager once torch.compile hits its recompile + # limit. + sc_pin = (~pin_mask[decode_slots]).unsqueeze(-1) + sc_embeds[decode_slots] = (soft_embeds * sc_keep * sc_pin).to(sc_embeds.dtype) + else: + # Every slot in this tile ends after this step, so the soft embed + # would never be read. The matmul is a full pass over the vocabulary + # matrix, so skip it. + sc_embeds[decode_slots] = 0 # Overwrite canvas with argmax for newly converged denoise requests newly_converged = (converged & is_denoise).unsqueeze(1) canvas[decode_slots] = torch.where( newly_converged, argmax_canvas[decode_slots], canvas[decode_slots] ) + is_encoder_phase[decode_slots] &= ~read_only[decode_slots] # ---- Phase 7: Copy canvas → draft_tokens for all slots ---- draft_tokens[all_slots, :CL] = canvas[all_slots] - return scaled - class DiffusionGemmaRequestStates: """Pre-allocated GPU tensors for DiffusionGemma per-request state. @@ -628,6 +720,38 @@ class DiffusionGemmaRequestStates: # Per-slot confidence flag, set by the sampler each step. self.confident = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + # Per-slot step cap, lowered per request from extra_args. + self.max_steps = torch.full( + (max_num_reqs,), max_denoising_steps, dtype=torch.int32, device=device + ) + # Seed canvases replace the random canvas after prefill. The host-side + # sets gate the seed and read-only work so plain generation skips it. + self.seed_canvas = torch.zeros( + max_num_reqs, canvas_length, dtype=torch.int64, device=device + ) + self.has_seed = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + self.seeded_slots: set[int] = set() + # Pinned positions keep their seed value on every denoise step, so a + # multi-step read denoises the canvas the request seeded. + self.pin_mask = torch.zeros( + max_num_reqs, canvas_length, dtype=torch.bool, device=device + ) + # Read-only slots emit on their converging step and skip the commit + # forward. + self.read_only = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + # Constrained reads: slot -> the allowed ids (the request's + # logprob_token_ids). A step whose decode slots all share one set + # runs the unembedding, sampler and self-conditioning over that set. + self.constrained: dict[int, tuple[int, ...]] = {} + self.step_allowed: torch.Tensor | None = None + self._allowed_cache: dict[tuple[int, ...], torch.Tensor] = {} + self.read_only_slots: set[int] = set() + # Slots capped at one denoise step never consume a soft embed. + self.single_step_slots: set[int] = set() + # Per-slot canvas width, at most canvas_length. The scheduler schedules + # this many draft tokens for the slot and the sampler pads the rest. + self.canvas_width_np = np.full(max_num_reqs, canvas_length, dtype=np.int32) + # Per-slot self-conditioning soft embedding (probs @ embed_weight) from # the previous denoise step. Storing the [.., hidden] soft embed instead # of the full [.., vocab] distribution shrinks this buffer by @@ -657,11 +781,83 @@ class DiffusionGemmaRequestStates: self.step[slot_idx].fill_(0) self.accepted_canvas_history_len[slot_idx].fill_(0) self.self_conditioning_embeds[slot_idx] = 0 + self.max_steps[slot_idx].fill_(self.max_denoising_steps) + self.has_seed[slot_idx].fill_(False) + self.seeded_slots.discard(slot_idx) + self.pin_mask[slot_idx].fill_(False) + self.read_only[slot_idx].fill_(False) + self.read_only_slots.discard(slot_idx) + self.single_step_slots.discard(slot_idx) + self.constrained.pop(slot_idx, None) + self.canvas_width_np[slot_idx] = self.canvas_length def remove_request(self, slot_idx: int) -> None: + # add_request resets the GPU flags before a slot is reused. The host + # sets gate work on every step, so they clear here. self.is_encoder_phase[slot_idx].fill_(False) self.accepted_canvas_history_len[slot_idx].fill_(0) self.self_conditioning_embeds[slot_idx] = 0 + self.seeded_slots.discard(slot_idx) + self.read_only_slots.discard(slot_idx) + self.single_step_slots.discard(slot_idx) + self.constrained.pop(slot_idx, None) + + def set_seed_canvas(self, slot_idx: int, ids: list[int]) -> None: + """``ids`` covers the slot's canvas width; positions past it are never + scheduled.""" + self.seed_canvas[slot_idx, : len(ids)] = async_tensor_h2d( + ids, dtype=torch.int64, device=self.device + ) + self.has_seed[slot_idx].fill_(True) + self.seeded_slots.add(slot_idx) + + def set_pins(self, slot_idx: int, positions: list[int]) -> None: + """Hold ``positions`` of the slot's seed canvas through every denoise + step. Validated against the request's canvas width upstream.""" + self.pin_mask[slot_idx].fill_(False) + self.pin_mask[ + slot_idx, async_tensor_h2d(positions, dtype=torch.int64, device=self.device) + ] = True + + def allowed_tensor(self, ids: tuple[int, ...]) -> torch.Tensor: + """``ids`` as an int64 tensor on the device, built once per tuple.""" + t = self._allowed_cache.get(ids) + if t is None: + t = torch.tensor(ids, dtype=torch.int64, device=self.device) + if len(self._allowed_cache) > 64: + self._allowed_cache.clear() + self._allowed_cache[ids] = t + return t + + def batch_allowed(self, slots: list[int]) -> torch.Tensor | None: + """The allowed ids shared by every one of ``slots``, or None. + + None means the step runs over the full vocabulary. The sampler then + masks each constrained slot's logit rows to its own set, so a + constrained request reads the same way whoever shares its batch. + """ + if not slots or not self.constrained: + return None + first = self.constrained.get(slots[0]) + if first is None or any(self.constrained.get(s) != first for s in slots[1:]): + return None + return self.allowed_tensor(first) + + def set_read_only(self, slot_idx: int) -> None: + self.read_only[slot_idx].fill_(True) + self.read_only_slots.add(slot_idx) + + def apply_seed_canvases( + self, slots_np: np.ndarray, slots_gpu: torch.Tensor + ) -> None: + """Replace the canvas of every seeded slot among ``slots_gpu``.""" + if self.seeded_slots.isdisjoint(slots_np.tolist()): + return + self.canvas[slots_gpu] = torch.where( + self.has_seed[slots_gpu, None], + self.seed_canvas[slots_gpu], + self.canvas[slots_gpu], + ) class DiffusionGemmaModelState(ModelState): @@ -708,6 +904,8 @@ class DiffusionGemmaModelState(ModelState): # the current step, so we add 1 to match the same behavior. stability_threshold=self.gen_config["stability_threshold"] + 1, ) + # compute_logits reads the step's shared allowed set from here. + self.model.diffusion_states = self.diffusion_states self._req_id_to_index: dict[str, int] = {} # Persistent buffer for per-request causal flags, updated in-place @@ -859,10 +1057,14 @@ class DiffusionGemmaModelState(ModelState): inputs_embeds[:num_tokens].copy_(self.model.embed_input_ids(input_ids)) # Apply self-conditioning ONLY for denoising decode requests. + states.step_allowed = None if input_batch.num_draft_tokens > 0 and self._req_id_to_index: slots_np = input_batch.idx_mapping_np[:num_reqs] num_logits_np = np.diff(input_batch.cu_num_logits_np[: num_reqs + 1]) is_decode_indices_np = np.where(num_logits_np > 0)[0] + states.step_allowed = states.batch_allowed( + slots_np[is_decode_indices_np].tolist() + ) self._apply_self_conditioning( slots_np[is_decode_indices_np], is_decode_indices_np, @@ -1030,6 +1232,43 @@ class DiffusionSampler: self._pending_logprobs.pop(req_idx, None) self.sampling_states.add_request(req_idx, sampling_params) self.logprob_token_ids_state.add_request(req_idx, sampling_params) + extra = getattr(sampling_params, "extra_args", None) or {} + states = self.diffusion_states + cap = extra.get("diffusion_max_steps") + if cap is not None: + cap = max(1, min(int(cap), states.max_denoising_steps)) + states.max_steps[req_idx].fill_(cap) + if cap == 1: + states.single_step_slots.add(req_idx) + width = extra.get("diffusion_canvas_length") + if width: + states.canvas_width_np[req_idx] = max( + 1, min(int(width), self.canvas_length) + ) + width = int(states.canvas_width_np[req_idx]) + seed = extra.get("diffusion_seed_canvas") + if seed is not None: + if len(seed) != width: + raise ValueError( + f"diffusion_seed_canvas must hold exactly {width} ids, " + f"got {len(seed)}" + ) + states.set_seed_canvas(req_idx, seed) + pins = extra.get("diffusion_pinned") + if pins and seed is not None: + states.set_pins(req_idx, [int(p) for p in pins]) + if extra.get("diffusion_read_only"): + states.set_read_only(req_idx) + if extra.get("diffusion_constrained"): + ids = list(getattr(sampling_params, "logprob_token_ids", None) or []) + if not ids: + raise ValueError( + "diffusion_constrained needs logprob_token_ids: they are " + "the allowed set." + ) + if self.tp_size > 1: + raise ValueError("diffusion_constrained needs tensor parallel 1.") + states.constrained[req_idx] = tuple(int(t) for t in ids) def apply_staged_writes(self) -> None: self.sampling_states.apply_staged_writes() @@ -1071,6 +1310,7 @@ class DiffusionSampler: ps.astype(np.int64), device=states.is_encoder_phase.device ) states.init_canvas(ps_gpu) + states.apply_seed_canvases(ps, ps_gpu) self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[ ps_gpu ] @@ -1151,7 +1391,6 @@ class DiffusionSampler: # --- CPU/NumPy setup (outside compile): split decode vs prefill, init # canvas for any new prefills, and stage decode slot indices to GPU. --- states = self.diffusion_states - CL = self.canvas_length slots_np = input_batch.idx_mapping_np[:num_reqs] per_req_nlogits_np = np.diff(input_batch.cu_num_logits_np[: num_reqs + 1]) @@ -1194,18 +1433,12 @@ class DiffusionSampler: if top_k is not None or top_p is not None: logits = apply_top_k_top_p(logits.float(), top_k, top_p) - # Pad any truncated canvas back to CL so the uniform-CL sampler math - # holds. Phantom (padded) positions are zeroed → uniform logits → high - # entropy (no premature convergence) and argmax 0 (stable); they are - # never committed (num_sampled == real length). masked_fill (not - # multiply) so -inf entries from top_k/top_p filtering above don't - # turn phantom rows into NaN. - if num_decode > 0 and valid_canvas_len_np.min() < CL: - ar = torch.arange(CL, device=device) - starts = valid_canvas_len.cumsum(0) - valid_canvas_len # row offset per req - valid = ar.unsqueeze(0) < valid_canvas_len.unsqueeze(1) # [num_decode, CL] - src = (starts.unsqueeze(1) + ar.unsqueeze(0)).clamp_max(logits.shape[0] - 1) - logits = logits[src.reshape(-1)].masked_fill_(~valid.reshape(-1, 1), 0) + # Where each decode request's rows start in the flat logits. Tiles + # below gather and pad a request to the tile's width. + row_starts_np = np.concatenate( + ([0], np.cumsum(valid_canvas_len_np)[:-1]) + ).astype(np.int64) + row_starts = async_tensor_h2d(row_starts_np, device=device) # Clear once: the tiled loop below only scatters its own decode slots, # so it must not re-clear earlier tiles' writes. @@ -1220,6 +1453,30 @@ class DiffusionSampler: # since it mutates is_encoder_phase (commit→False, converge→True). is_committing = states.is_encoder_phase[decode_slots].clone() + # Constrained step: logits are [rows, K] over the shared allowed set, + # so the self-conditioning matmul only needs those K embedding rows. + allowed = states.step_allowed + embed_weight = self.embed_weight + vocab_size = self.vocab_size + sc_vocab_start, sc_vocab_end = self.sc_vocab_start, self.sc_vocab_end + if allowed is not None: + assert logits.shape[-1] == allowed.numel() + embed_weight = self.embed_weight[allowed] + sc_vocab_start, sc_vocab_end = 0, allowed.numel() + elif states.constrained and num_decode > 0: + # The decode slots do not share one set, so this step runs over + # the full vocabulary. Mask each constrained request's rows to + # its own set so it reads the same as on the shared path. The + # helper masks a copy, since the runner owns `logits`: one + # full-vocab copy per step, as for top_k/top_p above. + per_row = [ + None if ids is None else states.allowed_tensor(ids) + for ids in (states.constrained.get(s) for s in decode_slots_np.tolist()) + ] + logits = _mask_rows_to_allowed( + logits, row_starts_np, valid_canvas_len_np, per_row + ) + slots_np = input_batch.idx_mapping_np[:num_reqs] max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np) # Requests may ask for specific token ids' logprobs instead of, or as @@ -1228,125 +1485,216 @@ class DiffusionSampler: want_logprobs = max_num_logprobs >= 0 or max_token_ids > 0 num_logprobs = max(max_num_logprobs, 0) - # Sample over the [num_decode * CL, vocab] logits. The fp32 pipeline in - # _compiled_sample_step keeps several live [group * CL, vocab] copies, so - # size each tile to a fraction of free memory to bound the transient at - # high concurrency. Tiling is bit-identical to a single pass. - group = max(num_decode, 1) - if num_decode > 0: - free, _ = current_platform.mem_get_info() - # ~10 transient fp32 copies of [group * CL, vocab] inside the step - # (eager peaks at ~8; pad for allocator overhead and small tensors). - bytes_per_req = CL * self.vocab_size * 4 * 10 - budget = int(free * 0.5) // max(bytes_per_req, 1) - group = max(1, min(num_decode, budget)) + # Sample per tile. Decode requests are grouped by canvas width and each + # tile runs the compiled step at that width over [:, :W] views of the + # state, so a narrow read pays for its own rows rather than the served + # canvas. Widths ascend, so the last tile's canvas-to-draft copy (over + # all slots) is the widest. The fp32 pipeline keeps several live + # [tile * W, vocab] copies, so a tile is also bounded by free memory. + widths_np = states.canvas_width_np[decode_slots_np] + order = np.argsort(widths_np, kind="stable") + free = current_platform.mem_get_info()[0] if num_decode > 0 else 0 + run_start = 0 + while run_start < num_decode: + W = int(widths_np[order[run_start]]) + run_end = run_start + while run_end < num_decode and widths_np[order[run_end]] == W: + run_end += 1 + # Transient [tile * W, vocab] tensors: the softmax in the embedding + # dtype for self-conditioning, plus the fp32 scaled logits when + # logprobs are wanted (pad for allocator overhead). + budget = max(1, int(free * 0.5) // max(W * self.vocab_size * 4 * 3, 1)) + for t0 in range(run_start, run_end, budget): + sel_np = order[t0 : min(t0 + budget, run_end)] + n = len(sel_np) + sel = async_tensor_h2d(sel_np.astype(np.int64), device=device) + tile_slots = decode_slots[sel] + tile_valid = valid_canvas_len[sel] + tile_valid_np = valid_canvas_len_np[sel_np] + contiguous = bool((sel_np == np.arange(sel_np[0], sel_np[0] + n)).all()) + if contiguous and tile_valid_np.min() == W: + r0 = int(row_starts_np[sel_np[0]]) + tile_logits = logits[r0 : r0 + n * W] + else: + # Pad each request to W. Phantom positions are zeroed: + # uniform logits, high entropy, argmax 0, never committed. + # masked_fill, not multiply, so -inf from top_k/top_p + # filtering above cannot turn a phantom row into NaN. + ar = torch.arange(W, device=device) + src = (row_starts[sel].unsqueeze(1) + ar.unsqueeze(0)).clamp_max( + logits.shape[0] - 1 + ) + valid = ar.unsqueeze(0) < tile_valid.unsqueeze(1) + tile_logits = logits[src.reshape(-1)].masked_fill_( + ~valid.reshape(-1, 1), 0 + ) + compute_sc = ( + not states.single_step_slots + or not states.single_step_slots.issuperset( + decode_slots_np[sel_np].tolist() + ) + ) - for start_req in range(0, num_decode, group): - end_req = min(start_req + group, num_decode) - tile = slice(start_req, end_req) - tile_slots = decode_slots[tile] + temp = _denoise_temperature( + states.step, + tile_slots, + float(states.max_denoising_steps), + self.t_min, + self.t_max, + ) + probs_dtype = self.embed_weight.dtype if compute_sc else None + if tile_logits.is_cuda: + seed = int(torch.randint(0, 2**31 - 1, (1,)).item()) + stats = sample_row_stats(tile_logits, temp, W, seed, probs_dtype) + else: + stats = sample_row_stats_reference( + tile_logits, temp, W, probs_dtype + ) + argmax_rows, sample_rows, entropy_rows, probs_rows = stats + if allowed is not None: + # K-space picks back to token ids. The softmax stays K wide + # for the K-row self-conditioning matmul. + argmax_rows = allowed[argmax_rows] + sample_rows = allowed[sample_rows] + probs = probs_rows.view(n, W, -1) if probs_rows is not None else None + # Only the logprob stash reads the scaled logits. + scaled = None + if want_logprobs: + scaled = tile_logits.float().view(n, W, -1) / temp[ + :, None, None + ].clamp(min=1e-10) - scaled = _compiled_sample_step( - logits[start_req * CL : end_req * CL], - tile_slots, - decode_idx[tile], - all_slots, - valid_canvas_len[tile], - # State - states.canvas, - states.argmax_canvas, - states.step, - states.is_encoder_phase, - states.confident, - states.self_conditioning_embeds, - self.embed_weight, - self.normalizer, - states.accepted_canvas_history, - states.accepted_canvas_history_len, - # Output - sampled, - num_sampled, - self.req_states.draft_tokens, - # Config - max_denoising_steps=float(states.max_denoising_steps), - t_min=self.t_min, - t_max=self.t_max, - confidence_threshold=self.confidence_threshold, - vocab_size=self.vocab_size, - CL=CL, - ST=states.stability_threshold, - entropy_bound=self.entropy_bound, - sc_vocab_start=self.sc_vocab_start, - sc_vocab_end=self.sc_vocab_end, - tp_size=self.tp_size, - tp_group_name=self.tp_group_name, - ) + _compiled_sample_step( + sample_rows.view(n, W), + argmax_rows.view(n, W), + entropy_rows.view(n, W), + probs, + tile_slots, + decode_idx[sel], + all_slots, + tile_valid, + # State, viewed at this tile's width + states.canvas[:, :W], + states.argmax_canvas[:, :W], + states.step, + states.is_encoder_phase, + states.confident, + states.self_conditioning_embeds[:, :W], + embed_weight, + self.normalizer, + states.accepted_canvas_history[:, :, :W], + states.accepted_canvas_history_len, + states.max_steps, + states.pin_mask[:, :W], + states.seed_canvas[:, :W], + states.read_only, + # Output + sampled[:, :W], + num_sampled, + self.req_states.draft_tokens, + # Config + confidence_threshold=self.confidence_threshold, + vocab_size=vocab_size, + CL=W, + ST=states.stability_threshold, + entropy_bound=self.entropy_bound, + sc_vocab_start=sc_vocab_start, + sc_vocab_end=sc_vocab_end, + tp_size=self.tp_size, + tp_group_name=self.tp_group_name, + compute_sc=compute_sc, + ) - # Logprobs for denoise steps that just converged (is_encoder_phase - # flipped False→True), stashed per tile so `scaled` is freed each tile. - if want_logprobs: - converged_mask = states.is_encoder_phase[tile_slots] - just_converged = converged_mask & ~is_committing[tile] - if just_converged.any(): - flat_logits = scaled.reshape(-1, scaled.shape[-1]) - argmax_tokens = scaled.argmax(dim=-1) - for local_idx in just_converged.nonzero(as_tuple=True)[0]: - li = local_idx.item() - slot = tile_slots[local_idx] - # Stash only the real canvas positions (== CL unless this - # canvas was truncated near max_model_len); padded tail - # positions are never emitted. - k_i = int(valid_canvas_len_np[start_req + li]) - pos = li * CL - per_req_ids = max_token_ids > 0 - self._pending_logprobs[slot.item()] = compute_topk_scores( - flat_logits[pos : pos + k_i], - num_logprobs, - argmax_tokens[local_idx][:k_i], - logprob_token_ids_state=( - self.logprob_token_ids_state if per_req_ids else None - ), - # every row of this stash belongs to one slot - expanded_idx_mapping=( - torch.full( - (k_i,), - slot.item(), - dtype=torch.int32, - device=flat_logits.device, + # Stash newly converged logprobs, including reads that emit now. + if want_logprobs: + converged_mask = states.is_encoder_phase[tile_slots] | ( + num_sampled[decode_idx[sel]] > 0 + ) + just_converged = converged_mask & ~is_committing[sel] + if just_converged.any(): + assert scaled is not None + flat_logits = scaled.reshape(-1, scaled.shape[-1]) + argmax_tokens = scaled.argmax(dim=-1) + raw_flat: torch.Tensor | None = None + for local_idx in just_converged.nonzero(as_tuple=True)[0]: + li = local_idx.item() + slot = tile_slots[local_idx].item() + # Stash only the real canvas positions; padded tail + # positions are never emitted. + k_i = int(tile_valid_np[li]) + pos = li * W + src = flat_logits + if slot in states.read_only_slots: + # Read-only slots report logprobs at temperature 1. + # The schedule-tempered logits share the argmax. + if raw_flat is None: + raw_flat = tile_logits.float() + src = raw_flat + if allowed is not None: + # Column j of the K-space logits is allowed[j]: + # report every allowed id, renormalized over + # the set, with the argmax in column 0. + rows = src[pos : pos + k_i] + lp = rows.log_softmax(dim=-1) + am = argmax_tokens[local_idx][:k_i] + sel_lp = lp.gather(1, am.unsqueeze(1)) + # Same convention as _ranks_kernel: 1-based, + # ties count, so the rank is the number of + # columns at or above the selected one. Out + # of set ids have zero probability, so the + # rank within the set is the real rank. + ranks = (lp >= sel_lp).sum(dim=1) + self._pending_logprobs[slot] = LogprobsTensors( + logprob_token_ids=torch.cat( + ( + allowed[am].unsqueeze(1), + allowed.unsqueeze(0).expand(k_i, -1), + ), + dim=1, + ), + logprobs=torch.cat((sel_lp, lp), dim=1), + selected_token_ranks=ranks.to(torch.int64), ) - if per_req_ids - else None - ), - max_per_req_token_ids=max_token_ids, - logits_mode=self.logits_mode, - ) + continue + per_req_ids = max_token_ids > 0 + self._pending_logprobs[slot] = compute_topk_scores( + src[pos : pos + k_i], + num_logprobs, + argmax_tokens[local_idx][:k_i], + logprob_token_ids_state=( + self.logprob_token_ids_state + if per_req_ids + else None + ), + # every row of this stash belongs to one slot + expanded_idx_mapping=( + torch.full( + (k_i,), slot, dtype=torch.int32, device=device + ) + if per_req_ids + else None + ), + max_per_req_token_ids=max_token_ids, + logits_mode=self.logits_mode, + ) + run_start = run_end - # Commit steps: is_committing was True at entry. Reassemble previously - # stashed logprobs and attach to SamplerOutput. + # Only emitting requests consume their stashed logprobs. logprobs_tensors = None - if want_logprobs and is_committing.any() and self._pending_logprobs: - committing_slots = set( - decode_slots_np[is_committing.cpu().numpy()].tolist() - ) - parts_ids, parts_lp, parts_ranks = [], [], [] + if want_logprobs and self._pending_logprobs: + emitting_slots = set(slots_np[num_sampled.cpu().numpy() > 0].tolist()) + parts: list[LogprobsTensors] = [] cu_gen: list[int] = [] flat_offset = 0 for i in range(num_reqs): cu_gen.append(flat_offset) slot = int(slots_np[i]) - if slot in committing_slots and slot in self._pending_logprobs: + if slot in emitting_slots and slot in self._pending_logprobs: lp = self._pending_logprobs.pop(slot) - parts_ids.append(lp.logprob_token_ids) - parts_lp.append(lp.logprobs) - parts_ranks.append(lp.selected_token_ranks) + parts.append(lp) flat_offset += lp.logprobs.shape[0] - if parts_ids: - logprobs_tensors = LogprobsTensors( - logprob_token_ids=torch.cat(parts_ids), - logprobs=torch.cat(parts_lp), - selected_token_ranks=torch.cat(parts_ranks), - cu_num_generated_tokens=cu_gen, - ) + if parts: + logprobs_tensors = _concat_logprob_stashes(parts, cu_gen) return self._build_output( input_batch, diff --git a/vllm/model_executor/models/diffusion_gemma_sampler.py b/vllm/model_executor/models/diffusion_gemma_sampler.py new file mode 100644 index 0000000000..9b15233791 --- /dev/null +++ b/vllm/model_executor/models/diffusion_gemma_sampler.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""One-pass row statistics for the DiffusionGemma denoise sampler. + +The step needs, for every canvas position, the argmax of the +temperature-scaled logits, a Gumbel-max sample from them, the entropy of +their softmax, and the softmax itself in the model dtype for the +self-conditioning matmul. As PyTorch ops these are several passes over a +``[rows, vocab]`` fp32 tensor plus a same-sized noise tensor. The Triton +kernel reads each row once with an online max and sum, draws the noise +inline, and writes the probabilities once. +""" + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit +def _row_stats_kernel( + logits_ptr, + logits_stride, + temps_ptr, + argmax_ptr, + sample_ptr, + entropy_ptr, + probs_ptr, + probs_stride, + seed, + V, + CL, + WRITE_PROBS: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + temp = tl.load(temps_ptr + row // CL).to(tl.float32) + inv = 1.0 / tl.maximum(temp, 1e-10) + noise = (temp > 0).to(tl.float32) + base = logits_ptr + row.to(tl.int64) * logits_stride + rng_base = row.to(tl.int64) * V + + # The running max stays in raw logit units and the temperature is applied + # to differences from it. Scaling the row first breaks at a zero + # temperature: the row is multiplied by 1e10, fp32 spacing there is about + # a thousand, and exp(x - max) becomes 0 or inf for the max element itself + # whenever the two sides round differently. + m = float("-inf") # running max of the raw row + z = 0.0 # sum exp(d), d = (x - m) * inv + s = 0.0 # sum d exp(d) + best = float("-inf") + best_idx = 0 + best_noisy = float("-inf") + best_noisy_idx = 0 + for start in range(0, V, BLOCK): + offs = start + tl.arange(0, BLOCK) + mask = offs < V + x = tl.load(base + offs, mask=mask, other=float("-inf")).to(tl.float32) + + block_max = tl.max(x, 0) + new_m = tl.maximum(m, block_max) + # Until a block holds a finite logit the running max stays -inf, and + # top_k/top_p leave whole blocks masked. x - new_m would then be + # -inf - -inf = NaN and poison z for the rest of the row, so subtract + # 0 instead: every d is still -inf and every e 0. + safe_m = tl.where(new_m > float("-inf"), new_m, 0.0) + # Rescale the running sums to the new max. While m is -inf, z and s + # are 0, so alpha and the shift are zeroed rather than computed from + # -inf - -inf. + alpha = tl.where(m > float("-inf"), tl.exp((m - safe_m) * inv), 0.0) + shift = tl.where(z > 0, (m - safe_m) * inv * z, 0.0) + d = (x - safe_m) * inv + e = tl.exp(d) + z = z * alpha + tl.sum(e, 0) + # A -inf logit (top_k/top_p, or the row's padding) has d = -inf and + # e = 0, and -inf * 0 is NaN. The where drops it from the sum. + s = (s + shift) * alpha + tl.sum(tl.where(e > 0, d * e, 0.0), 0) + m = new_m + + block_idx = tl.argmax(x, 0) + take = block_max > best + best_idx = tl.where(take, start + block_idx, best_idx) + best = tl.where(take, block_max, best) + + u = tl.rand(seed, rng_base + offs) + u = tl.maximum(u, 1e-20) + gumbel = -tl.log(-tl.log(u)) + noisy = tl.where(mask, x * inv + gumbel * noise, float("-inf")) + noisy_max = tl.max(noisy, 0) + noisy_idx = tl.argmax(noisy, 0) + take_n = noisy_max > best_noisy + best_noisy_idx = tl.where(take_n, start + noisy_idx, best_noisy_idx) + best_noisy = tl.where(take_n, noisy_max, best_noisy) + + # With d = (x - m) * inv: p = e^d / z, log Z = log z in d units, and + # H = -sum p (d - log z) = log z - s / z. + # tl.store casts to the pointee dtype. mypy type-checks the kernel body + # as Python and rejects .to() on the int accumulators. + tl.store(argmax_ptr + row, best_idx) + tl.store(sample_ptr + row, best_noisy_idx) + tl.store(entropy_ptr + row, tl.log(z) - s / z) + + if WRITE_PROBS: + out = probs_ptr + row.to(tl.int64) * probs_stride + for start in range(0, V, BLOCK): + offs = start + tl.arange(0, BLOCK) + mask = offs < V + x = tl.load(base + offs, mask=mask, other=float("-inf")).to(tl.float32) + p = tl.exp((x - m) * inv) / z + tl.store(out + offs, p.to(probs_ptr.dtype.element_ty), mask=mask) + + +def sample_row_stats( + logits: torch.Tensor, + temps: torch.Tensor, + canvas_len: int, + seed: int, + probs_dtype: torch.dtype | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: + """Argmax, Gumbel-max sample, entropy and (optionally) softmax per row. + + ``logits`` is ``[rows, vocab]`` and row ``i`` uses temperature + ``temps[i // canvas_len]``. A zero temperature is greedy: the sample is + the argmax, and the entropy is the reference's, which clamps the + temperature at 1e-10. + """ + rows, vocab = logits.shape + device = logits.device + argmax = torch.empty(rows, dtype=torch.int64, device=device) + sample = torch.empty(rows, dtype=torch.int64, device=device) + entropy = torch.empty(rows, dtype=torch.float32, device=device) + if probs_dtype is not None: + probs = torch.empty(rows, vocab, dtype=probs_dtype, device=device) + probs_arg, probs_stride = probs, probs.stride(0) + else: + probs = None + probs_arg, probs_stride = entropy, 0 + if rows == 0: + return argmax, sample, entropy, probs + _row_stats_kernel[(rows,)]( + logits, + logits.stride(0), + temps, + argmax, + sample, + entropy, + probs_arg, + probs_stride, + seed, + vocab, + canvas_len, + WRITE_PROBS=probs is not None, + BLOCK=4096, + num_warps=8, + ) + return argmax, sample, entropy, probs + + +def sample_row_stats_reference( + logits: torch.Tensor, + temps: torch.Tensor, + canvas_len: int, + probs_dtype: torch.dtype | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: + """The PyTorch form of the same statistics, for tests and non-CUDA runs.""" + temp = temps.repeat_interleave(canvas_len).float()[:, None] + scaled = logits.float() / temp.clamp(min=1e-10) + u = torch.rand_like(scaled).clamp(min=1e-20) + noisy = scaled + (-torch.log(-torch.log(u))) * (temp > 0).float() + log_probs = scaled.log_softmax(dim=-1) + probs = log_probs.exp() + # Masked (-inf) columns: 0 * -inf is NaN, so they are dropped from the sum. + entropy = -torch.where(probs > 0, probs * log_probs, 0.0).sum(dim=-1) + out_probs = probs.to(probs_dtype) if probs_dtype is not None else None + return scaled.argmax(dim=-1), noisy.argmax(dim=-1), entropy, out_probs diff --git a/vllm/model_executor/models/eagle2_5_vl.py b/vllm/model_executor/models/eagle2_5_vl.py index d4ee31aaf7..257907e659 100644 --- a/vllm/model_executor/models/eagle2_5_vl.py +++ b/vllm/model_executor/models/eagle2_5_vl.py @@ -8,7 +8,7 @@ from typing import Annotated, Literal, TypeAlias import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.quantization import QuantizationConfig @@ -193,7 +193,7 @@ class Eagle2_5_VLForConditionalGeneration( def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str, ): @@ -220,7 +220,7 @@ class Eagle2_5_VLForConditionalGeneration( prefix=prefix, ) - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1(self, config: PreTrainedConfig) -> nn.Module: """Initialize MLP projection layer.""" vit_hidden_size = config.vision_config.hidden_size llm_hidden_size = config.text_config.hidden_size diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index 2683e94f51..53d1ba6827 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -29,7 +29,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config @@ -41,12 +41,15 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, + MoERunner, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -116,7 +119,7 @@ class Ernie4_5_MoeMLP(nn.Module): class Ernie4_5_MoeMoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", enable_eplb: bool = False, @@ -150,12 +153,11 @@ class Ernie4_5_MoeMoE(nn.Module): f"the number of experts {config.moe_num_experts}." ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.moe_num_experts, - bias=False, + out_dtype=torch.float32, params_dtype=torch.float32, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -199,7 +201,7 @@ class Ernie4_5_MoeMoE(nn.Module): hidden_dim = hidden_states.shape[-1] hidden_states = hidden_states.view(-1, hidden_dim) - router_logits, _ = self.gate(hidden_states.to(dtype=torch.float32)) + router_logits, _ = self.gate(hidden_states) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits @@ -303,7 +305,7 @@ class Ernie4_5_MoeAttention(nn.Module): class Ernie4_5_MoeDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index a800458bbf..0a632b66e6 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -881,7 +881,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo): return self.ctx.model_config.hf_config def get_hf_processor(self, **kwargs: object): - return self.ctx.get_hf_processor(use_fast=True, **kwargs) + return self.ctx.get_hf_processor(**kwargs) def get_image_processor(self, **kwargs: object): return self.get_hf_processor(**kwargs).image_processor diff --git a/vllm/model_executor/models/ernie45_vl_moe.py b/vllm/model_executor/models/ernie45_vl_moe.py index ccc4b9b868..4965bcdcb3 100644 --- a/vllm/model_executor/models/ernie45_vl_moe.py +++ b/vllm/model_executor/models/ernie45_vl_moe.py @@ -29,7 +29,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig # from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -192,7 +192,7 @@ class Ernie4_5_VLMoeAttention(nn.Module): class Ernie4_5_VLMoeMoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -385,7 +385,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): class Ernie4_5_VLMoeDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/exaone.py b/vllm/model_executor/models/exaone.py index fef05d386a..16f07746f3 100644 --- a/vllm/model_executor/models/exaone.py +++ b/vllm/model_executor/models/exaone.py @@ -30,7 +30,7 @@ from itertools import islice import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -104,7 +104,7 @@ class ExaoneGatedMLP(nn.Module): class ExaoneAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -191,7 +191,7 @@ class ExaoneAttention(nn.Module): class ExaoneBlockAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -228,7 +228,7 @@ class ExaoneBlockAttention(nn.Module): class ExaoneDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/exaone4_5.py b/vllm/model_executor/models/exaone4_5.py index b325c6a304..5a53e60b7d 100644 --- a/vllm/model_executor/models/exaone4_5.py +++ b/vllm/model_executor/models/exaone4_5.py @@ -311,7 +311,6 @@ class Exaone4_5_ProcessingInfo(Qwen2VLProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Exaone4_5_Processor: return self.ctx.get_hf_processor( Exaone4_5_Processor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index 4c19f1ed0f..d5b8b40662 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -20,7 +20,7 @@ from itertools import islice import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config @@ -30,11 +30,13 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -63,7 +65,7 @@ from .utils import ( class ExaoneMoe(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, swiglu_limit: float | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -84,11 +86,9 @@ class ExaoneMoe(nn.Module): f"the number of experts {config.num_experts}." ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -289,7 +289,7 @@ class ExaoneMoeAttention(nn.Module): class ExaoneMoeDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, is_mtp: bool = False, diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index be2e8edeb2..f2cb959e47 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -15,7 +15,7 @@ from transformers.models.gemma3n import ( Gemma3nTextConfig, Gemma3nVisionConfig, ) -from transformers.models.siglip import SiglipImageProcessorFast +from transformers.models.siglip import SiglipImageProcessor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -239,7 +239,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): processor.feature_extractor ) audio_len = audio_feature_extractor.fft_length - image_processor: SiglipImageProcessorFast = processor.image_processor + image_processor: SiglipImageProcessor = processor.image_processor img_width = image_processor.size.get("width", 224) img_height = image_processor.size.get("height", 224) diff --git a/vllm/model_executor/models/gemma4_dspark.py b/vllm/model_executor/models/gemma4_dspark.py index 6b6d5d2731..33b1a1aada 100644 --- a/vllm/model_executor/models/gemma4_dspark.py +++ b/vllm/model_executor/models/gemma4_dspark.py @@ -24,7 +24,11 @@ from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal -from .qwen3_dspark import DSparkMarkovHead, Qwen3DSparkForCausalLM +from .qwen3_dspark import ( + DSparkConfidenceHead, + DSparkMarkovHead, + Qwen3DSparkForCausalLM, +) from .utils import extract_layer_index, maybe_prefix @@ -187,6 +191,18 @@ class Gemma4DSparkModel(DFlashQwen3Model): config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", False): + with_markov = getattr(config, "confidence_head_with_markov", False) + input_dim = config.hidden_size + if with_markov: + input_dim += config.markov_rank + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) * self.normalizer @@ -295,9 +311,8 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM): params = dict(self.named_parameters()) params.update(dict(self.named_buffers())) loaded: set[str] = set() + includes_confidence_head = False for name, w in weights: - if "confidence_head" in name: - continue if "lm_head" not in name: name = "model." + name for pn, wn, shard in stacked: @@ -310,5 +325,9 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM): p = params[name] getattr(p, "weight_loader", default_weight_loader)(p, w) loaded.add(name) + if "confidence_head" in name: + includes_confidence_head = True + if not includes_confidence_head: + self.model.confidence_head = None self.model._build_fused_kv_buffers() return loaded diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index 3c72a148d5..5e5974e209 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -43,6 +43,7 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, ) from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, @@ -137,14 +138,12 @@ class Glm4MoE(nn.Module): f"Unsupported activation: {config.hidden_act}. " "Only silu is supported for now." ) - # NOTE In the transformers implementation, the gate isn't an nn.Linear, - # so we cannot use ReplicatedLinear here. - # See: https://github.com/huggingface/transformers/blob/v4.55.1/src/transformers/models/glm4_moe/modeling_glm4_moe.py#L260 - self.gate = nn.Linear( + self.gate = GateLinear( config.hidden_size, config.n_routed_experts, - bias=False, - dtype=torch.float32, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", ) self.gate.e_score_correction_bias = nn.Parameter( torch.empty(config.n_routed_experts, dtype=torch.float32) @@ -212,7 +211,7 @@ class Glm4MoE(nn.Module): hidden_states = hidden_states.view(-1, hidden_dim) # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states.to(dtype=torch.float32)) + router_logits, _ = self.gate(hidden_states) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index af5a1e113f..f5a82b7231 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -28,7 +28,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.fused_moe import ( @@ -68,7 +68,7 @@ from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix class SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, ) -> None: diff --git a/vllm/model_executor/models/glm4_moe_mtp.py b/vllm/model_executor/models/glm4_moe_mtp.py index 6d951e04ac..330f891d1c 100644 --- a/vllm/model_executor/models/glm4_moe_mtp.py +++ b/vllm/model_executor/models/glm4_moe_mtp.py @@ -29,7 +29,7 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, ParallelConfig, VllmConfig from vllm.model_executor.layers.fused_moe import ( @@ -60,7 +60,7 @@ from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix class SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, ) -> None: @@ -80,7 +80,7 @@ class SharedHead(nn.Module): class Glm4MoeMultiTokenPredictorLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index bd4b38defc..7c10aaa458 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -22,6 +22,7 @@ from vllm.distributed import ( from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig @@ -29,7 +30,6 @@ from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -377,11 +377,10 @@ class MLPBlock(torch.nn.Module): self.hidden_size = config.hidden_size self.experts_per_token = config.num_experts_per_tok self.world_size = dist.get_world_size() if dist.is_initialized() else 1 - self.router = ReplicatedLinear( + self.router = GateLinear( config.hidden_size, config.num_local_experts, bias=True, - quant_config=None, prefix=f"{prefix}.router", return_bias=False, ) diff --git a/vllm/model_executor/models/granite.py b/vllm/model_executor/models/granite.py index 73d9184f12..81ab62937e 100644 --- a/vllm/model_executor/models/granite.py +++ b/vllm/model_executor/models/granite.py @@ -36,7 +36,7 @@ from itertools import islice import torch from torch import nn from transformers import GraniteConfig -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -72,7 +72,7 @@ from .utils import ( def granite_layer_attn_params( - config: PretrainedConfig, layer_idx: int + config: PreTrainedConfig, layer_idx: int ) -> tuple[int | None, float, bool]: """Resolve one layer's sliding window, RoPE base and sink usage. diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 53cbe7df43..3ae85cfaac 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -31,7 +31,7 @@ from typing import Annotated import torch import torch.nn.functional as F from torch import nn -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -229,7 +229,7 @@ class GraniteSpeechDummyInputsBuilder( class GraniteSpeechEncoderProjector(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -287,7 +287,7 @@ class GraniteSpeechConformerFeedForward(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -323,7 +323,7 @@ class GraniteSpeechConformerAttention(nn.Module): for more details. """ - def __init__(self, config: PretrainedConfig, prefix: str = ""): + def __init__(self, config: PreTrainedConfig, prefix: str = ""): super().__init__() inner_dim = config.dim_head * config.num_heads @@ -429,7 +429,7 @@ class GraniteSpeechConformerConvModule(nn.Module): convolutional layers. """ - def __init__(self, config: PretrainedConfig, prefix: str = ""): + def __init__(self, config: PreTrainedConfig, prefix: str = ""): super().__init__() inner_dim = config.hidden_dim * config.conv_expansion_factor @@ -460,7 +460,7 @@ class GraniteSpeechConformerBlock(nn.Module): """Conformer block, consisting largely of linear layers, attention, and convolutional layers.""" - def __init__(self, config: PretrainedConfig, prefix: str = ""): + def __init__(self, config: PreTrainedConfig, prefix: str = ""): super().__init__() self.ff1 = GraniteSpeechConformerFeedForward(config, prefix=f"{prefix}.ff1") self.attn = GraniteSpeechConformerAttention(config, prefix=f"{prefix}.attn") @@ -486,7 +486,7 @@ class GraniteSpeechCTCEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, ): @@ -622,7 +622,7 @@ class GraniteSpeechForConditionalGeneration( def _build_encoder( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str, ) -> "GraniteSpeechCTCEncoder": diff --git a/vllm/model_executor/models/granite_speech_plus.py b/vllm/model_executor/models/granite_speech_plus.py index 7e695b7ed5..946f3edbcc 100644 --- a/vllm/model_executor/models/granite_speech_plus.py +++ b/vllm/model_executor/models/granite_speech_plus.py @@ -3,7 +3,7 @@ """Inference-only IBM Granite Speech Plus model.""" import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY @@ -68,7 +68,7 @@ class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGener def _build_encoder( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str, ) -> GraniteSpeechCTCEncoder: diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py index 3a9872a605..5b9aa886f5 100644 --- a/vllm/model_executor/models/granitemoe.py +++ b/vllm/model_executor/models/granitemoe.py @@ -29,7 +29,7 @@ from itertools import islice import torch from torch import nn -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -39,11 +39,13 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -94,12 +96,10 @@ class GraniteMoeMoE(nn.Module): self.is_sequence_parallel = is_sequence_parallel # Gate always runs at half / full precision for now. - self.gate = ReplicatedLinear( + self.gate = GateLinear( hidden_size, num_experts, - bias=False, params_dtype=params_dtype, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -141,7 +141,7 @@ class GraniteMoeMoE(nn.Module): class GraniteMoeAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 596f24a3eb..e1c59d4ea0 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -10,7 +10,7 @@ # -------------------------------------------------------- import torch -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY @@ -158,7 +158,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn class H2OVLChatModel(InternVLChatModel): def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, diff --git a/vllm/model_executor/models/hrm_text.py b/vllm/model_executor/models/hrm_text.py index e5140bc035..2f11a2c454 100644 --- a/vllm/model_executor/models/hrm_text.py +++ b/vllm/model_executor/models/hrm_text.py @@ -25,7 +25,7 @@ from typing import Literal import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -122,7 +122,7 @@ class HrmTextAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_idx_in_stack: int, stack_kind: Literal["L", "H"], cache_config: CacheConfig | None = None, @@ -245,7 +245,7 @@ class HrmTextAttention(nn.Module): class HrmTextDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_idx_in_stack: int, stack_kind: Literal["L", "H"], cache_config: CacheConfig | None = None, @@ -310,7 +310,7 @@ class HrmTextStack(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, stack_kind: Literal["L", "H"], cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py index 248ae04747..2e24cf65e8 100644 --- a/vllm/model_executor/models/hy_v3.py +++ b/vllm/model_executor/models/hy_v3.py @@ -31,7 +31,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config @@ -220,7 +220,7 @@ class HYV3MoEFused(nn.Module): class HYV3Attention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -365,7 +365,7 @@ class HYV3Attention(nn.Module): class HYV3DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/hy_v3_mtp.py b/vllm/model_executor/models/hy_v3_mtp.py index aa9c2f35b5..06f6f27089 100644 --- a/vllm/model_executor/models/hy_v3_mtp.py +++ b/vllm/model_executor/models/hy_v3_mtp.py @@ -29,7 +29,7 @@ from collections.abc import Callable, Iterable import regex as re import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.model_executor.layers.fused_moe import ( @@ -59,7 +59,7 @@ from .utils import ( ) -def _is_moe(config: PretrainedConfig) -> bool: +def _is_moe(config: PreTrainedConfig) -> bool: return bool( getattr(config, "num_experts", None) and ( @@ -69,7 +69,7 @@ def _is_moe(config: PretrainedConfig) -> bool: ) -def _get_cla_factor(config: PretrainedConfig) -> int: +def _get_cla_factor(config: PreTrainedConfig) -> int: if not getattr(config, "use_cla", False): return 1 return getattr(config, "cla_share_factor", 1) @@ -78,7 +78,7 @@ def _get_cla_factor(config: PretrainedConfig) -> int: class HYV3SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() @@ -93,7 +93,7 @@ class HYV3SharedHead(nn.Module): class HYV3MultiTokenPredictorLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, model_config: ModelConfig, cache_config: CacheConfig | None = None, diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 80c71cfe79..7396e007b8 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -137,7 +137,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo): mm_kwargs=mm_kwargs, ) - return num_patches + return max(num_patches, 1) def _get_image_token(self, processor: Idefics3Processor) -> tuple[str, str, str]: image_token = processor.image_token diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 21a09cb7ba..7901a136f5 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1262,7 +1262,7 @@ class SupportsRealtime(Protocol): Override in subclasses based on the model's expected output length.""" @classmethod - async def buffer_realtime_audio( + def buffer_realtime_audio( cls, audio_stream: AsyncGenerator[np.ndarray, None], input_stream: asyncio.Queue[list[int]], diff --git a/vllm/model_executor/models/intern_vit.py b/vllm/model_executor/models/intern_vit.py index 959f241de9..873160f968 100644 --- a/vllm/model_executor/models/intern_vit.py +++ b/vllm/model_executor/models/intern_vit.py @@ -13,7 +13,7 @@ from functools import partial import torch import torch.nn as nn import torch.nn.functional as F -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import ( should_torch_compile_mm_encoder, @@ -47,7 +47,7 @@ NORM2FN = { class InternVisionEmbeddings(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config self.embed_dim = config.hidden_size @@ -114,40 +114,12 @@ class InternVisionEmbeddings(nn.Module): return embeddings -class InternVisionPatchModel(nn.Module): - def __init__(self, config: PretrainedConfig): - super().__init__() - self.config = config - self.embeddings = InternVisionEmbeddings(config) - - def get_input_embeddings(self): - return self.embeddings - - def forward( - self, - pixel_values: torch.Tensor | None = None, - pixel_embeds: torch.Tensor | None = None, - ) -> torch.FloatTensor: - if pixel_values is None and pixel_embeds is None: - raise ValueError("You have to specify pixel_values or pixel_embeds") - - if pixel_embeds is not None: - hidden_states = pixel_embeds - elif pixel_values is not None: - if pixel_values.ndim == 4: - hidden_states = self.embeddings(pixel_values) - else: - raise ValueError(f"wrong pixel_values size: {pixel_values.shape}") - - return hidden_states - - class InternParallelAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_dummy_heads: int = 0, @@ -250,7 +222,7 @@ class InternParallelAttention(nn.Module): class InternMLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -292,7 +264,7 @@ class InternMLP(nn.Module): class InternVisionEncoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_dummy_heads: int = 0, @@ -325,7 +297,7 @@ class InternVisionEncoderLayer(nn.Module): def _init_attn( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, num_dummy_heads: int, @@ -352,7 +324,7 @@ class InternVisionEncoderLayer(nn.Module): class InternVisionEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, @@ -397,7 +369,7 @@ class InternVisionModel(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, diff --git a/vllm/model_executor/models/internlm2.py b/vllm/model_executor/models/internlm2.py index 29dd124b86..e1490071c2 100644 --- a/vllm/model_executor/models/internlm2.py +++ b/vllm/model_executor/models/internlm2.py @@ -8,7 +8,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -195,7 +195,7 @@ class InternLM2Attention(nn.Module): class InternLMDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 8d818e9a78..b1ac9596fe 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -12,10 +12,10 @@ from typing import Annotated, Literal, TypeAlias, TypedDict import regex as re import torch import torch.nn as nn -from transformers import BatchFeature, InternVLProcessor, PretrainedConfig +from transformers import BatchFeature, InternVLProcessor, PreTrainedConfig from transformers.activations import ACT2FN -from transformers.models.got_ocr2.image_processing_got_ocr2_fast import ( - GotOcr2ImageProcessorFast, +from transformers.models.got_ocr2.image_processing_got_ocr2 import ( + GotOcr2ImageProcessor, ) from transformers.models.internvl.video_processing_internvl import ( InternVLVideoProcessor, @@ -202,7 +202,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo): processor: InternVLProcessor, mm_kwargs: Mapping[str, object], ) -> int: - image_processor: GotOcr2ImageProcessorFast = processor.image_processor + image_processor: GotOcr2ImageProcessor = processor.image_processor num_image_patches = image_processor.get_number_of_image_patches( image_height, @@ -612,7 +612,7 @@ class InternS1ForConditionalGeneration( def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, @@ -625,7 +625,7 @@ class InternS1ForConditionalGeneration( prefix=prefix, ) - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1(self, config: PreTrainedConfig) -> nn.Module: return InternS1MultiModalProjector(config) def pixel_shuffle(self, x, scale_factor=0.5): diff --git a/vllm/model_executor/models/interns1_pro.py b/vllm/model_executor/models/interns1_pro.py index 8374542cfd..e757241e94 100644 --- a/vllm/model_executor/models/interns1_pro.py +++ b/vllm/model_executor/models/interns1_pro.py @@ -30,7 +30,7 @@ from typing import Any, ClassVar import torch from torch import nn -from transformers import AutoProcessor, PretrainedConfig +from transformers import AutoProcessor, PreTrainedConfig from vllm.config import CacheConfig, VllmConfig from vllm.distributed import ( @@ -43,12 +43,12 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -181,10 +181,9 @@ class InternS1ProMoeSparseMoeBlock(nn.Module): custom_routing_function=self._custom_routing_function, ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, prefix=f"{prefix}.gate", ) @@ -564,7 +563,7 @@ class InternS1ProForConditionalGeneration( def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super(Qwen3VLForConditionalGeneration, self).__init__() - config: PretrainedConfig = vllm_config.model_config.hf_config + config: PreTrainedConfig = vllm_config.model_config.hf_config multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config diff --git a/vllm/model_executor/models/interns1_vit.py b/vllm/model_executor/models/interns1_vit.py index e0958a63e3..f633733937 100644 --- a/vllm/model_executor/models/interns1_vit.py +++ b/vllm/model_executor/models/interns1_vit.py @@ -11,7 +11,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.utils import torch_int from vllm.model_executor.layers.activation import get_act_fn @@ -65,7 +65,7 @@ class InternS1VisionPatchEmbeddings(nn.Module): class InternS1VisionEmbeddings(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) @@ -166,7 +166,7 @@ class InternSdpaAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, *, num_dummy_heads: int = 0, prefix: str = "", @@ -242,7 +242,7 @@ class InternSdpaAttention(nn.Module): class InternS1VisionMLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -276,7 +276,7 @@ class InternS1VisionMLP(nn.Module): class InternS1VisionLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_dummy_heads: int = 0, @@ -311,7 +311,7 @@ class InternS1VisionLayer(nn.Module): def _init_attn( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, num_dummy_heads: int, @@ -343,7 +343,7 @@ class InternS1VisionLayer(nn.Module): class InternS1VisionEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, @@ -382,7 +382,7 @@ class InternS1VisionEncoder(nn.Module): class InternS1VisionModel(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index 4efaa5a08e..c3fece062f 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -17,7 +17,10 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, tensor_model_parallel_reduce_scatter, ) -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -93,11 +96,9 @@ class InternS2MobiusMetaMoeBlock(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) self.experts = FusedMoEFactory( diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 3640f8d6a3..ec3b39b6ad 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -14,7 +14,7 @@ from typing import Annotated, Any, Literal, TypeAlias, TypedDict, TypeVar import torch import torch.nn as nn -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -629,7 +629,7 @@ class InternVLChatModel( ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig | None + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back @@ -643,7 +643,7 @@ class InternVLChatModel( def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, @@ -663,7 +663,7 @@ class InternVLChatModel( prefix=prefix, ) - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1(self, config: PreTrainedConfig) -> nn.Module: vit_hidden_size = config.vision_config.hidden_size llm_hidden_size = config.text_config.hidden_size diff --git a/vllm/model_executor/models/iquest_loopcoder.py b/vllm/model_executor/models/iquest_loopcoder.py index 87783ae9c8..5d39e87564 100644 --- a/vllm/model_executor/models/iquest_loopcoder.py +++ b/vllm/model_executor/models/iquest_loopcoder.py @@ -22,7 +22,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -60,7 +60,7 @@ from .utils import ( class LoopCoderAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -210,7 +210,7 @@ class LoopCoderAttention(nn.Module): class LoopCoderDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/jamba.py b/vllm/model_executor/models/jamba.py index 44ea289869..fe9349c30b 100644 --- a/vllm/model_executor/models/jamba.py +++ b/vllm/model_executor/models/jamba.py @@ -16,11 +16,11 @@ from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -74,11 +74,9 @@ class JambaMoE(nn.Module): self.intermediate_size = config.intermediate_size if self.num_total_experts > 1: - self.router = ReplicatedLinear( + self.router = GateLinear( self.hidden_size, self.num_total_experts, - bias=False, - quant_config=None, params_dtype=params_dtype, prefix=f"{prefix}.router", ) diff --git a/vllm/model_executor/models/k2_horizon.py b/vllm/model_executor/models/k2_horizon.py index 47e87c0dcb..db435eaf05 100644 --- a/vllm/model_executor/models/k2_horizon.py +++ b/vllm/model_executor/models/k2_horizon.py @@ -49,6 +49,7 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.config import ( @@ -398,7 +399,7 @@ class K2HorizonSparseMoeBlock(nn.Module): self.physical_expert_start + self.n_local_physical_experts ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, bias=config.moe_gate_bias, diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index bfd2af714b..7bf5d79709 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -10,7 +10,7 @@ import numpy as np import torch import torch.nn as nn from einops import rearrange -from transformers import BaseImageProcessor, PretrainedConfig +from transformers import BaseImageProcessor, PreTrainedConfig from transformers.activations import GELUActivation from transformers.feature_extraction_utils import BatchFeature from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling @@ -191,7 +191,7 @@ KeyeVideoInputs: TypeAlias = KeyeVideoPixelInputs | KeyeVideoEmbeddingInputs class KeyeVisionEmbeddings(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config self.embed_dim = config.hidden_size @@ -349,7 +349,7 @@ class KeyeSiglipAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -487,7 +487,7 @@ class SigLIPRotaryEmbedding(nn.Module): class KeyeSiglipEncoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -539,7 +539,7 @@ class KeyeSiglipEncoderLayer(nn.Module): class KeyeSiglipEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -634,7 +634,7 @@ class KeyeSiglipEncoder(nn.Module): class KeyeSiglipVisionTransformer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -710,7 +710,7 @@ class KeyeSiglipVisionTransformer(nn.Module): class KeyeSiglipVisionModel(nn.Module): - config_class = PretrainedConfig + config_class = PreTrainedConfig main_input_name = "pixel_values" hf_to_vllm_mapper = WeightsMapper( @@ -724,7 +724,7 @@ class KeyeSiglipVisionModel(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -787,8 +787,8 @@ class KeyeSiglipVisionModel(nn.Module): class Projector(nn.Module): def __init__( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -1232,7 +1232,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config: PretrainedConfig = vllm_config.model_config.hf_config + config: PreTrainedConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config @@ -1264,8 +1264,8 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): @abstractmethod def _build_projector( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: @@ -1511,8 +1511,8 @@ class KeyeForConditionalGeneration( ): def _build_projector( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index b52d50296a..3bc7f36420 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -9,7 +9,7 @@ import numpy as np import torch import torch.nn as nn from einops import rearrange -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.activations import GELUActivation from transformers.feature_extraction_utils import BatchFeature @@ -197,8 +197,8 @@ KeyeVL1_5VideoInputs: TypeAlias = ( class KeyeVL1_5Projector(nn.Module): def __init__( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -509,15 +509,15 @@ class KeyeVL1_5ForConditionalGeneration( ): def _build_projector( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: return KeyeVL1_5Projector(text_config, vision_config, quant_config, prefix) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - config: PretrainedConfig = vllm_config.model_config.hf_config + config: PreTrainedConfig = vllm_config.model_config.hf_config self.merge_size = config.vision_config.spatial_merge_size super().__init__(vllm_config=vllm_config, prefix=prefix) diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index 03c409a911..1b05f1f77b 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -21,6 +21,7 @@ from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention +from vllm.model_executor.layers.conv import Conv2dLayer from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -194,7 +195,7 @@ class MoonVision3dPatchEmbed(nn.Module): ) self.patch_size = patch_size - self.proj = nn.Conv2d( + self.proj = Conv2dLayer( in_dim, out_dim, kernel_size=patch_size, @@ -220,26 +221,13 @@ class MoonVision3dPatchEmbed(nn.Module): *, pos_embeds: torch.Tensor | None = None, ) -> torch.Tensor: - x = self._proj(x).view(x.size(0), -1) + # forward_native dispatches this non-overlapping patch projection to GEMM. + x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels) if pos_embeds is not None: return x + pos_embeds assert grid_thws is not None return self.pos_emb(x, grid_thws) - def _proj(self, x: torch.Tensor) -> torch.Tensor: - # MIOpen conv2d intermittently fails under load on ROCm; use aiter Triton. - if current_platform.is_rocm() and x.dtype in (torch.float16, torch.bfloat16): - from aiter.ops.triton.conv.conv2d import conv2d - - return conv2d( - x, - self.proj.weight, - self.proj.bias, - stride=self.patch_size, - layout="nchw", - ) - return self.proj(x) - class Rope2DPosEmbRepeated(nn.Module): """2D rotary position embedding with multi-resolution support.""" diff --git a/vllm/model_executor/models/laguna.py b/vllm/model_executor/models/laguna.py index 7fe412eb23..c1aaf8c9a8 100644 --- a/vllm/model_executor/models/laguna.py +++ b/vllm/model_executor/models/laguna.py @@ -19,12 +19,14 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -171,11 +173,9 @@ class LagunaMoE(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size # Router gate - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 92193f0c2a..29bc56a696 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -17,12 +17,12 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -125,10 +125,9 @@ class Lfm2MoeSparseMoeBlock(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, quant_config=quant_config, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/lfm2_siglip2.py b/vllm/model_executor/models/lfm2_siglip2.py index b3c7ee28ae..42c8f62641 100644 --- a/vllm/model_executor/models/lfm2_siglip2.py +++ b/vllm/model_executor/models/lfm2_siglip2.py @@ -334,7 +334,7 @@ class Siglip2Encoder(nn.Module): self attention layers. Each layer is a [`Siglip2EncoderLayer`]. Args: - config: PretrainedConfig + config: PreTrainedConfig """ diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 644da00c81..dd58707b85 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -13,8 +13,8 @@ from transformers import BatchFeature from transformers.activations import ACT2FN from transformers.models.lfm2_vl import Lfm2VlProcessor from transformers.models.lfm2_vl.configuration_lfm2_vl import Lfm2VlConfig -from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import ( - Lfm2VlImageProcessorFast, +from transformers.models.lfm2_vl.image_processing_lfm2_vl import ( + Lfm2VlImageProcessor, find_closest_aspect_ratio, round_by_factor, ) @@ -104,7 +104,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs): return self.ctx.get_hf_processor(Lfm2VlProcessor, **kwargs) - def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessorFast: + def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessor: return self.get_hf_processor(**kwargs).image_processor def get_default_tok_params(self) -> TokenizeParams: @@ -213,7 +213,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo): processor: Lfm2VlProcessor, mm_kwargs: Mapping[str, object], ) -> tuple[int, int, int]: - image_processor: Lfm2VlImageProcessorFast = processor.image_processor + image_processor: Lfm2VlImageProcessor = processor.image_processor mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) downsample_factor = mm_kwargs.get( @@ -336,7 +336,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo): processor: Lfm2VlProcessor, mm_kwargs: Mapping[str, object], ) -> tuple[int, int]: - image_processor: Lfm2VlImageProcessorFast = processor.image_processor + image_processor: Lfm2VlImageProcessor = processor.image_processor mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) downsample_factor = mm_kwargs.get( diff --git a/vllm/model_executor/models/lilicorr.py b/vllm/model_executor/models/lilicorr.py new file mode 100644 index 0000000000..9901a68615 --- /dev/null +++ b/vllm/model_executor/models/lilicorr.py @@ -0,0 +1,546 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# LiLiCorr head adapted from sgl-project/sglang PR #37462 (Apache-2.0). +"""DFlash backbone with the LiLiCorr candidate-lattice correlator.""" + +from collections.abc import Iterable +from typing import Any + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.compilation.backends import set_model_tag +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig +from vllm.model_executor.layers.linear import ( + LinearBase, + ReplicatedLinear, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod +from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Static +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead + +from .nemotron_h import NemotronHMLP +from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model +from .qwen3_dflash2 import DFlash2Qwen3DecoderLayer +from .utils import AutoWeightsLoader, maybe_prefix + + +class LiLiCorrLatticeAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + if num_heads <= 0 or hidden_size % num_heads != 0: + raise ValueError( + f"LiLiCorr hidden_size={hidden_size} must be divisible " + f"by num_heads={num_heads}." + ) + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = self.hidden_size // self.num_heads + # Exported lattice QKV parameters stay in the model's floating dtype. + self.in_proj_weight = nn.Parameter( + torch.empty(3 * hidden_size, hidden_size), requires_grad=False + ) + self.in_proj_bias = nn.Parameter( + torch.empty(3 * hidden_size), requires_grad=False + ) + self.out_proj = ReplicatedLinear( + hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "out_proj"), + ) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + bsz, seq_len, _ = hidden_states.shape + qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias) + q, k, v = qkv.chunk(3, dim=-1) + shape = (bsz, seq_len, self.num_heads, self.head_dim) + q = q.view(shape).transpose(1, 2) + k = k.view(shape).transpose(1, 2) + v = v.view(shape).transpose(1, 2) + # Attention and MMEncoderAttention cannot accept this learned per-head + # additive bias. The full candidate lattice has no KV-cache state. + out = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attention_bias, + ) + return self.out_proj( + out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size) + ) + + +class LiLiCorrLayer(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float, + rms_norm_eps: float, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.attn_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.attn = LiLiCorrLatticeAttention( + hidden_size, num_heads, quant_config, maybe_prefix(prefix, "attn") + ) + self.mlp_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + mlp_hidden_size = int(hidden_size * mlp_ratio) + self.mlp = NemotronHMLP( + config=None, + hidden_size=hidden_size, + intermediate_size=mlp_hidden_size, + hidden_act="silu", + bias=True, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "mlp"), + ) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.attn_norm(hidden_states), attention_bias + ) + return hidden_states + self.mlp(self.mlp_norm(hidden_states)) + + +@support_torch_compile +class LiLiCorrHead(nn.Module): + num_candidate_features = 5 + + def __init__( + self, + *, + model_hidden_size: int, + block_size: int, + rms_norm_eps: float, + config: dict[str, Any], + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + hidden_size = config["lilicorr_hidden_size"] or model_hidden_size + self.block_size = block_size + self.num_candidate_slots = block_size - 1 + self.candidate_topk = config["lilicorr_candidate_topk"] + if ( + self.candidate_topk <= 0 + or self.candidate_topk > 16 + or self.candidate_topk & (self.candidate_topk - 1) + ): + raise ValueError("LiLiCorr candidate_topk must be a power of two <= 16.") + self.hidden_size = hidden_size + self.num_heads = config["lilicorr_num_heads"] + self.mlp_ratio = config["lilicorr_mlp_ratio"] + self.factor_dim = config["lilicorr_factor_dim"] + self.vector_eps = config["lilicorr_vector_eps"] + self.logit_scale = config["lilicorr_logit_scale"] + # Candidate IDs and embeddings are replicated across TP ranks, so each + # rank scores the same lattice; the MLPs use standard tensor parallelism. + self.token_proj = ( + nn.Identity() + if model_hidden_size == hidden_size + else ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "token_proj"), + ) + ) + self.pass_hidden_proj = ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "pass_hidden_proj"), + ) + self.feature_norm = nn.LayerNorm(self.num_candidate_features) + self.feature_mlp = NemotronHMLP( + config=None, + hidden_size=self.num_candidate_features, + intermediate_size=hidden_size, + hidden_act="silu", + output_size=hidden_size, + bias=True, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "feature_mlp"), + ) + self.slot_embedding = nn.Parameter( + torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size) + ) + self.rank_embedding = nn.Parameter( + torch.zeros(1, 1, 1, self.candidate_topk, hidden_size) + ) + self.relative_slot_bias = nn.Parameter( + torch.zeros(self.num_heads, 2 * self.block_size - 1) + ) + self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads)) + self.context_proj = ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "context_proj"), + ) + self.layers = nn.ModuleList( + [ + LiLiCorrLayer( + hidden_size=hidden_size, + num_heads=self.num_heads, + mlp_ratio=self.mlp_ratio, + rms_norm_eps=rms_norm_eps, + quant_config=quant_config, + prefix=maybe_prefix(prefix, f"layers.{i}"), + ) + for i in range(config["lilicorr_num_layers"]) + ] + ) + self.output_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.anchor_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.factor_input_proj = ReplicatedLinear( + hidden_size * 3, + hidden_size, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "factor_input_proj"), + ) + # Each candidate gets outgoing and incoming vectors. After normalization, + # their dot product scores a transition from slot s to slot s+1: + # pair_score[s, i, j] = dot(out[s, i], incoming[s+1, j]) + # The anchor supplies the outgoing vector for the first slot. + self.out_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "out_head"), + ) + self.in_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "in_head"), + ) + self.anchor_out_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "anchor_out_head"), + ) + self._fused_edge_weight: torch.Tensor | None = None + self._fused_edge_bias: torch.Tensor | None = None + self._factor_input_splits: tuple[torch.Tensor, ...] + self._attn_bias: torch.Tensor | None = None + self._rank_frac_col: torch.Tensor + self._is_top1_col: torch.Tensor + + @torch.no_grad() + def materialize_inference_buffers( + self, device: torch.device, dtype: torch.dtype + ) -> None: + """Build inference-only views/copies after checkpoint weights are loaded.""" + topk = self.candidate_topk + self._attn_bias = self._build_attention_bias(device=device, dtype=dtype) + # Both projections consume the same candidate features f: + # out = W_out @ f + b_out + # incoming = W_in @ f + b_in + # Stack weights and biases to compute both in one linear call. score() + # splits the result and normalizes the two vectors separately. + self._fused_edge_weight = ( + torch.cat([self.out_head.weight, self.in_head.weight], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + self._fused_edge_bias = ( + torch.cat([self.out_head.bias, self.in_head.bias], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + # For candidate state x and request anchor a, the trained projection is: + # W @ concat(x, a, x*a) + b + # Split W into input blocks to evaluate the same expression as: + # W_x @ x + W_a @ a + W_cross @ (x*a) + b + # Here x*a is elementwise. This avoids concatenating inputs and reuses + # W_a @ a across candidates, adding the original bias only once. + weight = self.factor_input_proj.weight + hdim = self.hidden_size + self._factor_input_splits = ( + weight[:, :hdim].contiguous(), + weight[:, hdim : 2 * hdim].contiguous(), + weight[:, 2 * hdim :].contiguous(), + ) + if topk > 1: + rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view( + 1, 1, topk + ) / float(topk - 1) + else: + rank_frac = torch.zeros(1, 1, topk, device=device, dtype=torch.float32) + is_top1 = torch.zeros(1, 1, topk, device=device, dtype=torch.float32) + is_top1[..., 0] = 1.0 + self._rank_frac_col = rank_frac.contiguous() + self._is_top1_col = is_top1.contiguous() + + def _build_attention_bias( + self, *, device: torch.device, dtype: torch.dtype + ) -> torch.Tensor: + topk = self.candidate_topk + slot_ids = torch.arange( + self.num_candidate_slots, device=device, dtype=torch.long + ).repeat_interleave(topk) + # Attention flattens [slot, candidate] into one sequence. With topk=2, + # the candidate slot IDs are [0, 0, 1, 1, ...]. Each head learns: + # bias(q, k) = relative_bias[slot(q) - slot(k) + offset] + # + same_slot_bias * (slot(q) == slot(k)) + # The same-slot term includes distinct candidates at the same position. + # This is an additive preference, not a causal mask. Keep the trained + # offset = block_size-1 when score() selects a shorter draft's prefix, + # so each distance continues to use the same learned weight. + rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1) + bias = self.relative_slot_bias[:, rel + self.block_size - 1] + same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1) + bias = bias + same_slot.unsqueeze(0).to( + dtype=bias.dtype + ) * self.same_slot_bias.view(-1, 1, 1) + return bias.to(device=device, dtype=dtype).contiguous() + + def score( + self, + *, + token_embeddings: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self._attn_bias is None: + raise RuntimeError( + "Call materialize_inference_buffers() after loading the LiLiCorr head." + ) + bsz, n_slots, topk = candidate_log_probs.shape + if not 1 <= n_slots <= self.num_candidate_slots: + raise ValueError( + f"LiLiCorr supports 1..{self.num_candidate_slots} candidate slots, " + f"got {n_slots}." + ) + if topk != self.candidate_topk: + raise ValueError( + f"LiLiCorr expects candidate_topk={self.candidate_topk}, got {topk}." + ) + proj_dtype = self.slot_embedding.dtype + if token_embeddings.dtype != proj_dtype: + token_embeddings = token_embeddings.to(proj_dtype) + if pass_hidden.dtype != proj_dtype: + pass_hidden = pass_hidden.to(proj_dtype) + token_states = self.token_proj(token_embeddings) + pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2) + log_probs = candidate_log_probs.float() + features = torch.stack( + [ + log_probs, + log_probs.exp(), + log_probs - log_probs.max(dim=-1, keepdim=True).values, + self._rank_frac_col.expand_as(log_probs), + self._is_top1_col.expand_as(log_probs), + ], + dim=-1, + ) + hidden_states = token_states + pass_states + hidden_states = hidden_states + self.feature_mlp( + self.feature_norm(features.to(dtype=token_states.dtype)) + ) + hidden_states = hidden_states + self.slot_embedding[:, 0, :n_slots] + hidden_states = hidden_states + self.rank_embedding[:, 0] + hidden_states = hidden_states.reshape(bsz, n_slots * topk, self.hidden_size) + anchor_state = self.context_proj(anchor_hidden) + anchor_state = anchor_state * anchor_valid[:, None].to(anchor_state.dtype) + # Shorter drafts use the learned prefix; keep checkpoint parameter shapes. + lattice = n_slots * topk + attention_bias = self._attn_bias[None, :, :lattice, :lattice] + for layer in self.layers: + hidden_states = layer(hidden_states, attention_bias) + hidden_states = self.output_norm(hidden_states).reshape( + bsz, n_slots, topk, self.hidden_size + ) + anchor_state = self.anchor_norm(anchor_state) + w_self, w_anchor, w_cross = self._factor_input_splits + anchor_row = anchor_state[:, None, None, :] + pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias) + pre = pre + F.linear(anchor_row, w_anchor) + pre = pre + F.linear(hidden_states * anchor_row, w_cross) + factor_hidden = F.silu(pre) + edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias) + out_vec, in_vec = F.normalize( + edges.unflatten(-1, (2, self.factor_dim)), dim=-1, eps=self.vector_eps + ).unbind(-2) + anchor_out = F.normalize( + self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps + ) + # start_scores[b, j] scores anchor -> candidate j in slot zero. + # pair_scores[b, s, i, j] scores candidate i in slot s -> candidate j + # in slot s+1. Normalized factor dot products encode learned compatibility; + # forward() scales them into logits for the candidate-path selector. + start_scores = (anchor_out[:, None, :] * in_vec[:, 0, :, :]).sum(dim=-1) + pair_scores = torch.matmul(out_vec[:, :-1], in_vec[:, 1:].transpose(-1, -2)) + return (start_scores, pair_scores) + + def forward( + self, + token_embeddings: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + ) -> torch.Tensor: + start, pairs = self.score( + token_embeddings=token_embeddings, + candidate_log_probs=candidate_log_probs, + pass_hidden=pass_hidden, + anchor_hidden=anchor_hidden, + anchor_valid=anchor_valid, + ) + start = self.logit_scale * start.float() + pairs = self.logit_scale * pairs.float() + # The shared walk starts at predecessor index zero; all first rows agree. + first = start[:, None, None, :].expand(-1, 1, self.candidate_topk, -1) + return torch.cat((first, pairs), dim=1) + + +class LiLiCorr(DFlashQwen3Model): + def __init__( + self, *, vllm_config: VllmConfig, start_layer_id: int = 0, prefix: str = "" + ): + spec = vllm_config.speculative_config + assert spec is not None + config = spec.draft_model_config.hf_config + draft_config = config.dflash_config + if not draft_config.get("lilicorr_enabled", True): + raise ValueError("LiLiCorr requires lilicorr_enabled=True.") + block_size = draft_config.get("block_size", getattr(config, "block_size", 0)) + if not 1 <= spec.num_speculative_tokens < block_size: + raise ValueError( + "LiLiCorr requires 1 <= num_speculative_tokens < trained block_size." + ) + taps = draft_config.get("conv_kernel_size", 0) + groups = draft_config.get("conv_group_size", 0) + if taps < 0 or groups < 0 or bool(taps) != bool(groups): + raise ValueError( + "LiLiCorr convolution geometry must enable both taps and group size." + ) + if taps: + self.decoder_layer_cls = DFlash2Qwen3DecoderLayer + super().__init__( + vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix + ) + with set_model_tag("lilicorr_head"): + self.lilicorr = LiLiCorrHead( + model_hidden_size=config.hidden_size, + block_size=block_size, + rms_norm_eps=config.rms_norm_eps, + config=draft_config, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, "lilicorr"), + ) + + +class LiLiCorrForCausalLM(DFlashQwen3ForCausalLM): + model_cls = LiLiCorr + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + spec = vllm_config.speculative_config + assert spec is not None + super().__init__(vllm_config=vllm_config, prefix=prefix) + if self.draft_id_to_target_id is not None: + raise ValueError("LiLiCorr candidates require the full target vocabulary.") + self.has_own_lm_head = bool(getattr(self.config, "has_own_lm_head", False)) + # The correlator consumes raw candidate-head log-softmax features. + self.candidate_logits_processor = LogitsProcessor(self.config.vocab_size) + + def compute_candidates( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.candidate_logits_processor.get_top_k_tokens( + self.lm_head, + hidden_states, + self.model.lilicorr.candidate_topk, + return_log_probs=True, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + quantized_metadata: set[str] = set() + for name, module in self.named_modules(): + name = name.removeprefix("model.") + if not isinstance(module, (LinearBase, ParallelLMHead)) or isinstance( + module.quant_method, UnquantizedLinearMethod + ): + continue + required = {"weight", "bias"} + method = module.quant_method + if ( + isinstance(method, ModelOptLinearMethod) + and method.spec.weight == kNvfp4Static + ): + # NVFP4 scales are trained checkpoint data. Only W4A16's + # deprecated, unused input_scale may be absent. + required.update(("weight_scale", "weight_scale_2")) + if method.spec.activation is not None: + required.add("input_scale") + quantized_metadata.update( + f"{name}.{parameter}" + for parameter, _ in module.named_parameters(recurse=False) + if parameter not in required + ) + expected = { + name.removeprefix("model.") + for name, _ in self.named_parameters() + if ( + name.startswith("model.lilicorr.") + or ".attention_conv." in name + or ".mlp_conv." in name + or (self.has_own_lm_head and name.startswith("lm_head.")) + ) + and name.removeprefix("model.") not in quantized_metadata + } + seen: set[str] = set() + head_weights: list[tuple[str, torch.Tensor]] = [] + + def normalized_weights(): + for name, value in weights: + name = name.removeprefix("model.") + seen.add(name) + # The backbone stacks up_proj into gate_up_proj. Load the + # nongated correlator separately to avoid that transformation. + if name.startswith("lilicorr."): + head_weights.append((name.removeprefix("lilicorr."), value)) + else: + yield name, value + + super().load_weights(normalized_weights()) + AutoWeightsLoader(self.model.lilicorr).load_weights(head_weights) + missing = expected - seen + if missing: + raise ValueError( + f"LiLiCorr checkpoint coverage mismatch: missing={sorted(missing)}" + ) + head = self.model.lilicorr + parameter = head.slot_embedding + head.materialize_inference_buffers(parameter.device, parameter.dtype) diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index 3ffa47bba3..0eb34c70b6 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -38,12 +38,12 @@ from vllm.model_executor.layers.attention import ( ) from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig @@ -98,11 +98,9 @@ class Llama4MoE(nn.Module): self.ep_size = self.ep_group.size() intermediate_size_moe = config.intermediate_size - self.router = ReplicatedLinear( + self.router = GateLinear( config.hidden_size, config.num_local_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.router", ) diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 22cd34fc85..ead9bbb1e7 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -12,7 +12,7 @@ from transformers import ( CLIPVisionConfig, LlavaConfig, PixtralVisionConfig, - PretrainedConfig, + PreTrainedConfig, SiglipVisionConfig, ) from transformers.models.llava import LlavaProcessor @@ -156,7 +156,7 @@ class LlavaMultiModalProjector(nn.Module): class LlavaLikeConfig(Protocol): - vision_config: PretrainedConfig + vision_config: PreTrainedConfig image_token_index: int vision_feature_select_strategy: str vision_feature_layer: int | list[int] diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index d8d514be23..a1b12b3834 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -1307,7 +1307,6 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo): # ``_merge_mm_kwargs`` restricts caller ``mm_processor_kwargs`` to known # processor args and wraps values as hashable for the lru_cache. merged = _merge_mm_kwargs(model_config, AutoProcessor, **kwargs) - merged.setdefault("use_fast", True) return _load_ov2_processor( model_config.model, model_config.revision, diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 4178d63ff4..0c9802a8b3 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -39,7 +39,7 @@ from itertools import islice import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -48,12 +48,12 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -80,7 +80,7 @@ from .utils import ( logger = init_logger(__name__) -class FlashConfig(PretrainedConfig): +class FlashConfig(PreTrainedConfig): """Flash model configuration.""" moe_intermediate_size: int @@ -254,12 +254,12 @@ class LongcatRouter(nn.Module): else config.num_experts[0] ) self.n_routed_experts = self.n_routed_experts + zero_expert_num - self.classifier = ReplicatedLinear( + self.classifier = GateLinear( config.hidden_size, self.n_routed_experts, bias=config.router_bias, + out_dtype=router_params_dtype, params_dtype=router_params_dtype, - quant_config=None, prefix=f"{prefix}.classifier", ) self.e_score_correction_bias = nn.Parameter( @@ -331,9 +331,7 @@ class LongcatMoe(nn.Module): else: hidden_states_padded = hidden_states - router_logits_full = self.router( - hidden_states_padded.to(self.router_params_dtype) - ) + router_logits_full = self.router(hidden_states_padded) # MoERunner handles routing memoization and zero expert computation # internally. Pass full router_logits (including zero experts) so that diff --git a/vllm/model_executor/models/longcat_flash_mtp.py b/vllm/model_executor/models/longcat_flash_mtp.py index 851c694f37..770f931e81 100644 --- a/vllm/model_executor/models/longcat_flash_mtp.py +++ b/vllm/model_executor/models/longcat_flash_mtp.py @@ -7,7 +7,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.layernorm import RMSNorm @@ -30,7 +30,7 @@ from .utils import maybe_prefix class LongCatMultiTokenPredictorLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, vllm_config: VllmConfig, quant_config: QuantizationConfig | None = None, @@ -353,7 +353,7 @@ class LongCatFlashMTP(nn.Module): return name def get_spec_layer_idx_from_weight_name( - self, config: PretrainedConfig, weight_name: str + self, config: PreTrainedConfig, weight_name: str ) -> int | None: if "model.mtp" in weight_name: return config.num_hidden_layers * 2 diff --git a/vllm/model_executor/models/mimo_audio.py b/vllm/model_executor/models/mimo_audio.py index d0d2513f04..48c526bd92 100644 --- a/vllm/model_executor/models/mimo_audio.py +++ b/vllm/model_executor/models/mimo_audio.py @@ -21,7 +21,7 @@ import torch.nn as nn import torch.nn.functional as F from einops import rearrange, repeat from transformers.activations import ACT2FN -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.modeling_utils import PreTrainedModel from transformers.models.qwen2.configuration_qwen2 import Qwen2Config from transformers.models.qwen2.modeling_qwen2 import Qwen2Model @@ -518,7 +518,7 @@ class ResidualVectorQuantizer(nn.Module): # --------------------------------------------------------------------------- -class MiMoAudioTokenizerConfig(PretrainedConfig): +class MiMoAudioTokenizerConfig(PreTrainedConfig): model_type = "mimo_audio_tokenizer" def __init__( diff --git a/vllm/model_executor/models/mimo_mtp.py b/vllm/model_executor/models/mimo_mtp.py index 32d3b1f662..65566230a5 100644 --- a/vllm/model_executor/models/mimo_mtp.py +++ b/vllm/model_executor/models/mimo_mtp.py @@ -24,7 +24,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.model_executor.layers.layernorm import RMSNorm @@ -43,7 +43,7 @@ from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix class MiMoMultiTokenPredictorLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, model_config: ModelConfig, cache_config: CacheConfig | None = None, diff --git a/vllm/model_executor/models/mimo_v2_mtp.py b/vllm/model_executor/models/mimo_v2_mtp.py index 6fd8309350..a4e46977ec 100644 --- a/vllm/model_executor/models/mimo_v2_mtp.py +++ b/vllm/model_executor/models/mimo_v2_mtp.py @@ -22,7 +22,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.distributed import ( @@ -62,7 +62,7 @@ class MiMoV2MTPLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, ) -> None: @@ -145,7 +145,7 @@ class _MiMoV2MTPLayers(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, num_mtp_layers: int, quant_config: QuantizationConfig | None, prefix: str, diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index f55cf44f37..8c838c7cfb 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -10,7 +10,7 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from typing_extensions import TypedDict @@ -418,7 +418,7 @@ class MiMoVisionTransformer(nn.Module): def __init__( self, - vision_cfg: PretrainedConfig, + vision_cfg: PreTrainedConfig, *, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, diff --git a/vllm/model_executor/models/minicpm.py b/vllm/model_executor/models/minicpm.py index 0d7da44c3b..0c934386a7 100644 --- a/vllm/model_executor/models/minicpm.py +++ b/vllm/model_executor/models/minicpm.py @@ -31,7 +31,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -43,12 +43,15 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import FatreluAndMul, SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk +from vllm.model_executor.layers.fused_moe import ( + GateLinear, + fused_experts, + fused_topk, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -109,12 +112,10 @@ class MiniCPMMoE(nn.Module): params_dtype = torch.get_default_dtype() self.params_dtype = params_dtype - self.gate = ReplicatedLinear( + self.gate = GateLinear( self.hidden_size, self.num_total_experts, - bias=False, params_dtype=self.params_dtype, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -315,7 +316,7 @@ class MiniCPMAttention(nn.Module): class MiniCPMDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -426,7 +427,7 @@ class MiniCPMModel(nn.Module, EagleModelMixin): def _init_layers( self, prefix: str, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None, quant_config: QuantizationConfig | None, ): diff --git a/vllm/model_executor/models/minicpm3.py b/vllm/model_executor/models/minicpm3.py index 30a9cd34ed..08b8ec47f3 100644 --- a/vllm/model_executor/models/minicpm3.py +++ b/vllm/model_executor/models/minicpm3.py @@ -27,7 +27,7 @@ import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size @@ -52,7 +52,7 @@ from .utils import make_layers class MiniCPM3Attention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, qk_nope_head_dim: int, @@ -208,7 +208,7 @@ class MiniCPM3Model(MiniCPMModel): def _init_layers( self, prefix: str, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None, quant_config: QuantizationConfig | None, ): diff --git a/vllm/model_executor/models/minicpm_eagle.py b/vllm/model_executor/models/minicpm_eagle.py index a495fed22c..ebbbefd26b 100644 --- a/vllm/model_executor/models/minicpm_eagle.py +++ b/vllm/model_executor/models/minicpm_eagle.py @@ -30,7 +30,7 @@ from typing import TYPE_CHECKING import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -68,7 +68,7 @@ else: class EagleMiniCPMDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -189,7 +189,7 @@ class EagleMiniCPMModel(nn.Module): def _init_layers( self, prefix: str, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None, quant_config: QuantizationConfig | None, start_layer: int, diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 8027dc1027..6555e2f132 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -36,7 +36,7 @@ import torch import torch.types from torch import nn from torch.nn.init import trunc_normal_ -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from transformers.dynamic_module_utils import ( get_class_from_dynamic_module, resolve_trust_remote_code, @@ -468,7 +468,7 @@ class Resampler4_5(Resampler2_5): return x -def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]: +def get_version_by_config(config: PreTrainedConfig) -> tuple[int, ...]: version_float = getattr(config, "version", None) # The old configs do not include version number @@ -1235,7 +1235,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" super().__init__() # All MiniCPM-V models disable `tie_word_embeddings` but - # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot + # `PreTrainedConfig.tie_word_embeddings` defaults to True; we cannot # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model # and config class self.config = config @@ -1425,7 +1425,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> nn.Module: @@ -1957,7 +1957,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> nn.Module: @@ -2058,7 +2058,7 @@ class MiniCPMV2_5(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> nn.Module: @@ -2154,7 +2154,7 @@ class MiniCPMV2_6(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: @@ -2255,7 +2255,7 @@ class MiniCPMV4_0(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: @@ -2354,7 +2354,7 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): def init_vision_module( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index 139529e559..77f8993159 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -29,7 +29,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig @@ -70,7 +70,7 @@ from .utils import ( class MiniMaxM2MoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -249,7 +249,7 @@ class MiniMaxM2Attention(nn.Module): class MiniMaxM2DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, model_config: ModelConfig, cache_config: CacheConfig | None = None, diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py index 717c935751..248459ceb5 100644 --- a/vllm/model_executor/models/mixtral.py +++ b/vllm/model_executor/models/mixtral.py @@ -39,11 +39,13 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -106,12 +108,10 @@ class MixtralMoE(nn.Module): self.n_local_physical_experts = self.n_physical_experts // self.ep_size # Gate always runs at half / full precision for now. - self.gate = ReplicatedLinear( + self.gate = GateLinear( hidden_size, num_experts, - bias=False, params_dtype=params_dtype, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 89d4edbe86..fbf393b554 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -18,7 +18,6 @@ # limitations under the License. import math from collections.abc import Hashable, Iterable, Mapping -from itertools import tee from typing import Annotated, Any, Literal import torch @@ -26,7 +25,7 @@ from torch import nn from transformers import BatchFeature, Llama4Config, Llama4VisionConfig from transformers.image_utils import SizeDict from transformers.models.llama4 import Llama4Processor -from transformers.models.llama4.image_processing_llama4_fast import ( +from transformers.models.llama4.image_processing_llama4 import ( find_supported_resolutions, get_best_fit, ) @@ -555,9 +554,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo): return self.ctx.get_hf_config(Llama4Config) def get_hf_processor(self, **kwargs: object) -> Llama4Processor: - return self.ctx.get_hf_processor( - Llama4Processor, use_fast=kwargs.pop("use_fast", True), **kwargs - ) + return self.ctx.get_hf_processor(Llama4Processor, **kwargs) def get_supported_mm_limits(self) -> Mapping[str, int | None]: # Although vLLM can support more images from an infra capability @@ -1041,25 +1038,6 @@ class Llama4ForConditionalGeneration( ) -> torch.Tensor | None: return self.language_model.compute_logits(hidden_states) - def separate_weights( - self, - weights: Iterable[tuple[str, torch.Tensor]], - prefix: str, - ) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]: - weights1, weights2 = tee(weights, 2) - - def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]: - for name, data in weights1: - if name.startswith(prefix): - yield (name, data) - - def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]: - for name, data in weights2: - if not name.startswith(prefix): - yield (name, data) - - return get_prefix_weights(), get_other_weights() - def _consolidate_qkv_weights( self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[tuple[str, torch.Tensor]]: diff --git a/vllm/model_executor/models/modernbert.py b/vllm/model_executor/models/modernbert.py index df901a6566..3a8abee67e 100644 --- a/vllm/model_executor/models/modernbert.py +++ b/vllm/model_executor/models/modernbert.py @@ -19,7 +19,6 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.pooler import DispatchPooler -from vllm.model_executor.layers.pooler.activations import LambdaPoolerActivation from vllm.model_executor.layers.pooler.seqwise import ( EmbeddingPoolerHead, SequencePooler, @@ -367,8 +366,7 @@ class ModernBertPooler(SequencePooler): # Use lambdas so that weights are not registered under `self.head` self.head = EmbeddingPoolerHead( head_dtype=head_dtype, - projector=lambda x: self.dense(x), - activation=LambdaPoolerActivation(lambda x: self.norm(self.act(x))), + projector=lambda x: self.norm(self.act(self.dense(x))), ) diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 17ad7f0be1..ad6b1c4a8b 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -16,7 +16,7 @@ from einops import rearrange from transformers import ( BaseImageProcessor, BatchFeature, - PretrainedConfig, + PreTrainedConfig, ) from vllm.compilation.decorators import support_torch_compile @@ -405,7 +405,7 @@ class MolmoAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -515,7 +515,7 @@ class LanguageModelMLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, input_dim: int | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -557,7 +557,7 @@ class ImageProjectorMLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, input_dim: int | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -598,7 +598,7 @@ class ImageProjectorMLP(nn.Module): class MolmoDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -681,7 +681,7 @@ class MolmoVisionBackbone(nn.Module, SupportsQuant): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vision_config: VisionBackboneConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index b44419a42a..4598d9da2a 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -16,7 +16,7 @@ from transformers import ( BaseImageProcessor, BaseVideoProcessor, BatchFeature, - PretrainedConfig, + PreTrainedConfig, ProcessorMixin, ) from transformers.image_utils import ImageInput @@ -1305,7 +1305,7 @@ def exif_transpose( def build_flat_image_bool_length( image_grids: torch.LongTensor, - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, image_use_col_tokens: bool = True, use_single_crop_col_tokens: bool | None = None, use_single_crop_start_token: bool = True, @@ -1393,7 +1393,7 @@ def build_flat_image_bool_length( def build_flat_video_bool_length( video_grids: torch.LongTensor, - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, ) -> tuple[torch.LongTensor, torch.LongTensor]: image_patch_id = hf_config.image_patch_id frame_start_id = hf_config.frame_start_token_id diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 1e750fc9d0..278167d0f7 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -28,12 +28,15 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) -from vllm.model_executor.layers.fused_moe import MoEActivation, fused_experts +from vllm.model_executor.layers.fused_moe import ( + GateLinear, + MoEActivation, + fused_experts, +) from vllm.model_executor.layers.fused_moe.config import biased_moe_quant_config from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -453,11 +456,10 @@ class Moondream3TextMoE(nn.Module): self.num_local_experts = self.experts_per_rank # Router (gate) - use ReplicatedLinear for compatibility - self.gate = ReplicatedLinear( + self.gate = GateLinear( hidden_size, num_experts, bias=True, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 02551d5663..6e7f504150 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -13,7 +13,7 @@ import regex as re import torch import torch.nn.functional as F from torch import nn -from transformers import BatchFeature, PretrainedConfig, Qwen3Config +from transformers import BatchFeature, PreTrainedConfig, Qwen3Config from transformers.models.whisper import WhisperFeatureExtractor from typing_extensions import TypedDict @@ -263,7 +263,7 @@ class MossAudioEncoderConfig: return cls(**values) -class MossAudioConfig(PretrainedConfig): +class MossAudioConfig(PreTrainedConfig): model_type = "moss_audio" is_composition = True diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index de61880b5f..074a89abb9 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -16,7 +16,7 @@ from typing import Annotated, Literal, TypeAlias, TypedDict, cast import torch import torch.nn as nn -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from vllm import envs from vllm.config import VllmConfig @@ -34,6 +34,7 @@ from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, MultiModalEmbeddings, + SupportsEagle3, SupportsLoRA, SupportsMultiModal, SupportsMultiModalPruning, @@ -238,7 +239,7 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo): return self.ctx.get_mm_config().video_pruning_rate @property - def sound_config(self) -> PretrainedConfig | None: + def sound_config(self) -> PreTrainedConfig | None: return getattr(self.get_hf_config(), "sound_config", None) def get_default_tok_params(self) -> TokenizeParams: @@ -928,6 +929,7 @@ class NemotronH_Nano_VL_V2( nn.Module, HasInnerState, IsHybrid, + SupportsEagle3, SupportsMultiModal, SupportsMultiModalPruning, SupportsLoRA, diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index e5451d16ba..3f97deebe4 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -34,7 +34,7 @@ from vllm.config.parallel import ParallelConfig from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size from vllm.distributed.communication_op import tensor_model_parallel_all_gather from vllm.distributed.parallel_state import get_pp_group -from vllm.model_executor.layers.activation import ReLUSquaredActivation +from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, @@ -93,7 +93,7 @@ from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig class NemotronHMLP(nn.Module): def __init__( self, - config: NemotronHConfig, + config: NemotronHConfig | None, hidden_size: int, intermediate_size: int, quant_config: QuantizationConfig | None = None, @@ -101,6 +101,9 @@ class NemotronHMLP(nn.Module): reduce_results: bool = True, is_sequence_parallel: bool = False, prefix: str = "", + *, + hidden_act: str = "relu2", + output_size: int | None = None, ) -> None: super().__init__() @@ -114,14 +117,14 @@ class NemotronHMLP(nn.Module): ) self.down_proj = RowParallelLinear( input_size=intermediate_size, - output_size=hidden_size, + output_size=hidden_size if output_size is None else output_size, bias=bias, quant_config=quant_config, reduce_results=reduce_results, disable_tp=is_sequence_parallel, prefix=f"{prefix}.down_proj", ) - self.act_fn = ReLUSquaredActivation() + self.act_fn = get_act_fn(hidden_act) def forward(self, x: torch.Tensor): x, _ = self.up_proj(x) diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 5c4375b600..9ac38c0021 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -17,7 +17,7 @@ from einops import rearrange from transformers import ( BartConfig, BatchFeature, - PretrainedConfig, + PreTrainedConfig, ) from vllm.config import CacheConfig, VllmConfig @@ -440,7 +440,7 @@ class RadioWithNeck(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -480,7 +480,7 @@ class RadioWithNeck(nn.Module): def get_vit_model_from_radio_config( self, - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, ) -> RadioModel: hf_config_vision = hf_config.encoder diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index 7b2777a710..0ec4589e51 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -5,7 +5,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import AutoModel, PretrainedConfig +from transformers import AutoModel, PreTrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear @@ -146,7 +146,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig | None + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back @@ -160,7 +160,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, @@ -172,7 +172,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor def _init_mlp1( self, - config: PretrainedConfig, + config: PreTrainedConfig, vit_hidden_size: int | None = None, vision_projection_hidden_size: int | None = None, ) -> nn.Module: @@ -505,7 +505,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config, *, prefix: str, @@ -520,7 +520,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) def _init_mlp1( self, - config: PretrainedConfig, + config: PreTrainedConfig, vit_hidden_size: int | None = None, vision_projection_hidden_size: int | None = None, ) -> nn.Module: diff --git a/vllm/model_executor/models/nvlm_d.py b/vllm/model_executor/models/nvlm_d.py index 5efcdb6f53..1039fea95e 100644 --- a/vllm/model_executor/models/nvlm_d.py +++ b/vllm/model_executor/models/nvlm_d.py @@ -11,7 +11,7 @@ from collections.abc import Mapping import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict @@ -161,7 +161,7 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo dummy_inputs=NVLMDummyInputsBuilder, ) class NVLM_D_Model(InternVLChatModel): - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1(self, config: PreTrainedConfig) -> nn.Module: vit_hidden_size = config.vision_config.hidden_size llm_intermediate_size = config.text_config.intermediate_size llm_hidden_size = config.text_config.hidden_size @@ -179,7 +179,7 @@ class NVLM_D_Model(InternVLChatModel): def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, diff --git a/vllm/model_executor/models/olmoe.py b/vllm/model_executor/models/olmoe.py index 7867caa97c..175f25259e 100644 --- a/vllm/model_executor/models/olmoe.py +++ b/vllm/model_executor/models/olmoe.py @@ -32,11 +32,13 @@ from vllm.distributed import ( from vllm.distributed.utils import split_tensor_along_last_dim from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -84,11 +86,9 @@ class OlmoeMoE(nn.Module): self.hidden_size = hidden_size # Gate always runs at half / full precision for now. - self.gate = ReplicatedLinear( + self.gate = GateLinear( hidden_size, num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index 4aa603b458..b6f001dbb9 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -26,7 +26,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ParallelConfig, VllmConfig @@ -43,7 +43,10 @@ from vllm.model_executor.layers.attention import ( Attention, StaticSinkAttention, ) -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -131,7 +134,7 @@ class OpenPanguMLP(nn.Module): class OpenPanguMoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, parallel_config: ParallelConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -149,11 +152,9 @@ class OpenPanguMoE(nn.Module): self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe check_ffn_act_fn(config.hidden_act) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.n_routed_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) if ( @@ -239,7 +240,7 @@ class OpenPanguMoE(nn.Module): class OpenPanguMLAAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, qk_nope_head_dim: int, @@ -388,7 +389,7 @@ class OpenPanguMLAAttention(nn.Module): class OpenPanguEmbeddedAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -500,7 +501,7 @@ class OpenPanguEmbeddedAttention(nn.Module): def _init_rotary_emb( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, ) -> None: is_neox_style = True @@ -521,7 +522,7 @@ class OpenPanguEmbeddedAttention(nn.Module): class OpenPanguSinkAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -742,7 +743,7 @@ class OpenPanguSinkAttention(nn.Module): def _init_rotary_emb( self, - config: PretrainedConfig, + config: PreTrainedConfig, rope_parameters: dict[str, Any] | None, quant_config: QuantizationConfig | None, ) -> None: @@ -768,7 +769,7 @@ class OpenPanguSinkAttention(nn.Module): class OpenPanguDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, vllm_config: VllmConfig, ) -> None: diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py index 741ec2155f..b7d5c81032 100644 --- a/vllm/model_executor/models/openpangu_vl.py +++ b/vllm/model_executor/models/openpangu_vl.py @@ -686,7 +686,6 @@ class OpenPanguVLProcessingInfo(Qwen2_5_VLProcessingInfo): kwargs["fps"] = fps return self.ctx.get_hf_processor( - use_fast=kwargs.pop("use_fast", True), **kwargs, ) diff --git a/vllm/model_executor/models/orion.py b/vllm/model_executor/models/orion.py index 2b58ee1e01..4d50705c78 100644 --- a/vllm/model_executor/models/orion.py +++ b/vllm/model_executor/models/orion.py @@ -13,7 +13,7 @@ from typing import Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -163,7 +163,7 @@ class OrionAttention(nn.Module): class OrionDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index ed0ea7ab7b..7c1668ddab 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -26,7 +26,7 @@ import torch import torch.nn as nn from torch import Tensor from torch.nn.functional import gumbel_softmax, pad, softmax -from transformers import BatchFeature, PretrainedConfig +from transformers import BatchFeature, PreTrainedConfig from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -93,7 +93,7 @@ def st_argmax(y_soft: torch.Tensor, dim: int): # straight-through softmax class VisualTokenizer(torch.nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -120,7 +120,7 @@ class VisualTokenizer(torch.nn.Module): def _init_backbone( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> nn.Module: @@ -419,7 +419,7 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP): config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - self.config: PretrainedConfig = config + self.config: PreTrainedConfig = config with self._mark_language_model(vllm_config): self.llm = init_vllm_registered_model( diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 2520b38ae3..86a32e270a 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -8,7 +8,7 @@ from typing import Annotated, Literal, TypedDict import torch import torch.nn as nn -from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig +from transformers import BaseImageProcessor, BatchFeature, PreTrainedConfig from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -95,7 +95,7 @@ class VisualTokenizer(torch.nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, visual_vocab_size: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -121,7 +121,7 @@ class VisualTokenizer(torch.nn.Module): def _init_backbone( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -175,6 +175,18 @@ class VisualTokenizer(torch.nn.Module): class Ovis2_5ProcessingInfo(BaseProcessingInfo): + def __init__(self, ctx) -> None: + super().__init__(ctx) + # extra_special_tokens adds tokens to the tokenizer lazily, on first + # access, but the renderer deep-copies this tokenizer into a thread + # pool right after this processor is constructed; the copies never + # see mutations made afterwards. Force it now, before that happens, + # on the tokenizer we actually use (get_hf_processor()'s cache can + # otherwise hand back one bound to a self-loaded tokenizer instead). + hf_processor = self.get_hf_processor() + hf_processor.tokenizer = self.get_tokenizer() + hf_processor.extra_special_tokens # noqa: B018 + def get_hf_config(self): return self.ctx.get_hf_config() @@ -390,7 +402,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]) out_mm_kwargs: MultiModalKwargsItems, ) -> list[PromptReplacement]: hf_processor = self.info.get_hf_processor() - placeholder = { "image": hf_processor.get_token_value("image_token"), "video": hf_processor.get_token_value("video_token"), @@ -438,7 +449,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP): config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - self.config: PretrainedConfig = config + self.config: PreTrainedConfig = config with self._mark_language_model(vllm_config): self.llm = init_vllm_registered_model( diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 69dae97d8d..1fde4cd0a1 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -23,7 +23,7 @@ import numpy as np import torch import torch.nn as nn from einops import rearrange -from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig +from transformers import BaseImageProcessor, BatchFeature, PreTrainedConfig from transformers.activations import GELUActivation from transformers.image_utils import ChannelDimension from transformers.modeling_outputs import ( @@ -318,8 +318,8 @@ class PaddleOCRVLMultiModalProcessor( class Projector(nn.Module): def __init__( self, - text_config: PretrainedConfig, - vision_config: PretrainedConfig, + text_config: PreTrainedConfig, + vision_config: PreTrainedConfig, prefix: str = "", ): super().__init__() @@ -392,7 +392,7 @@ class PaddleOCRImagePixelInputs(TensorSchema): class SiglipVisionEmbeddings(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config self.embed_dim = config.hidden_size @@ -682,7 +682,7 @@ class SigLIPRotaryEmbedding(nn.Module): class SiglipEncoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -735,7 +735,7 @@ class SiglipEncoderLayer(nn.Module): class SiglipEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): @@ -837,7 +837,7 @@ class SiglipEncoder(nn.Module): class SiglipVisionTransformer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ): diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py index 2dd527bb65..652746e5ae 100644 --- a/vllm/model_executor/models/parakeet.py +++ b/vllm/model_executor/models/parakeet.py @@ -10,7 +10,7 @@ import numpy as np import torch import torch.nn as nn from transformers import ParakeetEncoder as HFParakeetEncoder -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.audio_utils import mel_filter_bank from vllm.logger import init_logger @@ -47,7 +47,7 @@ class ParakeetProjection(nn.Module): class ProjectedParakeet(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, *, dtype: torch.dtype, llm_hidden_size: int, @@ -135,7 +135,7 @@ LOG_ZERO_GUARD_VALUE = 2**-24 class ParakeetExtractor: - def __init__(self, config: PretrainedConfig) -> None: + def __init__(self, config: PreTrainedConfig) -> None: self.config = ExtractorConfig.from_hf_config(config) """`config` is named *exactly* for `._get_subsampling_output_length` below""" self._clip_target_samples = int( @@ -329,6 +329,6 @@ class ParakeetExtractor: } @staticmethod - def audio_length(raw_config: PretrainedConfig, audio_tokens: int) -> int: + def audio_length(raw_config: PreTrainedConfig, audio_tokens: int) -> int: config = ExtractorConfig.from_hf_config(raw_config) return int(audio_tokens * config.subsampling_factor * config.hop_length) diff --git a/vllm/model_executor/models/param2moe.py b/vllm/model_executor/models/param2moe.py index 8181097342..c635eab297 100644 --- a/vllm/model_executor/models/param2moe.py +++ b/vllm/model_executor/models/param2moe.py @@ -22,7 +22,6 @@ from collections.abc import Iterable, Iterator from itertools import islice import torch -import torch.nn.functional as F from torch import nn from vllm.config import CacheConfig, VllmConfig @@ -34,6 +33,7 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, MoERunner, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -289,10 +289,12 @@ class Param2MoEMoEBlock(nn.Module): self.norm_expert_prob: bool = getattr(config, "norm_topk_prob", True) self.score_function: str = getattr(config, "score_function", "sigmoid") - self.gate = nn.Linear( + self.gate = GateLinear( self.hidden_size, self.num_experts, - bias=False, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", ) if getattr(config, "moe_router_enable_expert_bias", True): @@ -350,15 +352,11 @@ class Param2MoEMoEBlock(nn.Module): num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) - # Router: both input and weight must be float32 for numerical - # stability (mirrors the original Param2MoEGate behaviour). - # The gate nn.Linear weight lives in the model dtype (bfloat16), - # so we must cast both explicitly via F.linear instead of calling - # self.gate() which would hit a dtype mismatch. - router_logits = F.linear( - hidden_states.float(), - self.gate.weight.float(), - ).to(hidden_states.dtype) + # Router logits are accumulated in float32 for numerical stability + # (mirrors the original Param2MoEGate behaviour) and handed to the + # experts in the activation dtype. + router_logits, _ = self.gate(hidden_states) + router_logits = router_logits.to(hidden_states.dtype) expert_output = self.experts( hidden_states=hidden_states, diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index 2486dc3e8c..e300ef6a44 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -24,7 +24,7 @@ import torch.nn as nn from transformers import ( BatchFeature, CLIPVisionConfig, - PretrainedConfig, + PreTrainedConfig, ProcessorMixin, ) @@ -95,7 +95,7 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( def _init_img_processor( - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> CLIPVisionModel: @@ -165,7 +165,7 @@ class Phi3HDImageEmbedding(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> None: diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 86cb485967..8524455975 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -10,7 +10,7 @@ import torch import torch.nn as nn from transformers import ( BatchFeature, - PretrainedConfig, + PreTrainedConfig, ProcessorMixin, SequenceFeatureExtractor, SiglipVisionConfig, @@ -123,7 +123,7 @@ class Phi4MMImageEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", model_dir: str = "", diff --git a/vllm/model_executor/models/phi4mm_audio.py b/vllm/model_executor/models/phi4mm_audio.py index 8f6df6a548..2a16b4ca75 100644 --- a/vllm/model_executor/models/phi4mm_audio.py +++ b/vllm/model_executor/models/phi4mm_audio.py @@ -18,7 +18,7 @@ from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( CheckpointWrapper, ) from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.models.phi4mm_utils import ( AbsolutePositionalEncoding, @@ -1152,7 +1152,7 @@ class WindowQformer(nn.Module): class AudioEmbedding(nn.Module): """Image embedding.""" - def __init__(self, config: PretrainedConfig, **kwargs: Any) -> None: + def __init__(self, config: PreTrainedConfig, **kwargs: Any) -> None: super().__init__() self.config = config # n_embed or hidden_size for text LM diff --git a/vllm/model_executor/models/phi4siglip.py b/vllm/model_executor/models/phi4siglip.py index 078f3155c2..e9ad4a795d 100644 --- a/vllm/model_executor/models/phi4siglip.py +++ b/vllm/model_executor/models/phi4siglip.py @@ -11,7 +11,7 @@ from typing import Annotated, Any, Literal import torch import torch.nn as nn -from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig +from transformers import BatchFeature, PreTrainedConfig, Siglip2VisionConfig from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -235,7 +235,7 @@ class Phi4ForCausalLMV(nn.Module, SupportsMultiModal, SupportsPP): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() - config: PretrainedConfig = vllm_config.model_config.hf_config + config: PreTrainedConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config diff --git a/vllm/model_executor/models/phimoe.py b/vllm/model_executor/models/phimoe.py index b3b7b995ee..2cf41a701f 100644 --- a/vllm/model_executor/models/phimoe.py +++ b/vllm/model_executor/models/phimoe.py @@ -29,16 +29,18 @@ from itertools import islice import torch from torch import nn -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.linear import ( QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -60,7 +62,7 @@ from .utils import ( ) -class PhiMoEConfig(PretrainedConfig): +class PhiMoEConfig(PreTrainedConfig): model_type = "phimoe" keys_to_ignore_at_inference = ["past_key_values"] @@ -262,12 +264,10 @@ class PhiMoE(nn.Module): self.hidden_size = hidden_size # Gate always runs at half / full precision for now. - self.gate = ReplicatedLinear( + self.gate = GateLinear( hidden_size, num_experts, - bias=False, params_dtype=params_dtype, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/plamo3.py b/vllm/model_executor/models/plamo3.py index 8d870abe2f..d30a257574 100644 --- a/vllm/model_executor/models/plamo3.py +++ b/vllm/model_executor/models/plamo3.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -54,7 +54,7 @@ from vllm.sequence import IntermediateTensors # Only used for type hinting. if TYPE_CHECKING: - class Plamo3Config(PretrainedConfig): # type: ignore + class Plamo3Config(PreTrainedConfig): # type: ignore model_type: str = "plamo3" hidden_size: int diff --git a/vllm/model_executor/models/qianfan_ocr.py b/vllm/model_executor/models/qianfan_ocr.py index ef2bec1e29..23b1092419 100644 --- a/vllm/model_executor/models/qianfan_ocr.py +++ b/vllm/model_executor/models/qianfan_ocr.py @@ -5,10 +5,16 @@ # The model architecture and weights are fully compatible with InternVLChatModel, # only the config model_type / architectures strings differ. -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kFp8StaticTensorSym, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.transformers_utils.processors.internvl import ( InternVLImageProcessor, @@ -30,14 +36,14 @@ class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo): config = self.get_hf_config() vision_config = config.vision_config - kwargs = self.ctx.get_merged_mm_kwargs(kwargs) - kwargs.setdefault("image_size", vision_config.image_size) - kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) - kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) - kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) - kwargs.setdefault("use_thumbnail", config.use_thumbnail) + merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs) + merged_kwargs.setdefault("image_size", vision_config.image_size) + merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) + merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) + merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) + merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail) - image_processor = InternVLImageProcessor(**kwargs) + image_processor = InternVLImageProcessor(**merged_kwargs) image_size = image_processor.image_size patch_size = vision_config.patch_size downsample_ratio = config.downsample_ratio @@ -67,11 +73,16 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel): """ def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ) -> None: super()._patch_quant_config(config, quant_config) # ignore vit layers to preserve model performance - if isinstance(quant_config, Fp8Config): + if isinstance(quant_config, Fp8Config) or ( + isinstance(quant_config, OnlineQuantizationConfig) + and quant_config.args.linear is not None + and quant_config.args.linear.weight + in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym) + ): _FP8_IGNORED_LAYERS = [ *( layer @@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel): "mlp1.1", "mlp1.3", ] + quant_config.ignored_layers = list(quant_config.ignored_layers) for layer in _FP8_IGNORED_LAYERS: if layer not in quant_config.ignored_layers: quant_config.ignored_layers.append(layer) diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py index 7513936ebb..88600b3767 100644 --- a/vllm/model_executor/models/qwen2.py +++ b/vllm/model_executor/models/qwen2.py @@ -187,6 +187,12 @@ class Qwen2Attention(nn.Module): if attn_type == AttentionType.ENCODER_ONLY else Attention ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = attn_cls( self.num_heads, self.head_dim, @@ -196,12 +202,7 @@ class Qwen2Attention(nn.Module): quant_config=quant_config, attn_type=attn_type, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) def forward( diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index ce7cb058b7..986de3e096 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -29,7 +29,7 @@ from typing import Annotated, Any, Literal import numpy as np import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.feature_extraction_utils import BatchFeature from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import ( Qwen2_5OmniConfig, @@ -42,6 +42,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import ( Qwen2_5OmniProcessor, ) from transformers.models.whisper import WhisperFeatureExtractor +from typing_extensions import TypeVar from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -63,7 +64,10 @@ from vllm.model_executor.models.qwen2_audio import ( Qwen2AudioProcessingInfo, _get_feat_extract_output_lengths, ) -from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser +from vllm.model_executor.models.qwen2_vl import ( + Qwen2VLMultiModalDataParser, + Qwen2VLProcessingInfo, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( ImageItem, @@ -79,6 +83,7 @@ from vllm.multimodal.parse import ( ) from vllm.multimodal.processing import ( BaseDummyInputsBuilder, + BaseProcessingInfo, ) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, @@ -118,6 +123,14 @@ except (ImportError, ModuleNotFoundError): logger = init_logger(__name__) +def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor: + data = feature.data + assert data is not None + tensor = data[key].data + assert isinstance(tensor, torch.Tensor) + return tensor + + def unpad_and_flat_audio_features( input_audio_features: torch.Tensor, audio_feature_lengths: torch.Tensor, @@ -282,7 +295,7 @@ def create_qwen2_5_omni_thinker_field_factory( def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]): audio_feature_lengths = hf_inputs.get( "audio_feature_lengths", torch.empty((0,)) - ) + ).flatten() image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3))) image_pixel_grid_sizes = image_grid_thw.prod(-1) @@ -332,6 +345,14 @@ def create_qwen2_5_omni_thinker_field_factory( class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser): + embedding_fields = { + **Qwen2VLMultiModalDataParser.embedding_fields, + "audio": { + "input_audio_features": "values", + "audio_feature_lengths": "metadata", + }, + } + def __init__(self, spatial_merge_size: int, *args, **kwargs): self._spatial_merge_size = spatial_merge_size super().__init__(self._spatial_merge_size, *args, **kwargs) @@ -339,12 +360,17 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser): def _parse_audio_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("audio") + data = dict(data) + if "audio_feature_lengths" in data: + data["audio_feature_lengths"] = data["audio_feature_lengths"].flatten() return DictEmbeddingItems( data, modality="audio", - required_fields={"input_audio_features", "audio_feature_lengths"}, + required_fields=required, + optional_fields=optional, fields_factory=create_qwen2_5_omni_thinker_field_factory( self._spatial_merge_size ), @@ -362,7 +388,6 @@ class Qwen2_5OmniThinkerProcessingInfo( def get_hf_processor(self, **kwargs: object) -> Qwen2_5OmniProcessor: return self.ctx.get_hf_processor( Qwen2_5OmniProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) @@ -394,7 +419,7 @@ class Qwen2_5OmniThinkerProcessingInfo( self, seq_len: int, mm_counts: Mapping[str, int] | None = None, - ) -> Mapping[str, int] | None: + ) -> Mapping[str, int]: mm_counts = mm_counts or {} requested_modalities = {m for m, c in mm_counts.items() if c > 0} mm_max_tokens: dict[str, int] = {} @@ -424,9 +449,13 @@ class Qwen2_5OmniThinkerProcessingInfo( return mm_max_tokens -class Qwen2_5OmniThinkerDummyInputsBuilder( - BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo] -): +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo) +_VI = TypeVar( + "_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo +) + + +class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_audios = mm_counts.get("audio", 0) num_images = mm_counts.get("image", 0) @@ -450,7 +479,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( mm_counts: Mapping[str, int], mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - feature_extractor = self.info.get_feature_extractor() + feature_extractor = self.info.get_hf_processor().feature_extractor target_audio_length = ( min( @@ -489,9 +518,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( return mm_data -class Qwen2_5OmniThinkerMultiModalProcessor( - BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo] -): +class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]): def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) @@ -639,6 +666,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( for item in mm_kwargs["video"]: if item and item.get("use_audio_in_video"): use_audio_in_video_tensor = item["use_audio_in_video"].data + assert isinstance(use_audio_in_video_tensor, torch.Tensor) if use_audio_in_video_tensor.numel() > 0: use_audio_in_video = bool(use_audio_in_video_tensor.item()) break @@ -680,7 +708,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( @classmethod def omni_get_updates_use_audio_in_video( cls, - thinker_config: PretrainedConfig, + thinker_config: PreTrainedConfig, audio_len: int, video_grid_thw: list[int] | torch.Tensor, video_second_per_grid_t: float, @@ -737,6 +765,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ) -> Sequence[PromptUpdate]: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() + assert isinstance(self.info, Qwen2VLProcessingInfo) image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) vocab = tokenizer.get_vocab() @@ -774,6 +803,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" @@ -875,6 +905,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor( class Qwen2_5OmniConditionalGenerationMixin: + audio_tower: nn.Module + visual: nn.Module + vllm_config: VllmConfig + def _parse_and_validate_audio_input( self, **kwargs: object ) -> Qwen2_5OmniAudioFeatureInputs | None: @@ -904,7 +938,7 @@ class Qwen2_5OmniConditionalGenerationMixin: def _parse_and_validate_image_input( self, - **kwargs: dict[str, Any], + **kwargs: object, ) -> Qwen2_5_VLImageInputs | None: pixel_values = kwargs.pop("pixel_values", None) image_embeds = kwargs.pop("image_embeds", None) @@ -920,7 +954,7 @@ class Qwen2_5OmniConditionalGenerationMixin: image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -929,7 +963,7 @@ class Qwen2_5OmniConditionalGenerationMixin: def _parse_and_validate_video_input( self, - **kwargs: dict[str, Any], + **kwargs: object, ) -> Qwen2_5_VLVideoInputs | None: pixel_values_videos = kwargs.pop("pixel_values_videos", None) video_embeds = kwargs.pop("video_embeds", None) @@ -945,7 +979,7 @@ class Qwen2_5OmniConditionalGenerationMixin: video_grid_thw=video_grid_thw, ) - if video_embeds is not None: + else: if not isinstance(video_embeds, torch.Tensor): raise ValueError( "Incorrect type of video embeddings. " @@ -1118,7 +1152,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration( ) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, + Qwen2_5_VLImageInputs + | Qwen2_5_VLVideoInputs + | Qwen2_5OmniAudioFeatureInputs + | None, + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1163,8 +1203,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( f for f in mm_features if f.modality == "video" + and f.data is not None and f.data.get("use_audio_in_video") - and f.data["use_audio_in_video"].data.item() + and _get_feature_tensor(f, "use_audio_in_video").item() ] audios = [f for f in mm_features if f.modality == "audio"] @@ -1173,7 +1214,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( paired_audio_offsets: set[int] = set() for i, video_f in enumerate(videos_with_audio): if i < len(audios): - audio_len = audios[i].data["audio_feature_lengths"].data.item() + audio_len = int( + _get_feature_tensor(audios[i], "audio_feature_lengths").item() + ) mapping[video_f.mm_position.offset] = audio_len paired_audio_offsets.add(audios[i].mm_position.offset) return mapping, paired_audio_offsets @@ -1208,9 +1251,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration( for mm_feature in sorted_features: offset = mm_feature.mm_position.offset modality = mm_feature.modality + assert mm_feature.data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist() yield ( offset, "image", @@ -1222,16 +1266,16 @@ class Qwen2_5OmniThinkerForConditionalGeneration( }, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist() second_per_grid_ts = 1.0 if mm_feature.data.get("second_per_grid_ts"): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts = _get_feature_tensor( + mm_feature, "second_per_grid_ts" + ).item() use_audio_in_video = False if mm_feature.data.get("use_audio_in_video"): use_audio_in_video = bool( - mm_feature.data["use_audio_in_video"].data.item() + _get_feature_tensor(mm_feature, "use_audio_in_video").item() ) yield ( @@ -1249,7 +1293,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( elif modality == "audio": # Skip audio that's paired with video (handled in video case) if offset not in paired_audio_offsets: - audio_len = mm_feature.data["audio_feature_lengths"].data.item() + audio_len = _get_feature_tensor( + mm_feature, "audio_feature_lengths" + ).item() yield offset, "audio", {"audio_feature_length": audio_len} def _compute_interleaved_positions( diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 381ffb03c3..864a77f095 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -55,6 +55,10 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_and_mul_fn from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -120,7 +124,6 @@ from .utils import ( maybe_prefix, ) from .vision import ( - FusedInputNorm, get_fp8_padded_hidden_size, get_vit_attn_backend, is_vit_use_data_parallel, @@ -651,7 +654,10 @@ class Qwen2_5_VisionTransformer(nn.Module): self.spatial_merge_size = vision_config.spatial_merge_size self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.spatial_merge_unit = self.spatial_merge_size**2 - self._rope_by_thw_cache = LRUCache(capacity=1024) + self._rope_by_thw_cache: LRUCache[ + tuple[int, int, int], + tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], + ] = LRUCache(capacity=1024) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( 1 @@ -664,9 +670,7 @@ class Qwen2_5_VisionTransformer(nn.Module): in_channels=in_channels, hidden_size=self.hidden_size, ) - self.input_norm = ( - input_norm if input_norm is not None else FusedInputNorm.identity() - ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -884,7 +888,7 @@ class Qwen2_5_VisionTransformer(nn.Module): def prepare_encoder_metadata( self, - grid_thw: list[list[int]], + grid_thw: Sequence[Sequence[int]], *, max_batch_size: int | None = None, max_frames_per_batch: int | None = None, @@ -926,9 +930,9 @@ class Qwen2_5_VisionTransformer(nn.Module): # patchify rotary_pos_emb_cos = [] rotary_pos_emb_sin = [] - window_index: list = [] - cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)] - cu_seqlens: list = [] + window_indices: list[torch.Tensor] = [] + cu_window_seqlens_parts = [torch.tensor([0], dtype=torch.int32)] + cu_seqlens_parts: list[torch.Tensor] = [] window_index_id = 0 cu_window_seqlens_last = 0 @@ -945,26 +949,26 @@ class Qwen2_5_VisionTransformer(nn.Module): cu_seqlens_thw, ) = self.get_rope_by_thw(t, h, w) - window_index.append(window_index_thw + window_index_id) + window_indices.append(window_index_thw + window_index_id) window_index_id += t * llm_h * llm_w cu_seqlens_window_thw = cu_seqlens_window_thw + cu_window_seqlens_last cu_window_seqlens_last = cu_seqlens_window_thw[-1] - cu_window_seqlens.append(cu_seqlens_window_thw) + cu_window_seqlens_parts.append(cu_seqlens_window_thw) rotary_pos_emb_cos.append(cos_thw) rotary_pos_emb_sin.append(sin_thw) - cu_seqlens.append(cu_seqlens_thw) + cu_seqlens_parts.append(cu_seqlens_thw) rotary_pos_emb_cos = torch.cat(rotary_pos_emb_cos) rotary_pos_emb_sin = torch.cat(rotary_pos_emb_sin) - window_index = torch.cat(window_index) + window_index = torch.cat(window_indices) # compute reverse indices reverse_indices = self.invert_permutation(window_index) - cu_window_seqlens = torch.cat(cu_window_seqlens) + cu_window_seqlens = torch.cat(cu_window_seqlens_parts) cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) - cu_seqlens = torch.cat(cu_seqlens) + cu_seqlens = torch.cat(cu_seqlens_parts) cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32) cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) @@ -1087,6 +1091,7 @@ class Qwen2_5_VisionTransformer(nn.Module): seq_len = hidden_states.shape[0] if encoder_metadata is None: + assert grid_thw is not None encoder_metadata = self.prepare_encoder_metadata(grid_thw) rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"] @@ -1161,7 +1166,7 @@ class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder): ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -1183,7 +1188,6 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Qwen2_5_VLProcessor: return self.ctx.get_hf_processor( Qwen2_5_VLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) @@ -1216,6 +1220,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor): hf_data = hf_inputs.hf_data if videos := hf_data.get("videos"): + assert isinstance(videos, list) # HF expects video metadata as a separate argument. hf_data["videos"] = [video for video, _ in videos] hf_data["video_metadata"] = [ @@ -1336,17 +1341,22 @@ class Qwen2_5_VLForConditionalGeneration( tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid = data["image_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0 elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid = data["video_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() second_per_grid_ts = 1.0 - if mm_feature.data.get("second_per_grid_ts", None): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + if seconds := data.get("second_per_grid_ts"): + assert isinstance(seconds.data, torch.Tensor) + second_per_grid_ts = seconds.data.item() t_factor = second_per_grid_ts * tokens_per_second yield ( offset, @@ -1409,7 +1419,7 @@ class Qwen2_5_VLForConditionalGeneration( def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config @@ -1426,7 +1436,7 @@ class Qwen2_5_VLForConditionalGeneration( vision_config=config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=self.quant_config, - input_norm=FusedInputNorm.from_model_config(self.model_config), + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -1458,7 +1468,7 @@ class Qwen2_5_VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -1484,7 +1494,7 @@ class Qwen2_5_VLForConditionalGeneration( second_per_grid_ts=second_per_grid_ts, ) - if video_embeds is not None: + else: return Qwen2_5_VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -1549,8 +1559,7 @@ class Qwen2_5_VLForConditionalGeneration( ) emb = torch.cat([emb, positions], dim=1) image_embeds_out.append(emb) - image_embeds_split = image_embeds_out - return tuple(image_embeds_split) + return tuple(image_embeds_out) def _process_video_input( self, video_input: Qwen2_5_VLVideoInputs @@ -1614,6 +1623,7 @@ class Qwen2_5_VLForConditionalGeneration( tokens_per_second = self.config.vision_config.tokens_per_second video_embeds_out = [] + assert self.video_pruning_rate is not None for emb, size, video_second_per_grid_t in zip( video_embeds_split, grid_thw_list, second_per_grid_ts ): @@ -1702,7 +1712,9 @@ class Qwen2_5_VLForConditionalGeneration( return mm_embeddings_out, positions, mrope_positions_delta def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1808,6 +1820,7 @@ class Qwen2_5_VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen2VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index b643fe4227..a305eaba89 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -128,17 +128,22 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor): def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]): return dict( audio_embeds=MultiModalFieldConfig.batched("audio"), + audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), input_features=MultiModalFieldConfig.batched("audio"), feature_attention_mask=MultiModalFieldConfig.batched("audio"), ) class Qwen2AudioMultiModalDataParser(MultiModalDataParser): + embedding_fields = { + "audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"}, + } + def _parse_audio_data( self, data: dict[str, torch.Tensor] | ModalityData[AudioItem], ) -> ModalityDataItems[Any, Any] | None: - if isinstance(data, dict): + if isinstance(data, dict) and "audio_num_tokens" not in data: return DictEmbeddingItems( data, modality="audio", @@ -169,6 +174,7 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo): target_sr=feature_extractor.sampling_rate, target_channels=self.get_target_channels(), expected_hidden_size=self._get_expected_hidden_size(), + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_target_channels(self) -> int: @@ -279,10 +285,15 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing audio_output_lengths = audio_output_lens.tolist() def get_replacement_qwen2_audio(item_idx: int): - if audio_output_lengths: + if "audio_num_tokens" in out_mm_data: + audio_num_tokens = out_mm_data["audio_num_tokens"] + assert isinstance(audio_num_tokens, torch.Tensor) + num_features = int(audio_num_tokens[item_idx]) + elif audio_output_lengths: num_features = audio_output_lengths[item_idx] else: audio_embeds = out_mm_data["audio_embeds"][item_idx] + assert isinstance(audio_embeds, torch.Tensor) assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor" num_features = audio_embeds.shape[0] diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index 3d1345397f..1c095d6f09 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -40,7 +40,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -136,11 +139,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module): f"the number of experts {config.num_experts}." ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -152,6 +153,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert_gate", ) + self.shared_expert: Qwen2MoeMLP | None if config.shared_expert_intermediate_size > 0: self.shared_expert = Qwen2MoeMLP( hidden_size=config.hidden_size, @@ -251,6 +253,12 @@ class Qwen2MoeAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = Attention( self.num_heads, self.head_dim, @@ -259,12 +267,7 @@ class Qwen2MoeAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) def forward( diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 4bbddb0167..3dae3ffd2a 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -51,6 +51,10 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import QuickGELU from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, RowParallelLinear, @@ -105,7 +109,6 @@ from .utils import ( maybe_prefix, ) from .vision import ( - FusedInputNorm, get_vit_attn_backend, is_vit_use_data_parallel, run_dp_sharded_mrope_vision_model, @@ -562,9 +565,7 @@ class Qwen2VisionTransformer(nn.Module): in_channels=in_channels, embed_dim=embed_dim, ) - self.input_norm = ( - input_norm if input_norm is not None else FusedInputNorm.identity() - ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(nn.LayerNorm, eps=norm_eps) head_dim = embed_dim // num_heads @@ -848,7 +849,6 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Qwen2VLProcessor: return self.ctx.get_hf_processor( Qwen2VLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) @@ -1211,17 +1211,23 @@ class Qwen2VLForConditionalGeneration( tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + image_grid_thw = data["image_grid_thw"] + assert isinstance(image_grid_thw.data, torch.Tensor) + t, h, w = image_grid_thw.data.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0 elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + video_grid_thw = data["video_grid_thw"] + assert isinstance(video_grid_thw.data, torch.Tensor) + t, h, w = video_grid_thw.data.tolist() second_per_grid_ts = 1.0 - if mm_feature.data.get("second_per_grid_ts", None): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts_field = data.get("second_per_grid_ts") + if second_per_grid_ts_field is not None: + assert isinstance(second_per_grid_ts_field.data, torch.Tensor) + second_per_grid_ts = second_per_grid_ts_field.data.item() t_factor = second_per_grid_ts * tokens_per_second yield ( offset, @@ -1285,7 +1291,7 @@ class Qwen2VLForConditionalGeneration( super().__init__() config: Qwen2VLConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.model_config = vllm_config.model_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config @@ -1296,7 +1302,7 @@ class Qwen2VLForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, - input_norm=FusedInputNorm.from_model_config(self.model_config), + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -1328,7 +1334,7 @@ class Qwen2VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -1352,7 +1358,7 @@ class Qwen2VLForConditionalGeneration( video_grid_thw=video_grid_thw, ) - if video_embeds is not None: + else: return Qwen2VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -1411,7 +1417,7 @@ class Qwen2VLForConditionalGeneration( return video_embeds.split(sizes) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + modalities: dict[str, Qwen2VLImageInputs | Qwen2VLVideoInputs | None] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1481,6 +1487,7 @@ class Qwen2VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen2VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, diff --git a/vllm/model_executor/models/qwen3.py b/vllm/model_executor/models/qwen3.py index 79434bfac9..7a1d76234f 100644 --- a/vllm/model_executor/models/qwen3.py +++ b/vllm/model_executor/models/qwen3.py @@ -130,6 +130,12 @@ class Qwen3Attention(nn.Module): if attn_type == AttentionType.ENCODER_ONLY else Attention ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = attn_cls( self.num_heads, self.head_dim, @@ -140,12 +146,7 @@ class Qwen3Attention(nn.Module): per_layer_sliding_window=per_layer_sliding_window, prefix=f"{prefix}.attn", attn_type=attn_type, - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 9204d5305d..ac27296461 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -25,6 +25,7 @@ """Inference-only Qwen3.5 Series compatible with HuggingFace weights.""" from collections.abc import Iterable +from typing import ClassVar import torch from torch import nn @@ -54,6 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig @@ -442,7 +444,7 @@ class Qwen3_5ForCausalLMBase( def get_mrope_input_positions( self, input_tokens: list[int], - mm_features: list[object], + mm_features: list[MultiModalFeatureSpec], ) -> tuple[torch.Tensor, int]: positions = torch.arange(len(input_tokens), dtype=torch.long) return positions.unsqueeze(0).expand(3, -1), 0 @@ -488,7 +490,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) nn.Module.__init__(self) config: Qwen3_5Config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -648,6 +650,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts): + language_model: Qwen3_5ForCausalLM + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, @@ -700,14 +705,14 @@ class Qwen3_5MoeForConditionalGeneration( Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts ): # For MoE LoRA weights loading - is_3d_moe_weight: bool = True + is_3d_moe_weight: ClassVar[bool] = True def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"): # protocols have not __init__ method, so we need to use nn.Module.__init__ nn.Module.__init__(self) config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 7fb699b183..421292e880 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -286,7 +286,7 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP): return inputs_embeds - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 241e26fa4e..e8e291d8a5 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -189,7 +189,6 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor: processor = self.ctx.get_hf_processor( Qwen3ASRProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) if not hasattr(processor, "audio_token"): @@ -275,7 +274,7 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser): class Qwen3ASRMultiModalProcessor( - Qwen3OmniMoeThinkerMultiModalProcessor, + Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo] ): def _get_mm_fields_config( self, @@ -317,6 +316,7 @@ class Qwen3ASRMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" diff --git a/vllm/model_executor/models/qwen3_asr_forced_aligner.py b/vllm/model_executor/models/qwen3_asr_forced_aligner.py index 2cbf4c1111..a7aad6628d 100644 --- a/vllm/model_executor/models/qwen3_asr_forced_aligner.py +++ b/vllm/model_executor/models/qwen3_asr_forced_aligner.py @@ -72,8 +72,9 @@ class Qwen3ASRForcedAlignerForTokenClassification( # Remove the unused generation head created by the base class; # the forced aligner uses a classifier head instead. - self.language_model.lm_head = None - self.language_model.logits_processor = None + language_model: nn.Module = self.language_model + language_model.lm_head = None + language_model.logits_processor = None self.classify_num = thinker_config.classify_num diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py index 6eb762b628..ef6a0be73b 100644 --- a/vllm/model_executor/models/qwen3_asr_realtime.py +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -120,6 +120,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): ) audio_data = audios[0] + assert audio_data is not None audio_feature_lengths = audio_data.get("audio_feature_lengths") if audio_feature_lengths is not None: if isinstance(audio_feature_lengths.data, torch.Tensor): diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 1636e3852c..36a6c4c2c1 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -4,6 +4,7 @@ import io from collections.abc import Iterable +import regex as re import torch import torch.nn.functional as F from torch import nn @@ -23,6 +24,7 @@ from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig @@ -54,6 +56,31 @@ logger = init_logger(__name__) _SLIDING_ATTENTION = "sliding_attention" +_DRAFT_LAYER_PATTERN = re.compile(r"(? None: + """Add runtime layer aliases for checkpoint-local quant exclusions.""" + if quant_config is None or start_layer_id == 0: + return + exclusions = getattr(quant_config, "exclude_modules", None) + if not isinstance(exclusions, list): + return + + def offset_local_layer(match: re.Match[str]) -> str: + layer_idx = int(match.group(1)) + if layer_idx >= num_hidden_layers: + return match.group(0) + return f"layers.{layer_idx + start_layer_id}" + + for exclusion in tuple(exclusions): + global_exclusion = _DRAFT_LAYER_PATTERN.sub(offset_local_layer, exclusion) + if global_exclusion != exclusion and global_exclusion not in exclusions: + exclusions.append(global_exclusion) def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool: @@ -65,7 +92,9 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool: if override is not None: return bool(override) layer_types = getattr(config, "layer_types", None) - return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION + if not layer_types: + return False + return layer_types[layer_idx] == _SLIDING_ATTENTION def dflash_has_any_non_causal(config: Qwen3Config) -> bool: @@ -78,6 +107,7 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool: def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int: spec_config = vllm_config.speculative_config + assert spec_config is not None config = spec_config.draft_model_config.hf_config aux_layers = get_eagle3_aux_layers_from_config(spec_config) num_features_to_use = len(aux_layers) if aux_layers else config.num_hidden_layers @@ -391,9 +421,14 @@ class DFlashQwen3Model(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size self.quant_config = get_draft_quant_config(vllm_config) + _add_global_draft_layer_exclusions( + self.quant_config, start_layer_id, self.config.num_hidden_layers + ) drafter_config = getattr(self.config, "eagle_config", {}) drafter_config.update(getattr(self.config, "dflash_config", {})) @@ -473,14 +508,25 @@ class DFlashQwen3Model(nn.Module): has_bias: bool, ) -> None: self._hidden_norm_weight = self.hidden_norm.weight.data + self._context_qkv_projs = [a.qkv_proj for a in layers_attn] + self._context_q_sizes = [a.q_size for a in layers_attn] - # KV projection weights: [num_layers * 2 * kv_size, hidden_size] - kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn] - self._fused_kv_weight = torch.cat(kv_weights, dim=0) - if has_bias: - kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn] - self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0) + if all( + isinstance(proj.quant_method, UnquantizedLinearMethod) + for proj in self._context_qkv_projs + ): + # KV projection weights: [num_layers * 2 * kv_size, hidden_size] + kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn] + self._fused_kv_weight: torch.Tensor | None = torch.cat(kv_weights, dim=0) + if has_bias: + kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn] + self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0) + else: + self._fused_kv_bias = None else: + # Quantized linear weights may use packed storage that cannot be + # consumed by F.linear. Run each projection through its quant method. + self._fused_kv_weight = None self._fused_kv_bias = None # K-norm weights stacked into one contiguous [num_layers, head_dim] @@ -548,17 +594,26 @@ class DFlashQwen3Model(nn.Module): self._hidden_norm_weight, self._rms_norm_eps, ) - all_kv_flat = F.linear( - normed_context_states, self._fused_kv_weight, self._fused_kv_bias - ) + if self._fused_kv_weight is not None: + all_kv_flat = F.linear( + normed_context_states, self._fused_kv_weight, self._fused_kv_bias + ) + all_kv = all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim) + else: + layer_kv = [] + for proj, q_size in zip( + self._context_qkv_projs, self._context_q_sizes, strict=True + ): + qkv, _ = proj(normed_context_states) + layer_kv.append( + qkv[:, q_size:].view(num_ctx, 2, num_kv_heads, head_dim) + ) + all_kv = torch.stack(layer_kv, dim=1) + # Single contiguous copy that separates K/V and transposes to - # layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous. + # layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous. # Indexing dim-0 gives contiguous [L, num_ctx, nkv, hd] for K and V. - all_kv = ( - all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim) - .permute(2, 1, 0, 3, 4) - .contiguous() - ) + all_kv = all_kv.permute(2, 1, 0, 3, 4).contiguous() all_k = all_kv[0] # [L, num_ctx, nkv, hd], contiguous all_v = all_kv[1] # [L, num_ctx, nkv, hd], contiguous return all_k, all_v @@ -709,7 +764,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.draft_model_config = speculative_config.draft_model_config self.config = self.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: self.config.draft_vocab_size = getattr(self.config, "vocab_size", None) @@ -724,6 +781,11 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): self.lm_head = ParallelLMHead( self.config.draft_vocab_size, self.config.hidden_size, + quant_config=( + get_draft_quant_config(vllm_config) + if getattr(self.config, "has_own_lm_head", False) + else None + ), prefix=maybe_prefix(prefix, "lm_head"), ) self.logits_processor = LogitsProcessor( @@ -746,7 +808,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): ) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, @@ -841,7 +903,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): model_weights["model.mask_embedding"] = mask_embedding self.model.has_separate_mask_embedding = True - orig_to_new_substr = {} + orig_to_new_substr: dict[str, None] = {} if not includes_draft_id_mapping: orig_to_new_substr["draft_id_to_target_id"] = None if not includes_embed_tokens: diff --git a/vllm/model_executor/models/qwen3_dflash2.py b/vllm/model_executor/models/qwen3_dflash2.py index 2043307d02..861a7c5412 100644 --- a/vllm/model_executor/models/qwen3_dflash2.py +++ b/vllm/model_executor/models/qwen3_dflash2.py @@ -176,6 +176,7 @@ class DFlashGroupedConv(nn.Module): block_size: int, params_dtype: torch.dtype, prefix: str, + quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() if hidden_size % group_size: @@ -195,7 +196,7 @@ class DFlashGroupedConv(nn.Module): 2 * taps * self.num_groups, bias=False, params_dtype=params_dtype, - quant_config=None, + quant_config=quant_config, prefix=maybe_prefix(prefix, "kernel_projection"), return_bias=False, ) @@ -254,6 +255,7 @@ class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer): # Query tokens per request: the bonus token plus the mask tokens. block_size=1 + speculative_config.num_speculative_tokens, params_dtype=vllm_config.model_config.dtype, + quant_config=quant_config, ) self.attention_conv = DFlashGroupedConv( **conv_args, prefix=maybe_prefix(prefix, "attention_conv") diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 7318d225f8..156f55c373 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -233,7 +233,9 @@ class Qwen3DSparkModel(DFlashQwen3Model): class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: nn.Module.__init__(self) - self.draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.draft_model_config = speculative_config.draft_model_config self.config = self.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: self.config.draft_vocab_size = getattr(self.config, "vocab_size", None) diff --git a/vllm/model_executor/models/qwen3_eagle3.py b/vllm/model_executor/models/qwen3_eagle3.py index 745c77a977..124c1f82ca 100644 --- a/vllm/model_executor/models/qwen3_eagle3.py +++ b/vllm/model_executor/models/qwen3_eagle3.py @@ -101,7 +101,7 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer): hidden_states = self.hidden_norm(hidden_states) return hidden_states, residual - def forward( + def forward( # type: ignore[override] self, positions: torch.Tensor, embeds: torch.Tensor, @@ -159,7 +159,9 @@ class Qwen3Eagle3Model(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size # Get drafter's quantization config @@ -210,6 +212,7 @@ class Qwen3Eagle3Model(nn.Module): self.config, "target_hidden_size", self.config.hidden_size ) self.fc_input_size = target_hidden_size * num_aux_features + self.input_norm: RMSNorm | None if self.norm_before_fc: self.input_norm = RMSNorm( self.fc_input_size, @@ -282,7 +285,9 @@ class Qwen3Eagle3Model(nn.Module): class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config # Ensure draft_vocab_size is set # default to the base vocab size when absent if getattr(self.config, "draft_vocab_size", None) is None: @@ -318,7 +323,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): else: self.draft_id_to_target_id = None - self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting + self.use_parallel_drafting = speculative_config.parallel_drafting if self.use_parallel_drafting: self.register_buffer( @@ -335,7 +340,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): ) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index 5df982d741..3f293550ac 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -42,7 +42,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -164,10 +167,9 @@ class Qwen3MoeSparseMoeBlock(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, quant_config=quant_config, prefix=f"{prefix}.gate", ) @@ -175,6 +177,8 @@ class Qwen3MoeSparseMoeBlock(nn.Module): shared_expert_intermediate_size = getattr( config, "shared_expert_intermediate_size", 0 ) + self.shared_expert_gate: ReplicatedLinear | None + self.shared_expert: Qwen3MoeMLP | None if shared_expert_intermediate_size > 0: self.shared_expert_gate = ReplicatedLinear( config.hidden_size, @@ -300,6 +304,12 @@ class Qwen3MoeAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = Attention( self.num_heads, self.head_dim, @@ -308,12 +318,7 @@ class Qwen3MoeAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 7ea3ed1319..e431c36f30 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -18,7 +18,10 @@ from vllm.distributed import ( tensor_model_parallel_reduce_scatter, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, +) from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, resolve_layer_fused_shared_expert, @@ -180,11 +183,9 @@ class Qwen3NextSparseMoeBlock(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) @@ -756,6 +757,9 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): class QwenNextMixtureOfExperts(MixtureOfExperts): + model: Qwen3NextModel + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 85d0bcaad0..f9338b1189 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -24,13 +24,13 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import partial -from typing import Any, cast +from typing import Any, ClassVar import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.feature_extraction_utils import BatchFeature from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import ( Qwen3OmniMoeAudioEncoderConfig, @@ -41,12 +41,13 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import ( Qwen3OmniMoeProcessor, ) from transformers.models.whisper import WhisperFeatureExtractor +from typing_extensions import TypeVar from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.inputs import PromptType +from vllm.inputs import PromptType, TextPrompt from vllm.logger import init_logger from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY from vllm.model_executor.layers.attention.mm_encoder_attention import ( @@ -73,6 +74,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems from vllm.multimodal.processing.processor import ( + BaseProcessingInfo, HFMultiModalInputs, MultiModalProcessingResult, MultiModalPromptUpdates, @@ -100,13 +102,17 @@ from .qwen2_5_omni_thinker import ( Qwen2_5OmniAudioFeatureInputs, Qwen2_5OmniConditionalGenerationMixin, Qwen2_5OmniThinkerDummyInputsBuilder, + Qwen2_5OmniThinkerMultiModalDataParser, Qwen2_5OmniThinkerMultiModalProcessor, + _get_feature_tensor, check_interleaved_audio_video, merge_interleaved_embeddings, ) from .qwen2_5_vl import ( Qwen2_5_VisionAttention, + Qwen2_5_VLImageInputs, Qwen2_5_VLProcessingInfo, + Qwen2_5_VLVideoInputs, ) from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel from .utils import ( @@ -962,7 +968,7 @@ class Qwen3Omni_VisionTransformer(nn.Module): def forward( self, x: torch.Tensor, - grid_thw: list[list[int]], + grid_thw: torch.Tensor, ) -> torch.Tensor: hidden_states = x.to(device=self.device, dtype=self.dtype) hidden_states = self.patch_embed(hidden_states) @@ -1152,6 +1158,13 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM): ) +class Qwen3OmniMoeThinkerMultiModalDataParser(Qwen2_5OmniThinkerMultiModalDataParser): + # Keep the existing raw-media fallback for vision inputs. + embedding_fields = { + "audio": Qwen2_5OmniThinkerMultiModalDataParser.embedding_fields["audio"], + } + + class Qwen3OmniMoeThinkerProcessingInfo( Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo ): @@ -1161,7 +1174,6 @@ class Qwen3OmniMoeThinkerProcessingInfo( def get_hf_processor(self, **kwargs: object) -> Qwen3OmniMoeProcessor: processor = self.ctx.get_hf_processor( Qwen3OmniMoeProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) if not hasattr(processor, "audio_token"): @@ -1178,6 +1190,15 @@ class Qwen3OmniMoeThinkerProcessingInfo( assert isinstance(feature_extractor, WhisperFeatureExtractor) return feature_extractor + def get_data_parser(self): + return Qwen3OmniMoeThinkerMultiModalDataParser( + spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size, + target_sr=self.get_feature_extractor().sampling_rate, + target_channels=self.get_target_channels(), + expected_hidden_size=self._get_expected_hidden_size(), + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"audio": None, "image": None, "video": None} @@ -1185,7 +1206,7 @@ class Qwen3OmniMoeThinkerProcessingInfo( self, seq_len: int, mm_counts: Mapping[str, int] | None = None, - ) -> Mapping[str, int] | None: + ) -> Mapping[str, int]: mm_counts = mm_counts or {} requested_modalities = {m for m, c in mm_counts.items() if c > 0} mm_max_tokens: dict[str, int] = {} @@ -1215,11 +1236,15 @@ class Qwen3OmniMoeThinkerProcessingInfo( return mm_max_tokens -Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder +Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder[ + Qwen3OmniMoeThinkerProcessingInfo +] + +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen3OmniMoeThinkerProcessingInfo) class Qwen3OmniMoeThinkerMultiModalProcessor( - Qwen2_5OmniThinkerMultiModalProcessor, + Qwen2_5OmniThinkerMultiModalProcessor[_I], ): def _get_hf_mm_inputs( self, @@ -1230,7 +1255,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( audios = hf_inputs.hf_data.get("audio") if audios: - feature_extractor = self.info.get_feature_extractor() + assert isinstance(audios, (list, tuple)) + feature_extractor = self.info.get_hf_processor().feature_extractor hop_length = feature_extractor.hop_length def pad_to_hop_length(x: np.ndarray) -> np.ndarray: @@ -1267,9 +1293,10 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( ): audios = mm_data.get("audio", []) if audios: - feature_extractor = self.info.get_feature_extractor( + assert isinstance(audios, (list, tuple)) + feature_extractor = self.info.get_hf_processor( **hf_processor_mm_kwargs - ) + ).feature_extractor hop_length = feature_extractor.hop_length audio_num_frames = [] @@ -1351,7 +1378,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( def get_updates_use_audio_in_video( self, - thinker_config: PretrainedConfig, + thinker_config: PreTrainedConfig, audio_len: int, video_grid_thw: list[int] | torch.Tensor, video_second_per_grid_t: float, @@ -1405,6 +1432,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( ) -> Sequence[PromptUpdate]: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() + assert isinstance(self.info, Qwen2_5_VLProcessingInfo) image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) vocab = tokenizer.get_vocab() @@ -1444,6 +1472,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" @@ -1591,8 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( Qwen3OmniMoeConditionalGenerationMixin, SupportsTranscription, ): - is_3d_moe_weight: bool = True - supports_tower_connector_lora: bool = True + is_3d_moe_weight: ClassVar[bool] = True + supports_tower_connector_lora: ClassVar[bool] = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "thinker.lm_head.": "language_model.lm_head.", @@ -1753,7 +1782,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( self.deepstack_input_embeds_num_tokens = 0 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, + Qwen2_5_VLImageInputs + | Qwen2_5_VLVideoInputs + | Qwen2_5OmniAudioFeatureInputs + | None, + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1846,6 +1881,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( if self.visual.deepstack_visual_indexes is not None and any( has_vision_embeddings ): + multimodal_embeddings = list(multimodal_embeddings) multiscale_len = len(self.visual.deepstack_visual_indexes) multimodal_embeddings_multiscale = [] @@ -1990,8 +2026,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( f for f in mm_features if f.modality == "video" + and f.data is not None and f.data.get("use_audio_in_video") - and f.data["use_audio_in_video"].data.item() + and _get_feature_tensor(f, "use_audio_in_video").item() ] audios = [f for f in mm_features if f.modality == "audio"] @@ -1999,7 +2036,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( paired_audio_offsets: set[int] = set() for i, video_f in enumerate(videos_with_audio): if i < len(audios): - audio_len = audios[i].data["audio_feature_lengths"].data.item() + audio_len = int( + _get_feature_tensor(audios[i], "audio_feature_lengths").item() + ) mapping[video_f.mm_position.offset] = audio_len paired_audio_offsets.add(audios[i].mm_position.offset) return mapping, paired_audio_offsets @@ -2027,9 +2066,10 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( for mm_feature in sorted_features: offset = mm_feature.mm_position.offset modality = mm_feature.modality + assert mm_feature.data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist() yield ( offset, "image", @@ -2041,15 +2081,15 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( }, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist() second_per_grid_ts = 2.0 if mm_feature.data.get("second_per_grid_ts"): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts = _get_feature_tensor( + mm_feature, "second_per_grid_ts" + ).item() use_audio_in_video = bool( mm_feature.data.get("use_audio_in_video") - and mm_feature.data["use_audio_in_video"].data.item() + and _get_feature_tensor(mm_feature, "use_audio_in_video").item() ) yield ( @@ -2066,7 +2106,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) elif modality == "audio": if offset not in paired_audio_offsets: - audio_len = mm_feature.data["audio_feature_lengths"].data.item() + audio_len = _get_feature_tensor( + mm_feature, "audio_feature_lengths" + ).item() yield offset, "audio", {"audio_feature_length": audio_len} def _compute_interleaved_positions( @@ -2154,8 +2196,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( to_language = "en" # Get full language names from supported_languages mapping - full_lang_name = cls.supported_languages.get(language, "") - full_lang_name_to = cls.supported_languages.get(to_language, "") + full_lang_name = ( + cls.supported_languages.get(language, "") if language is not None else "" + ) + full_lang_name_to = ( + cls.supported_languages.get(to_language, "") + if to_language is not None + else "" + ) if task_type == "transcribe" and full_lang_name: instruction += f" into {full_lang_name}" @@ -2185,8 +2233,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) audio_data = (audio, stt_config.sample_rate) - prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt} - return cast(PromptType, prompts_dict) + prompts_dict: TextPrompt = { + "multi_modal_data": {"audio": audio_data}, + "prompt": prompt, + } + return prompts_dict def get_mrope_input_positions( self, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index f7b553926c..c9aa0d2066 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -582,7 +582,9 @@ class Qwen3_VisionTransformer(nn.Module): else [] ) self.num_grid_per_side = int(self.num_position_embeddings**0.5) - self._rot_pos_ids_cache = LRUCache(capacity=1024) + self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = ( + LRUCache(capacity=1024) + ) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( @@ -705,7 +707,7 @@ class Qwen3_VisionTransformer(nn.Module): self._rot_pos_ids_cache[cache_key] = result return result - def rot_pos_emb(self, grid_thw: list[list[int]]): + def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]): max_grid_size = max(max(h, w) for _, h, w in grid_thw) pos_ids = [ self.rot_pos_ids(h, w, self.spatial_merge_size) @@ -731,7 +733,9 @@ class Qwen3_VisionTransformer(nn.Module): return cos_combined, sin_combined - def fast_pos_embed_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor: + def fast_pos_embed_interpolate( + self, grid_thw: Sequence[Sequence[int]] + ) -> torch.Tensor: interpolate_fn = ( triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native ) @@ -752,7 +756,7 @@ class Qwen3_VisionTransformer(nn.Module): def prepare_encoder_metadata( self, - grid_thw_list: list[list[int]], + grid_thw_list: Sequence[Sequence[int]], *, max_batch_size: int | None = None, max_frames_per_batch: int | None = None, @@ -915,7 +919,6 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor: return self.ctx.get_hf_processor( Qwen3VLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) @@ -1006,9 +1009,10 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): self, seq_len: int, mm_counts: Mapping[str, int], + max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES, ) -> int: return super().get_num_frames_with_most_features( - seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES + seq_len, mm_counts, max_frames_per_video=max_frames_per_video ) def get_max_video_tokens( @@ -1254,7 +1258,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -1340,6 +1344,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # are processed into several image patches video_input_ids_lst: list[list[int]] = [] if videos := hf_data.pop("videos", []): + assert isinstance(videos, list) video_grid_thw_lst = [] pixel_values_videos_lst = [] timestamps_per_video = [] @@ -1362,7 +1367,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # NOTE: a copy of is created to update do_sample_frames, # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs = dict(**hf_kwargs) + video_mm_kwargs: dict[str, Any] = dict(hf_kwargs) merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video") if merged.keys() & {"size", "min_pixels", "max_pixels"}: video_size = dict(self.info.get_video_processor().size) @@ -1552,6 +1557,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) sampled_fps = sampled_fps[item_idx] timestamps = out_item["timestamps"].data + assert isinstance(timestamps, (list, torch.Tensor)) assert len(timestamps) == grid_thw[0], ( f"The timestamps length({len(timestamps)}) should be equal " f"video length ({grid_thw[0]})." @@ -1618,7 +1624,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) def get_video_repl( *, tokens_per_frame: list[int], - timestamps: list[float | int] | torch.Tensor, + timestamps: Sequence[float] | torch.Tensor, tokenizer: TokenizerLike, vision_start_token_id: int, vision_end_token_id: int, @@ -1843,7 +1849,7 @@ class Qwen3VLForConditionalGeneration( super().__init__() config: Qwen3VLConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -2007,6 +2013,7 @@ class Qwen3VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen3VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, @@ -2255,7 +2262,7 @@ class Qwen3VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -2283,7 +2290,7 @@ class Qwen3VLForConditionalGeneration( timestamps=timestamps, ) - if video_embeds is not None: + else: return Qwen2_5_VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -2406,6 +2413,7 @@ class Qwen3VLForConditionalGeneration( # Apply EVS to each video. video_embeds_out = [] + assert video_input.timestamps is not None for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)): # Compute positions. timestamps = video_input.timestamps[video_idx] @@ -2413,6 +2421,7 @@ class Qwen3VLForConditionalGeneration( t, h, w = size if self.is_multimodal_pruning_enabled: + assert self.video_pruning_rate is not None # Compute the retention mask for each video (EVS or VidCom2). if self.video_pruning_method == "vidcom2": mask_fn = vidcom2_compute_retention_mask @@ -2584,7 +2593,7 @@ class Qwen3VLForConditionalGeneration( { "video_grid_thw": MultiModalFieldElem( data=torch.tensor(video_grid_thw), - field=None, # HACK. + field=MultiModalFieldConfig.batched("video").field, ), } ), @@ -2612,7 +2621,9 @@ class Qwen3VLForConditionalGeneration( return expanded_positions def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None + ] = {} for input_key in kwargs: if ( input_key in ("pixel_values", "image_embeds") @@ -2660,14 +2671,20 @@ class Qwen3VLForConditionalGeneration( """ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid = data["image_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() assert t == 1, f"Image must have 1 frame, got {t}" llm_grid_h = h // spatial_merge_size llm_grid_w = w // spatial_merge_size yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid = data["video_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() llm_grid_h = h // spatial_merge_size llm_grid_w = w // spatial_merge_size @@ -2722,7 +2739,7 @@ class Qwen3VLForConditionalGeneration( mm_features: list[MultiModalFeatureSpec], config: Qwen3VLConfig, ): - llm_pos_ids_list = [] + llm_pos_ids_list: list[np.ndarray] = [] st = 0 for ( offset, diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index 193298a186..a345f577b1 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -25,6 +25,7 @@ """Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights.""" from itertools import islice +from typing import ClassVar import torch from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import ( @@ -176,6 +177,9 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM): class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts): + language_model: Qwen3MoeLLMForCausalLM + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, @@ -223,7 +227,7 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts): class Qwen3VLMoeForConditionalGeneration( Qwen3VLForConditionalGeneration, Qwen3VLMoeMixtureOfExperts ): - is_3d_moe_weight: bool = True + is_3d_moe_weight: ClassVar[bool] = True packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -236,7 +240,7 @@ class Qwen3VLMoeForConditionalGeneration( super(Qwen3VLForConditionalGeneration, self).__init__() config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -293,8 +297,8 @@ class Qwen3VLMoeForConditionalGeneration( ) # Whether to include the gate_up_proj mapping is determined by - # the language model. - self.packed_modules_mapping = ( + # the language model. Keep this override local to the instance. + self.packed_modules_mapping = ( # type: ignore[misc] self.packed_modules_mapping | self.language_model.packed_modules_mapping ) diff --git a/vllm/model_executor/models/radio.py b/vllm/model_executor/models/radio.py index bc89591324..355bfb056a 100644 --- a/vllm/model_executor/models/radio.py +++ b/vllm/model_executor/models/radio.py @@ -18,7 +18,7 @@ import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -109,7 +109,7 @@ class ClsToken(nn.Module): class ViTPatchGenerator(nn.Module): def __init__( self, - # config: PretrainedConfig, + # config: PreTrainedConfig, patch_size: int, embed_dim: int, input_dims: input_dim_t, @@ -535,7 +535,7 @@ class RadioInternVisionModel(nn.Module): def __init__( self, - config: PretrainedConfig = None, + config: PreTrainedConfig = None, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, @@ -670,7 +670,7 @@ class RadioModel(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, *, num_hidden_layers_override: int | None = None, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index cdb13df8f5..b435d77ad7 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -633,6 +633,7 @@ _SPECULATIVE_DECODING_MODELS = { "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"), "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"), "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"), + "LiLiCorrDraftModel": ("lilicorr", "LiLiCorrForCausalLM"), "DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"), # Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash # implementation. EAGLEConfig rewrites a dflash draft's architecture to diff --git a/vllm/model_executor/models/seed_oss.py b/vllm/model_executor/models/seed_oss.py index df0ed7e72e..976a35a78a 100644 --- a/vllm/model_executor/models/seed_oss.py +++ b/vllm/model_executor/models/seed_oss.py @@ -28,7 +28,7 @@ from itertools import islice import torch from torch import nn -from transformers import PretrainedConfig as SeedOssConfig +from transformers import PreTrainedConfig as SeedOssConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig diff --git a/vllm/model_executor/models/siglip2navit.py b/vllm/model_executor/models/siglip2navit.py index 1d0c3c8d96..ef0181002b 100644 --- a/vllm/model_executor/models/siglip2navit.py +++ b/vllm/model_executor/models/siglip2navit.py @@ -9,7 +9,7 @@ import torch from torch import nn from torch.nn import functional as F from transformers import Siglip2VisionConfig -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.model_executor.layers.activation import get_act_fn @@ -47,7 +47,7 @@ class VisionRotaryEmbedding(nn.Module): class Siglip2VisionEmbeddings(nn.Module): - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config self.embed_dim = config.hidden_size @@ -364,7 +364,7 @@ class Siglip2Encoder(nn.Module): self attention layers. Each layer is a [`Siglip2EncoderLayer`]. Args: - config: PretrainedConfig + config: PreTrainedConfig """ diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index 961ee7acb8..edf7fbeb47 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -12,7 +12,7 @@ from typing import Annotated, Literal, TypeAlias import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear @@ -200,7 +200,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP): ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig | None + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back @@ -214,7 +214,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP): def _init_vision_model( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, *, prefix: str, @@ -236,7 +236,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP): def _init_mlp1( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None, prefix: str = "", ) -> nn.Module: diff --git a/vllm/model_executor/models/solar.py b/vllm/model_executor/models/solar.py index cb175f7c7a..194d7b8ffe 100644 --- a/vllm/model_executor/models/solar.py +++ b/vllm/model_executor/models/solar.py @@ -28,7 +28,7 @@ from collections.abc import Iterable import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -102,7 +102,7 @@ class SolarMLP(nn.Module): class SolarAttention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, num_kv_heads: int, @@ -185,7 +185,7 @@ class SolarAttention(nn.Module): class SolarDecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/step3_text.py b/vllm/model_executor/models/step3_text.py index 57c35ed8ce..5baeeb69eb 100644 --- a/vllm/model_executor/models/step3_text.py +++ b/vllm/model_executor/models/step3_text.py @@ -20,6 +20,7 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -77,11 +78,9 @@ class FusedMoEBlock(nn.Module): quant_config=quant_config, prefix=f"{prefix}.experts", ) - self.gate = ReplicatedLinear( + self.gate = GateLinear( config.hidden_size, config.moe_num_experts, - bias=False, - quant_config=None, prefix=f"{prefix}.gate", ) diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index 9d649ba4c0..b56d4d3ed8 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -8,7 +8,6 @@ from typing import Any import torch from torch import nn -from torch.nn.parameter import Parameter from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -25,6 +24,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + GateLinear, MoERunner, fused_moe_make_expert_params_mapping, ) @@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, QKVParallelLinear, - ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -50,7 +49,7 @@ from vllm.v1.attention.backend import AttentionType if typing.TYPE_CHECKING: from vllm.transformers_utils.configs.step3p5 import Step3p5Config else: - from transformers import PretrainedConfig as Step3p5Config + from transformers import PreTrainedConfig as Step3p5Config from .interfaces import MixtureOfExperts, SupportsPP from .utils import ( @@ -68,17 +67,6 @@ from .utils import ( logger = init_logger(__name__) -class FP32ReplicatedLinear(ReplicatedLinear): - """Use FP32 for higher precision.""" - - def forward( - self, - x: torch.Tensor, - ) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]: - assert self.params_dtype == torch.float32 - return super().forward(x.to(torch.float32)) - - class Step3p5MLP(nn.Module): def __init__( self, @@ -326,12 +314,12 @@ class FusedMoEBlock(nn.Module): f"the number of experts {config.moe_num_experts}." ) - self.gate = FP32ReplicatedLinear( + # Router logits are accumulated in FP32 for higher precision. + self.gate = GateLinear( config.hidden_size, config.moe_num_experts, - bias=False, - quant_config=None, - params_dtype=torch.float32, # Use FP32 for higher precision. + out_dtype=torch.float32, + params_dtype=torch.float32, prefix=f"{prefix}.gate", ) self.use_moe_router_bias = config.use_moe_router_bias diff --git a/vllm/model_executor/models/step3p5_mtp.py b/vllm/model_executor/models/step3p5_mtp.py index ac78384b60..d17b579c1a 100644 --- a/vllm/model_executor/models/step3p5_mtp.py +++ b/vllm/model_executor/models/step3p5_mtp.py @@ -4,7 +4,7 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import VllmConfig from vllm.logger import init_logger @@ -30,7 +30,7 @@ logger = init_logger(__name__) class SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: diff --git a/vllm/model_executor/models/teleflm.py b/vllm/model_executor/models/teleflm.py index bebd7bcaa9..b8235daacb 100644 --- a/vllm/model_executor/models/teleflm.py +++ b/vllm/model_executor/models/teleflm.py @@ -23,47 +23,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch -import torch.nn as nn from vllm.config import VllmConfig from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.models.llama import ( - LlamaDecoderLayer, LlamaForCausalLM, - LlamaModel, ) -class TeleFLMModel(LlamaModel): - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = LlamaDecoderLayer, - ): - super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type) - """ - This implementation is based on the µScaling paper presented at - the ICLR 2025 Workshop: - NanoLM: An Affordable LLM Study Benchmark \ - via Accurate Loss Prediction across Scales - by Yiqun Yao et al. - Available at: https://openreview.net/forum?id=IwaPYg1SCA - arXiv preprint: https://arxiv.org/abs/2304.06875 - """ - self.use_mup = self.config.use_mup - if self.use_mup: - self.input_mult = self.config.input_mult - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - embedding = self.embed_tokens(input_ids) - if self.use_mup: - embedding = embedding * self.input_mult - return embedding - - class TeleFLMForCausalLM(LlamaForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index ff8fc98dbc..ec0f474ee3 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -29,7 +29,7 @@ import torch import transformers from packaging.version import Version from torch import nn -from transformers import AutoModel, PretrainedConfig +from transformers import AutoModel, PreTrainedConfig from transformers.conversion_mapping import ( WeightRenaming, get_model_conversion_mapping, @@ -433,9 +433,9 @@ class Base( def _vocab_embeddings(self) -> set[nn.Embedding]: """The `nn.Embedding`s in `self.model` which hold a vocab table.""" - def vocab_sizes(config: PretrainedConfig): + def vocab_sizes(config: PreTrainedConfig): for key, value in vars(config).items(): - if isinstance(value, PretrainedConfig): + if isinstance(value, PreTrainedConfig): yield from vocab_sizes(value) elif "vocab_size" in key and isinstance(value, int): yield value diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py index f7972f03a6..c1c7ea76ab 100644 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ b/vllm/model_executor/models/transformers/fusers/rms_norm.py @@ -36,33 +36,53 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def _operand(node: fx.Node, index: int, name: str) -> object | None: + """Operand `index` of `node`, whether it was passed positionally or as `name`.""" + if len(node.args) > index: + return node.args[index] + return node.kwargs.get(name) + + def _is_squared(node: object, x: fx.Node) -> bool: """`x**2`, `x.square()` or `x * x`, through any dtype casts.""" node = peel(node) if is_op(node, "pow"): - base, exp = node.args - return peel(base) is x and exp == 2 + return ( + peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2 + ) if is_op(node, "square"): - return peel(node.args[0]) is x + return peel(_operand(node, 0, "input")) is x if is_op(node, "mul"): - a, b = node.args - return peel(a) is x and peel(b) is x + return ( + peel(_operand(node, 0, "input")) is x + and peel(_operand(node, 1, "other")) is x + ) + return False + + +def _is_inverse_sqrt(node: object) -> bool: + """Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it.""" + if is_op(node, "rsqrt"): + return True + if is_op(node, "pow"): + return _operand(node, 1, "exponent") == -0.5 return False def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None: - """Eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape.""" - add = peel(rsqrt.args[0]) + """`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape.""" + add = peel(_operand(rsqrt, 0, "input")) if not is_op(add, "add"): return None - consts = [a for a in add.args if isinstance(a, (int, float))] - nodes = [a for a in add.args if isinstance(a, fx.Node)] + operands = [_operand(add, 0, "input"), _operand(add, 1, "other")] + consts = [a for a in operands if isinstance(a, (int, float))] + nodes = [a for a in operands if isinstance(a, fx.Node)] if len(consts) != 1 or len(nodes) != 1: return None mean = peel(nodes[0]) if not is_op(mean, "mean"): return None - if not _is_squared(mean.args[0], x): + if not _is_squared(_operand(mean, 0, "input"), x): return None return float(consts[0]) @@ -72,7 +92,10 @@ def _is_one_plus(node: object) -> bool: node = peel(node) if not is_op(node, "add"): return False - return any(isinstance(a, (int, float)) and a == 1 for a in node.args) + return any( + isinstance(a, (int, float)) and a == 1 + for a in (_operand(node, 0, "input"), _operand(node, 1, "other")) + ) def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool: @@ -144,7 +167,7 @@ class RMSNormFuser(BaseFuser): return None # Handle native torch `rms_norm` op. rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm")) - if rms_norm is not None and rms_norm.args and peel(rms_norm.args[0]) is x: + if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x: if _has_trailing_compute(graph, rms_norm): return None eps_attr, eps = cls._eps_source(graph, module) @@ -158,14 +181,16 @@ class RMSNormFuser(BaseFuser): # The rsqrt over the mean-square variance is the spine of the norm. rsqrt = None for node in graph.nodes: - if is_op(node, "rsqrt") and _variance_eps(node, x) is not None: + if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None: rsqrt = node break if rsqrt is None: return None # The `x * rsqrt(...)` normalize multiply. normalize = find_node( - graph, lambda n: is_op(n, "mul") and rsqrt in map(peel, n.args) + graph, + lambda n: is_op(n, "mul") + and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))), ) if normalize is None: return None @@ -174,7 +199,11 @@ class RMSNormFuser(BaseFuser): for node in graph.nodes: if not is_op(node, "mul") or node is normalize: continue - operands = [peel(a) for a in node.args if isinstance(a, fx.Node)] + operands = [ + peel(a) + for a in (_operand(node, 0, "input"), _operand(node, 1, "other")) + if isinstance(a, fx.Node) + ] if len(operands) == 2 and normalize in operands: weight = next(o for o in operands if o is not normalize) tail, zero_centered = node, _is_one_plus(weight) @@ -234,12 +263,12 @@ class RMSNormFuser(BaseFuser): if (x := find_node(graph, lambda n: n.op == "placeholder")) is None: return None fused = find_node(graph, lambda n: is_op(n, "rms_norm")) - if fused is not None and fused.args and peel(fused.args[0]) is x: + if fused is not None and peel(_operand(fused, 0, "input")) is x: args, kwargs = fused.args, fused.kwargs eps = args[3] if len(args) > 3 else kwargs.get("eps") return float(eps) if isinstance(eps, (int, float)) else None for node in graph.nodes: - if is_op(node, "rsqrt") and (eps := _variance_eps(node, x)) is not None: + if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None: return eps return None diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index bb92833f77..a454ba5962 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -114,6 +114,12 @@ UltravoxAudioInputs: TypeAlias = ( ) +class UltravoxMultiModalDataParser(MultiModalDataParser): + embedding_fields = { + "audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"}, + } + + class UltravoxProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: config = self.ctx.model_config.hf_config @@ -145,10 +151,11 @@ class UltravoxProcessingInfo(BaseProcessingInfo): def get_data_parser(self): feature_extractor = self.get_feature_extractor() - return MultiModalDataParser( + return UltravoxMultiModalDataParser( target_sr=feature_extractor.sampling_rate, target_channels=self.get_target_channels(), expected_hidden_size=self._get_expected_hidden_size(), + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_target_channels(self) -> int: @@ -247,6 +254,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo # num_chunks can convert audio_chunked to audio batch dimension audio_num_chunks=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), audio_embeds=MultiModalFieldConfig.batched("audio"), + audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), ) def _get_prompt_updates( @@ -273,6 +281,8 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo ) def get_replacement_ultravox(item_idx: int): + if "audio_num_tokens" in out_mm_data: + return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx]) start = chunks_start_idx[item_idx] end = chunks_start_idx[item_idx + 1] audio_token_len = out_mm_data["audio_token_len"][start:end].sum() diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 947a6f6276..6375e0572e 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -32,7 +32,7 @@ from vllm.utils.torch_utils import ( ) if TYPE_CHECKING: - from transformers import PretrainedConfig + from transformers import PreTrainedConfig from transformers.conversion_mapping import WeightRenaming from vllm.config.model import ModelConfig @@ -78,11 +78,15 @@ class WeightsMapper: orig_to_new_suffix={**self.orig_to_new_suffix, **other.orig_to_new_suffix}, ) - def _map_name(self, key: str) -> str | None: - """Map a weight name (backward-compatible wrapper that discards shard_id).""" + def map_name(self, key: str) -> str | None: + """Map a weight name; returns ``None`` if the weight should be ignored.""" result = self._map_name_with_shard(key) return result[0] if result is not None else None + def _map_name(self, key: str) -> str | None: + """Backward-compatible alias for :meth:`map_name`.""" + return self.map_name(key) + def _map_name_with_shard(self, key: str) -> tuple[str, ShardIds | None] | None: """Map a weight name and extract any shard_id metadata. @@ -606,7 +610,7 @@ def init_vllm_registered_model( vllm_config: VllmConfig, *, prefix: str = "", - hf_config: "PretrainedConfig | None" = None, + hf_config: "PreTrainedConfig | None" = None, architectures: list[str] | None = None, ) -> nn.Module: """Helper function to initialize an inner model registered to vLLM, diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index f74a66cdfa..b35db4de27 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -8,10 +8,9 @@ from collections.abc import Callable from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar import torch -import torch.nn as nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none +from vllm.config import MultiModalConfig, get_current_vllm_config_or_none from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -19,13 +18,12 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.transformers_utils.processor import get_processor, get_processor_config from vllm.utils.math_utils import round_up from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) -_C = TypeVar("_C", bound=PretrainedConfig) +_C = TypeVar("_C", bound=PreTrainedConfig) class _RootConfig(Protocol[_C]): @@ -62,7 +60,7 @@ class VisionEncoderInfo(ABC, Generic[_C]): class VisionLanguageConfig(Protocol): - vision_config: Final[PretrainedConfig] + vision_config: Final[PreTrainedConfig] def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo: @@ -578,165 +576,3 @@ def run_dp_sharded_mrope_vision_model( "Found unassigned embeddings" ) return out_embeddings - - -class FusedInputNorm(nn.Module): - """Module that applies rescaling and normalization to input images. - Equivalent to: output = (input * rescale_factor - mean) / std - """ - - def __init__( - self, - image_mean: list[float], - image_std: list[float], - rescale_factor: float, - channel: int = 3, - dtype: torch.dtype = torch.float32, - ): - super().__init__() - - self.channel = channel - - # Model construction can set the accelerator as PyTorch's default - # device. Determine whether the normalization is an identity on CPU - # so torch.allclose does not introduce a device synchronization while - # the model is being initialized. The actual buffers below still use - # the caller's default device. - image_mean_cpu = torch.tensor(image_mean, dtype=dtype, device="cpu") * ( - 1.0 / rescale_factor - ) - image_std_cpu = torch.tensor(image_std, dtype=dtype, device="cpu") * ( - 1.0 / rescale_factor - ) - weight_cpu = 1.0 / image_std_cpu - bias_cpu = -image_mean_cpu / image_std_cpu - self.is_identity = bool( - torch.allclose(weight_cpu, torch.ones_like(weight_cpu)) - and torch.allclose(bias_cpu, torch.zeros_like(bias_cpu)) - ) - - if not self.is_identity: - image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * ( - 1.0 / rescale_factor - ) - image_std_tensor = torch.tensor(image_std, dtype=dtype) * ( - 1.0 / rescale_factor - ) - weight = 1.0 / image_std_tensor - bias = -image_mean_tensor / image_std_tensor - self.register_buffer("weight", weight) - self.register_buffer("bias", bias) - else: - self.register_buffer("weight", None) - self.register_buffer("bias", None) - - @property - def dtype(self) -> torch.dtype: - return self.weight.dtype - - @classmethod - def identity( - cls, channel: int = 3, dtype: torch.dtype = torch.float32 - ) -> "FusedInputNorm": - return cls( - image_mean=[0.0, 0.0, 0.0], - image_std=[1.0, 1.0, 1.0], - rescale_factor=1.0, - channel=channel, - dtype=dtype, - ) - - @classmethod - def from_model_config(cls, model_config: "ModelConfig") -> nn.Module: - if not model_config.multimodal_config.mm_device_do_normalize: - return cls.identity() - - model = model_config.model - revision = model_config.revision - - # Try to read parameters from the processor config - config = get_processor_config(model, revision=revision) - do_rescale = config.get("do_rescale", None) - do_normalize = config.get("do_normalize", None) - image_mean = config.get("image_mean", None) - image_std = config.get("image_std", None) - rescale_factor = config.get("rescale_factor", None) - - # Fallback to the image_processor object if any parameter is missing - if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]: - image_processor = get_processor(model, revision=revision).image_processor - - if do_rescale is None: - do_rescale = getattr(image_processor, "do_rescale", None) - if do_normalize is None: - do_normalize = getattr(image_processor, "do_normalize", None) - if image_mean is None: - image_mean = getattr(image_processor, "image_mean", None) - if image_std is None: - image_std = getattr(image_processor, "image_std", None) - if rescale_factor is None: - rescale_factor = getattr(image_processor, "rescale_factor", None) - - # Apply defaults based on flags - if not do_rescale: - rescale_factor = 1.0 - if not do_normalize: - image_mean = [0.0, 0.0, 0.0] - image_std = [1.0, 1.0, 1.0] - - # Ensure all required parameters are resolved - assert None not in [ - do_rescale, - do_normalize, - image_mean, - image_std, - rescale_factor, - ], "Some normalization parameters are still None after resolution." - - # If no processing is needed, return an identity module - if not do_rescale and not do_normalize: - return cls.identity() - - return cls( - image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor - ) - - def forward( - self, - grid_thw: torch.Tensor, - visual_dtype: torch.dtype, - ) -> torch.Tensor: - if self.is_identity: - return grid_thw.to(visual_dtype) - - assert grid_thw.ndim == 2 - patches, size = grid_thw.shape - patch_size = size // self.channel - - # On XPU, fuse the whole rescale + normalize into a single custom - # kernel. The eager path below materializes an fp32 intermediate and - # then casts back, which adds device-side compute that cancels the - # bandwidth saving of transferring uint8 pixel_values. The fused - # kernel reads uint8 directly and writes ``visual_dtype`` in one pass. - if ( - current_platform.is_xpu() - and grid_thw.dtype == torch.uint8 - and self.weight.dtype == torch.float32 - ): - return torch.ops.vllm.xpu_fused_input_norm( - grid_thw, self.weight, self.bias, visual_dtype - ) - - # Apply the per-channel affine transform directly instead of via - # F.batch_norm. batch_norm dispatches to cuDNN, whose batch-norm - # kernels cap the batch dimension near the CUDA grid limit (~65535); - # here that dimension is the number of patches, which grows unbounded - # with image resolution and batch size and overflows the cap on large - # image-heavy requests (CUDNN_STATUS_INTERNAL_ERROR). The plain - # broadcasted multiply-add is numerically identical and has no such - # limit. - x = grid_thw.to(self.dtype).view(patches, self.channel, patch_size) - x = x * self.weight.view(1, self.channel, 1) + self.bias.view( - 1, self.channel, 1 - ) - return x.view(patches, size).to(visual_dtype) diff --git a/vllm/model_executor/offloader/base.py b/vllm/model_executor/offloader/base.py index 2780ecdd08..3206df34e8 100644 --- a/vllm/model_executor/offloader/base.py +++ b/vllm/model_executor/offloader/base.py @@ -129,14 +129,14 @@ def get_offloader() -> BaseOffloader: return _instance -def set_offloader(instance: BaseOffloader) -> None: - """Set the global offloader instance.""" +def set_offloader(instance: BaseOffloader | None) -> None: + """Set or reset the global offloader instance.""" global _instance - _instance = instance - if isinstance(instance, NoopOffloader): + _instance = NoopOffloader() if instance is None else instance + if isinstance(_instance, NoopOffloader): logger.debug_once("Offloader set to NoopOffloader (no offloading).") else: - logger.info_once("Offloader set to %s", type(instance).__name__) + logger.info_once("Offloader set to %s", type(_instance).__name__) def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader: diff --git a/vllm/model_executor/warmup/jit_warmup_triton_helper.py b/vllm/model_executor/warmup/jit_warmup_triton_helper.py index 198e6fdbe2..f7765cf232 100644 --- a/vllm/model_executor/warmup/jit_warmup_triton_helper.py +++ b/vllm/model_executor/warmup/jit_warmup_triton_helper.py @@ -4,11 +4,14 @@ import ast import inspect from abc import abstractmethod from collections.abc import Callable, Hashable, Iterable, Mapping +from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext, suppress +from contextvars import copy_context from dataclasses import dataclass, field from functools import cache, cached_property, update_wrapper from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload +from vllm import envs from vllm.model_executor.warmup.jit_warmup import ( VllmJitKernel, get_ast_full_name, @@ -227,6 +230,33 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): self._warming = False self._warming_compile_key = None + def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None: + """Compile CUDA startup warmup variants in parallel, then wait.""" + keys = list(compile_keys) + num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys)) + async_compile = getattr(triton, "AsyncCompileMode", None) + # AMD LLVM code generation can abort under concurrent compilation. + if ( + not current_platform.is_cuda() + or self._run_autotune + or async_compile is None + or num_threads <= 1 + ): + return super().compile_many(keys) + + def compile_all() -> None: + with ( + ThreadPoolExecutor(max_workers=num_threads) as executor, + async_compile(executor), + ): + # Dispatch stays on this thread: compile() mutates owner state. + # Only Triton's underlying compiler work runs in the pool. + super(VllmTritonJitKernel, self).compile_many(keys) + + # Older Triton versions leave async mode active on compilation failure. + # Isolate that state so it cannot affect later runtime JIT compilation. + copy_context().run(compile_all) + @cached_property def _kernel_arg_names(self) -> tuple[str, ...]: arg_names = getattr(self.kernel, "arg_names", None) diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index 0ac154505f..9456239809 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -7,9 +7,9 @@ MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA attend. The same model code serves any DSA checkpoint, including GLM-5.2 (``glm_moe_dsa``), which reuses this architecture. -The CUDA implementation selects capability-specific kernels internally and -falls back when an optimization is unavailable. Other platforms use the -generic implementation by default. +The CUDA and ROCm implementations select capability-specific kernels +internally and fall back when an optimization is unavailable. Other platforms +use the generic implementation by default. """ from vllm.platforms import current_platform @@ -20,8 +20,12 @@ if current_platform.is_cuda(): from .nvidia.model import DeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP +elif current_platform.is_rocm(): + from .amd.model import DeepseekV32ForCausalLM + from .amd.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM + from .amd.mtp import DeepseekV32MTP else: - # ROCm, XPU, and CPU keep the generic implementation. + # XPU and CPU keep the generic implementation. from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP from vllm.model_executor.models.deepseek_v2 import ( DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, diff --git a/vllm/models/deepseek_v32/amd/rocm.py b/vllm/models/deepseek_v32/amd/rocm.py index 6d8e7ea855..b9ac31f8be 100644 --- a/vllm/models/deepseek_v32/amd/rocm.py +++ b/vllm/models/deepseek_v32/amd/rocm.py @@ -3,12 +3,14 @@ import torch +from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.forward_context import get_forward_context from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q +from vllm.platforms import current_platform from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( @@ -16,21 +18,15 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( ) -class DeepseekV32MLASparseBackend(ROCMAiterMLASparseBackend): - @staticmethod - def get_supported_kernel_block_sizes() -> list: - return [16, 32] - - -class DeepseekV32ROCmIndexerBackend(DeepseekV32IndexerBackend): - @staticmethod - def get_supported_kernel_block_sizes() -> list: - return [16, 32] - - class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache): def get_attn_backend(self): - return DeepseekV32ROCmIndexerBackend + return DeepseekV32IndexerBackend + + @property + def uses_shuffled_layout(self) -> bool: + # aiter's gather/insert pair shuffles the cache above block size 1: + # [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim]. + return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1 class DeepseekV32ROCmIndexer(DeepseekV32Indexer): @@ -46,7 +42,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): config, prefix, topk_indices_buffer, - attn_backend=DeepseekV32MLASparseBackend, + attn_backend=ROCMAiterMLASparseBackend, ) self.indexer_op: SparseAttnIndexer | None = None @@ -100,14 +96,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): q_nope = q_nope.transpose(0, 1) # (N, tokens, P) if self.is_aiter_triton_fp4_bmm_enabled: - from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 - - ql_nope = batched_gemm_a16wfp4( + ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4( q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True ) elif self.is_aiter_triton_fp8_bmm_enabled: - from vllm._aiter_ops import rocm_aiter_ops - ql_nope = rocm_aiter_ops.triton_fp8_bmm( q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True ) @@ -152,14 +144,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): ) if self.is_aiter_triton_fp4_bmm_enabled: - from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 - - batched_gemm_a16wfp4( + rocm_aiter_ops.batched_gemm_a16wfp4( x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True ) elif self.is_aiter_triton_fp8_bmm_enabled: - from vllm._aiter_ops import rocm_aiter_ops - rocm_aiter_ops.triton_fp8_bmm( x, self.W_V, @@ -202,6 +190,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): indexer_k_norm_eps = self.indexer.k_norm.eps indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache indexer_k_cache = self.indexer.k_cache.kv_cache + indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout indexer_softmax_scale = self.indexer.softmax_scale indexer_n_head_scale = self.indexer.n_head**-0.5 else: @@ -211,6 +200,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): indexer_k_norm_eps = 1e-6 indexer_k_rope_cos_sin_cache = None indexer_k_cache = None + indexer_cache_shuffled = False indexer_softmax_scale = 0.0 indexer_n_head_scale = 0.0 @@ -241,6 +231,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): self.topk_indices_buffer, slot_mapping=mla_slot, indexer_k_cache=indexer_k_cache, + indexer_cache_shuffled=indexer_cache_shuffled, mla_kv_cache=mla_kv_cache, mla_kv_cache_dtype=self.kv_cache_dtype, mla_k_scale=mla_k_scale, @@ -282,7 +273,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined] kv_cache = self.kv_cache if self._fp8_kv_needs_view: - kv_cache = kv_cache.view(torch.float8_e4m3fn) + kv_cache = kv_cache.view(current_platform.fp8_dtype()) q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual) attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined] diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index b8e70e4ad3..f7d0e76e54 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -327,6 +327,7 @@ class DeepseekV32Attention(MLAAttention): indexer_k_norm_bias = self.indexer.k_norm.bias indexer_k_norm_eps = self.indexer.k_norm.eps indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache + indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache index_k_out = torch.empty_like(index_k) if self.use_pcp else None indexer_softmax_scale = self.indexer.softmax_scale @@ -338,6 +339,7 @@ class DeepseekV32Attention(MLAAttention): indexer_k_norm_eps = 1e-6 indexer_k_rope_cos_sin_cache = None indexer_k_cache = None + indexer_cache_shuffled = False index_k_out = None indexer_softmax_scale = 0.0 indexer_n_head_scale = 0.0 @@ -377,6 +379,7 @@ class DeepseekV32Attention(MLAAttention): slot_mapping=mla_slot, indexer_slot_mapping=indexer_slot, indexer_k_cache=indexer_k_cache, + indexer_cache_shuffled=indexer_cache_shuffled, mla_kv_cache=mla_kv_cache, mla_kv_cache_dtype=self.kv_cache_dtype, mla_k_scale=mla_k_scale, @@ -492,6 +495,7 @@ class DeepseekV32Attention(MLAAttention): self._vllm_config.parallel_config.cp_kv_cache_interleave_size ), skip_topk_buffer_clear=True, + topk_backend=self.indexer.indexer_op.topk_backend, ) self.impl.record_logical_topk_ready() # type: ignore[attr-defined] @@ -544,7 +548,7 @@ class DeepseekV32Attention(MLAAttention): return if self._fp8_kv_needs_view: - kv_cache = kv_cache.view(torch.float8_e4m3fn) + kv_cache = kv_cache.view(current_platform.fp8_dtype()) if self._fp8_query: # FlashInfer sparse: single packed fp8 query. mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[ diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index 1808d12f64..e0259c46fa 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -12,6 +12,18 @@ from vllm.utils.torch_utils import is_quantized_kv_cache # has_indexer=False path so the indexer args don't allocate every call. _DUMMY_CACHE: dict[tuple, torch.Tensor] = {} +# Tile shape of the indexer-K cache's shuffled layout, used when the cache reports +# uses_shuffled_layout. The block tile is a token count; the head tile is a byte +# count, which the kernel converts to cache elements. +_INDEXER_CACHE_BLOCK_TILE = 16 +_INDEXER_CACHE_HEAD_TILE_BYTES = 16 + +# FNUZ (e4m3fnuz) on gfx942, OCP (e4m3fn) elsewhere. 224.0 keeps the ue8m0 +# power-of-two scale inside the FNUZ range after its ceil. +_FP8_DTYPE = current_platform.fp8_dtype() +_USE_FNUZ = torch.float8_e4m3fnuz == _FP8_DTYPE +_FP8_MAX = 224.0 if _USE_FNUZ else 448.0 + def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor: key = (shape, dtype, device) @@ -47,16 +59,17 @@ def _get_cos_sin( @triton.jit -def _fp8_ue8m0_quantize(vals): +def _fp8_ue8m0_quantize(vals, FP8_MAX: tl.constexpr, USE_FNUZ: tl.constexpr): """Quantize float32 values to FP8 E4M3 with a ue8m0 (power-of-2) scale. Returns (fp8_vals, scale) so the caller can store them or reuse the scale. """ vals = vals.to(tl.float32) amax = tl.max(tl.abs(vals)) - scale = tl.div_rn(tl.maximum(amax, 1e-4), 448.0) + scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX) scale = tl.math.exp2(tl.math.ceil(tl.math.log2(scale))) - fp8_vals = tl.div_rn(vals, scale).to(tl.float8e4nv) + fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv + fp8_vals = tl.div_rn(vals, scale).to(fp8_dtype) return fp8_vals, scale @@ -71,18 +84,32 @@ def _fp8_quant_and_cache_write( cache_block_stride, offsets, HEAD_DIM: tl.constexpr, + SHUFFLE: tl.constexpr, + BLOCK_TILE: tl.constexpr, + HEAD_TILE: tl.constexpr, + FP8_MAX: tl.constexpr, + USE_FNUZ: tl.constexpr, ): - k_fp8, scale = _fp8_ue8m0_quantize(vals) + k_fp8, scale = _fp8_ue8m0_quantize(vals, FP8_MAX, USE_FNUZ) block_idx = slot_idx // cache_block_size block_offset = slot_idx % cache_block_size block_start = block_idx * cache_block_stride - tl.store( - kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets, - k_fp8, - mask=mask, - ) + # Shuffled layout: [blk/BLOCK_TILE, HEAD_DIM/HEAD_TILE, BLOCK_TILE, HEAD_TILE], + # so one contiguous run holds BLOCK_TILE tokens x HEAD_TILE bytes and the + # reader's coalesced loads land on the bytes we wrote. + if SHUFFLE: + value_off = ( + block_offset // BLOCK_TILE * BLOCK_TILE * HEAD_DIM + + block_offset % BLOCK_TILE * HEAD_TILE + + offsets // HEAD_TILE * BLOCK_TILE * HEAD_TILE + + offsets % HEAD_TILE + ) + else: + value_off = block_offset * HEAD_DIM + offsets + + tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask) scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4 tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale) @@ -137,6 +164,9 @@ def _fused_norm_rope_kernel( indexer_cache_scale_ptr, indexer_cache_block_size, indexer_cache_stride, + INDEXER_CACHE_SHUFFLE: tl.constexpr, + INDEXER_CACHE_BLOCK_TILE: tl.constexpr, + INDEXER_CACHE_HEAD_TILE: tl.constexpr, # MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr) mla_cache_ptr, mla_cache_block_stride, @@ -161,12 +191,15 @@ def _fused_norm_rope_kernel( HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, USE_PDL: tl.constexpr, + FP8_MAX: tl.constexpr, + USE_FNUZ: tl.constexpr, ): tok_idx = tl.program_id(0).to(tl.int64) pid = tl.program_id(1) if USE_PDL: tl.extra.cuda.gdc_wait() tl.extra.cuda.gdc_launch_dependents() + fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv if pid == 3: if not HAS_INDEXER: # Shared layer: reuse the previous indexer layer's top-k; do not @@ -327,9 +360,11 @@ def _fused_norm_rope_kernel( ) kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM)) tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True) - tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1e-4) + # scale = amax / FP8_MAX (fp8 e4m3 max), matching the reference + # concat_and_cache_ds_mla kernel; floored to FLT_MIN. + tile_scale = tl.maximum(tile_amax * (1.0 / FP8_MAX), 1e-4) tile_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(tile_scale))) - kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,)) + kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(fp8_dtype), (KV_DIM,)) tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8) tile_off = tl.arange(0, MLA_NUM_TILES) tl.store( @@ -349,16 +384,16 @@ def _fused_norm_rope_kernel( # kv_c_normed (KV_DIM elements) if MLA_CACHE_FP8: scale = tl.load(mla_cache_scale_ptr) - kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv) + kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(fp8_dtype) tl.store(dst + kv_block, kv_c_fp8) else: tl.store(dst + kv_block, kv_c) # k_pe_roped (from registers, interleaved layout) if MLA_CACHE_FP8: - tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv)) + tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(fp8_dtype)) tl.store( dst + KV_DIM + dim_off * 2 + 1, - (r2 / scale).to(tl.float8e4nv), + (r2 / scale).to(fp8_dtype), ) else: tl.store(dst + KV_DIM + dim_off * 2, r1) @@ -463,6 +498,11 @@ def _fused_norm_rope_kernel( indexer_cache_stride, index_k_block, INDEX_K_DIM, + INDEXER_CACHE_SHUFFLE, + INDEXER_CACHE_BLOCK_TILE, + INDEXER_CACHE_HEAD_TILE, + FP8_MAX, + USE_FNUZ, ) @@ -486,6 +526,7 @@ def fused_norm_rope( slot_mapping: torch.Tensor | None = None, indexer_slot_mapping: torch.Tensor | None = None, indexer_k_cache: torch.Tensor | None = None, + indexer_cache_shuffled: bool = False, mla_kv_cache: torch.Tensor | None = None, mla_kv_cache_dtype: str = "auto", mla_k_scale: torch.Tensor | None = None, @@ -532,13 +573,28 @@ def fused_norm_rope( indexer_slot_mapping = slot_mapping idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32) idx_cache_block_size = indexer_k_cache.shape[1] + idx_cache_stride = indexer_k_cache.shape[2] + # The caller's cache reports whether its reader expects the shuffled + # layout; see DeepseekV32IndexerCache.uses_shuffled_layout. + idx_cache_shuffle = indexer_cache_shuffled + if idx_cache_shuffle: + assert idx_cache_block_size % _INDEXER_CACHE_BLOCK_TILE == 0, ( + f"indexer K cache block size {idx_cache_block_size} must be a " + f"multiple of {_INDEXER_CACHE_BLOCK_TILE} for the shuffled layout" + ) idx_cache_stride = indexer_k_cache.stride(0) if indexer_k_cache.dtype == torch.uint8: - indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn) + indexer_k_cache = indexer_k_cache.view(_FP8_DTYPE) + # The head tile is a byte count; the kernel indexes the cache in elements. + idx_cache_head_tile = ( + _INDEXER_CACHE_HEAD_TILE_BYTES // indexer_k_cache.element_size() + ) else: idx_cache_scale_view = None idx_cache_block_size = 1 idx_cache_stride = 0 + idx_cache_shuffle = False + idx_cache_head_tile = _INDEXER_CACHE_HEAD_TILE_BYTES # --- MLA KV cache setup --- mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla" @@ -561,12 +617,12 @@ def fused_norm_rope( mla_entry_stride = u8_cache.stride(1) mla_ds_scale_view = u8_cache.view(torch.float32) mla_ds_rope_view = u8_cache.view(torch.bfloat16) - mla_kv_cache = u8_cache.view(torch.float8_e4m3fn) + mla_kv_cache = u8_cache.view(_FP8_DTYPE) else: mla_block_stride = mla_kv_cache.stride(0) mla_entry_stride = mla_kv_cache.stride(1) if mla_cache_fp8 and mla_kv_cache.dtype == torch.uint8: - mla_kv_cache = mla_kv_cache.view(torch.float8_e4m3fn) + mla_kv_cache = mla_kv_cache.view(_FP8_DTYPE) if mla_k_scale is None: mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) else: @@ -592,6 +648,9 @@ def fused_norm_rope( assert index_k_out.shape == index_k.shape index_k_out_stride = index_k_out.stride(0) use_pdl = current_platform.is_arch_support_pdl() + pdl_kwargs = ( + {"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl} + ) _fused_norm_rope_kernel[(num_tokens, 4)]( positions, # Q RMS norm @@ -639,6 +698,9 @@ def fused_norm_rope( idx_cache_scale_view, idx_cache_block_size, idx_cache_stride, + idx_cache_shuffle, + _INDEXER_CACHE_BLOCK_TILE, + idx_cache_head_tile, # MLA KV cache (uses same slot_mapping) mla_kv_cache, mla_block_stride, @@ -659,8 +721,9 @@ def fused_norm_rope( TOPK_BLOCK_SIZE=1024, HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, - USE_PDL=use_pdl, - launch_pdl=use_pdl, + FP8_MAX=_FP8_MAX, + USE_FNUZ=_USE_FNUZ, + **pdl_kwargs, ) return q_c_out @@ -715,6 +778,8 @@ def _fused_q_kernel( INDEX_ROPE_INTERLEAVE: tl.constexpr, QUANTIZE_MQA: tl.constexpr, USE_PDL: tl.constexpr, + FP8_MAX: tl.constexpr, + USE_FNUZ: tl.constexpr, ): tok_idx = tl.program_id(0).to(tl.int64) pid = tl.program_id(1) @@ -722,6 +787,7 @@ def _fused_q_kernel( if USE_PDL: tl.extra.cuda.gdc_wait() tl.extra.cuda.gdc_launch_dependents() + fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv if pid == 2: # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 @@ -744,7 +810,7 @@ def _fused_q_kernel( + ql_nope_off, mask=ql_nope_mask, ).to(tl.float32) - ql_nope_fp8 = (ql_nope / scale).to(tl.float8e4nv) + ql_nope_fp8 = (ql_nope / scale).to(fp8_dtype) tl.store( mqa_q_fp8_ptr + tok_idx * mqa_q_fp8_stride0 @@ -794,7 +860,7 @@ def _fused_q_kernel( + q_head_idx * mqa_q_fp8_stride1 + QL_NOPE_DIM + rot_off * 2, - (r1 / scale).to(tl.float8e4nv), + (r1 / scale).to(fp8_dtype), ) tl.store( mqa_q_fp8_ptr @@ -803,7 +869,7 @@ def _fused_q_kernel( + QL_NOPE_DIM + rot_off * 2 + 1, - (r2 / scale).to(tl.float8e4nv), + (r2 / scale).to(fp8_dtype), ) else: # bf16 query: write the RoPE'd q_pe unquantized. @@ -861,7 +927,7 @@ def _fused_q_kernel( index_q = tl.where(in_rope, roped, index_q) # Index Q Quantize (from registers) - index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q) + index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q, FP8_MAX, USE_FNUZ) tl.store( index_q_fp8_ptr + tok_idx * index_q_fp8_stride0 @@ -955,7 +1021,7 @@ def fused_q( q_pe.shape[0], q_pe.shape[1], ql_nope.shape[2] + q_pe.shape[2], - dtype=torch.float8_e4m3fn, + dtype=_FP8_DTYPE, device=q_pe.device, ) # Placeholder; pid 0 packs q_pe into mqa_q_fp8 instead. @@ -967,7 +1033,7 @@ def fused_q( mqa_q_fp8 = q_pe_out # unused placeholder for the fp8 pack pointer mqa_q = q_pe_out - index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn) + index_q_fp8 = torch.empty_like(index_q, dtype=_FP8_DTYPE) index_weights_out = torch.empty_like(index_weights, dtype=torch.float32) if cutedsl_kernel is not None: cutedsl_kernel( @@ -990,6 +1056,9 @@ def fused_q( return index_q_fp8, index_weights_out, mqa_q use_pdl = current_platform.is_arch_support_pdl() + pdl_kwargs = ( + {"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl} + ) _fused_q_kernel[(num_tokens, 3, grid_heads)]( positions, q_pe, @@ -1031,8 +1100,9 @@ def fused_q( HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, QUANTIZE_MQA=quantize_mqa, - USE_PDL=use_pdl, - launch_pdl=use_pdl, + FP8_MAX=_FP8_MAX, + USE_FNUZ=_USE_FNUZ, + **pdl_kwargs, # num_warps=1 is optimal here: each program is a single 128-element # rope+quant, so the kernel is program-count/occupancy bound, not # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index b9478b508f..78dc10ba00 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -37,6 +37,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( build_ragged_indices_from_dense, rocm_inv_rope_einsum, + rocm_inverse_rope_rows_, rocm_sparse_attn_decode, rocm_sparse_attn_prefill, ) @@ -1175,6 +1176,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): self.n_local_groups, self.o_lora_rank, self.wo_a, + inverse_rope=False, ) zf = z.flatten(1) if self._wo_b_scale is not None and zf.dim() == 2: @@ -1245,9 +1247,15 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): attn_metadata=rocm_metadata, swa_metadata=swa_metadata, ) + # The fp8 wo_a path rotates inside inverse_rope_group_quant, so folding + # the rotation into the decode reduce would apply it twice. Only the + # BF16 einsum path hands its rotation off to the decode. + fuse_inv_rope = self._wo_a_fp8_weight is None + rotated = 0 if num_decodes > 0: - self._forward_decode( + rotated = self._forward_decode( q=q[:num_decode_tokens], + positions=positions[:num_decode_tokens] if fuse_inv_rope else None, kv_cache=self_kv_cache, swa_metadata=swa_metadata, attn_metadata=rocm_metadata, @@ -1261,17 +1269,31 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): and rocm_metadata.for_cudagraph_capture ), ) + if fuse_inv_rope: + # Only the decode reduce rotates its own rows, and only the leading + # `rotated` of them; prefill rows and any decode path that did not + # fuse still owe the standalone pass. Settle that here rather than + # in _o_proj: the split is batch-dependent and _o_proj runs + # compiled, where such a value freezes at its trace-time value. + rocm_inverse_rope_rows_( + output[rotated:, : self.n_local_heads, :], + positions[rotated:], + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + ) def _forward_decode( self, q: torch.Tensor, + positions: torch.Tensor | None, kv_cache: torch.Tensor | None, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None, swa_only: bool, output: torch.Tensor, adaptive_splits: bool, - ) -> None: + ) -> int: + """Returns how many leading rows the decode epilogue inverse-RoPE'd.""" num_decodes = swa_metadata.num_decodes num_decode_tokens = swa_metadata.num_decode_tokens @@ -1303,7 +1325,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): topk_ragged_indices = attn_metadata.c128a_decode_topk_ragged_indices topk_ragged_indptr = attn_metadata.c128a_decode_topk_ragged_indptr - rocm_sparse_attn_decode( + return rocm_sparse_attn_decode( q=q, kv_cache=kv_cache, swa_k_cache=self.swa_cache_layer.kv_cache, @@ -1323,6 +1345,8 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): rope_head_dim=self.rope_head_dim, output=output, adaptive_splits=adaptive_splits, + inv_rope_positions=positions, + inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache, extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free( self.kv_cache_dtype, self._has_kv_transfer, diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 8a151dba1d..2315bedfeb 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -43,6 +43,7 @@ from vllm.distributed import get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.models.utils import extract_layer_index @@ -171,14 +172,20 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, - output: torch.Tensor, + output: "torch.Tensor | QuantizedActivation", ) -> None: """Platform-specific sparse MLA forward; writes attention into ``output``.""" raise NotImplementedError @abstractmethod - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - """Inverse-RoPE + wo_a + wo_b output projection (platform-specific).""" + def _o_proj( + self, o: "torch.Tensor | QuantizedActivation", positions: torch.Tensor + ) -> torch.Tensor: + """Inverse-RoPE + wo_a + wo_b output projection (platform-specific). + + ``o`` is the live heads of the bf16 attention output, or the + QuantizedActivation of a layer whose ``_alloc_attn_out`` returns one. + """ raise NotImplementedError def _uses_fp8_ds_mla_layout(self) -> bool: @@ -467,14 +474,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): hidden_states: torch.Tensor, llama_4_scaling: torch.Tensor | None = None, ) -> torch.Tensor: - # Pre-allocate attention output with FlashMLA-padded head count. - # The op writes into `o_padded`; we slice to n_local_heads after. - num_tokens = hidden_states.shape[0] - o_padded = torch.empty( - (num_tokens, self.padded_heads, self.head_dim), - dtype=hidden_states.dtype, - device=hidden_states.device, - ) + # The eager attention region writes into a caller-owned buffer + # (breakable_cudagraph needs in-place outputs). + attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states) # Keep the attention input preparation in the captured graph. Only the # sparse indexer and MLA attention run in the eager break below. @@ -492,12 +494,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): indexer_kv_score, indexer_weights, positions, - o_padded, + attn_out, ) - o = o_padded[:, : self.n_local_heads, :] + if isinstance(attn_out, torch.Tensor): + attn_out = attn_out[:, : self.n_local_heads, :] + return self._o_proj(attn_out, positions) - # Inverse-RoPE + wo_a + wo_b output projection (platform-specific). - return self._o_proj(o, positions) + def _alloc_attn_out( + self, num_tokens: int, hidden_states: torch.Tensor + ) -> "torch.Tensor | QuantizedActivation": + """The buffer ``forward_mqa`` fills. + + A bf16 ``[num_tokens, padded_heads, head_dim]`` buffer by default, whose + padding heads are sliced off before ``_o_proj``. A layer whose kernel + also does the inverse RoPE and the FP8 cast returns a + QuantizedActivation instead and gets it back in ``_o_proj`` whole. + """ + return torch.empty( + (num_tokens, self.padded_heads, self.head_dim), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) def _split_qkv_and_norm( self, qr_kv: torch.Tensor @@ -529,7 +546,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): indexer_kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, - o_padded: torch.Tensor, + o_padded: "torch.Tensor | QuantizedActivation", ) -> None: """Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly. @@ -558,7 +575,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): indexer_kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, - o_padded: torch.Tensor, + o_padded: "torch.Tensor | QuantizedActivation", ) -> None: """Attention input preparation followed by the sparse indexer and MLA. @@ -723,7 +740,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, - out: torch.Tensor, + out: "torch.Tensor | QuantizedActivation", ) -> None: if self.indexer is not None and index_q is not None: assert index_weights is not None @@ -736,7 +753,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): ) # MLA attention writes into the pre-allocated `out` buffer - # ([num_tokens, padded_heads, head_dim]). + # (see _alloc_attn_out). self.forward_mqa(q, kv, positions, out) def _fused_qnorm_rope_kv_insert( diff --git a/vllm/models/deepseek_v4/common/ops/cache_utils.py b/vllm/models/deepseek_v4/common/ops/cache_utils.py index 12fdb04711..cea755bd1a 100644 --- a/vllm/models/deepseek_v4/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v4/common/ops/cache_utils.py @@ -1099,6 +1099,7 @@ def build_flashinfer_mixed_sparse_indices( prefill_left_visible: torch.Tensor | None = None, prefill_right_visible: torch.Tensor | None = None, max_image_tokens: int = 0, + num_rows: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Build the FlashInfer DSV4 sparse-index matrix for decode-first batches. @@ -1109,6 +1110,9 @@ def build_flashinfer_mixed_sparse_indices( tokens derive their SWA window from the position and translate local compressed indices to global slots via the block tables. + ``num_rows`` (>= ``num_tokens``) sizes both outputs for a kernel call that + spans a CUDA-graph-padded batch; the rows past ``num_tokens`` stay unset. + When ``prefill_left_visible``/``prefill_right_visible`` are given (vision variant), the SWA column region widens by ``max_image_tokens`` and prefill tokens inside an image span get a bidirectionally widened window; decode @@ -1170,13 +1174,15 @@ def build_flashinfer_mixed_sparse_indices( # by ``sparse_topk_lens``, so padding never changes the attention result. padded_topk = max(topk, decode_compressed_topk) padded_topk = (padded_topk + 3) // 4 * 4 + num_rows = num_tokens if num_rows is None else num_rows + assert num_rows >= num_tokens sparse_indices = torch.empty( - (num_tokens, swa_total_width + padded_topk), + (num_rows, swa_total_width + padded_topk), dtype=torch.int32, device=decode_swa_indices.device, ) sparse_topk_lens = torch.empty( - num_tokens, dtype=torch.int32, device=decode_swa_indices.device + num_rows, dtype=torch.int32, device=decode_swa_indices.device ) if num_tokens == 0: return sparse_indices, sparse_topk_lens @@ -1193,9 +1199,10 @@ def build_flashinfer_mixed_sparse_indices( if compressed_block_span is None else compressed_block_span ) + # The launch grid follows the output rows, so hand over only the real ones. _BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL( - sparse_indices, - sparse_topk_lens, + sparse_indices[:num_tokens], + sparse_topk_lens[:num_tokens], decode_swa_indices, decode_compressed_indices, decode_compressed_topk_lens, @@ -1234,7 +1241,7 @@ def build_flashinfer_mixed_sparse_indices( def _remap_flashinfer_index(values, block_size, block_span): # FlashInfer's DSv4 kernel indexes sparse KV by physical token stride, so # packed pages (#44577) need block*block_size+off -> block*block_span+off. - # TODO: remove once flashinfer-ai/flashinfer#3856 is fixed. + # The launcher declares these indices with sparse_indices_are_storage_offsets=True. is_valid = values >= 0 safe_values = tl.where(is_valid, values, 0) values = (safe_values // block_size) * block_span @@ -1322,12 +1329,52 @@ class BuildFlashinferMixedSparseIndicesKernel( SWA_TOTAL_WIDTH: tl.constexpr = SWA_INDEX_WIDTH + IMAGE_WIDTH if token_idx < NUM_DECODE_TOKENS: + # A decode row wider than the window is DSpark's non-causal block: + # the block-anchored window of context followed by the whole block, + # contiguous from column 0. The kernel treats every entry of its + # active ranges as a key -- the first min(WINDOW_SIZE, pos + 1) + # columns and [WINDOW_SIZE, sparse_topk_lens) -- so -1 must not + # appear in either. Fill the causal window with the first visible + # entries and put the rest from column WINDOW_SIZE on, followed + # directly by the compressed entries. + NONCAUSAL: tl.constexpr = SWA_INDEX_WIDTH > WINDOW_SIZE + compressed_start = SWA_TOTAL_WIDTH + if NONCAUSAL: + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + query_start = tl.load(query_start_loc_ptr + req_idx) + query_len = tl.load(query_start_loc_ptr + req_idx + 1) - query_start + context_len = tl.load(seq_lens_ptr + req_idx) - query_len + token_in_query = token_idx - query_start + window_len = tl.minimum(context_len + token_in_query + 1, WINDOW_SIZE) + spill_len = ( + tl.minimum(context_len, WINDOW_SIZE) + query_len - window_len + ) + # A CUDA-graph pad token sits outside its empty request, so + # the kernel never reads its row; keep it the shape of a + # causal pad row (all -1, WINDOW_SIZE long) rather than let + # the formulas above go negative. + is_block_token = (token_in_query >= 0) & (token_in_query < query_len) + window_len = tl.where(is_block_token, window_len, 0) + spill_len = tl.where(is_block_token, spill_len, 0) + compressed_start = WINDOW_SIZE + spill_len for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE): offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) mask = offset < SWA_TOTAL_WIDTH + src = offset + src_mask = offset < SWA_INDEX_WIDTH + if NONCAUSAL: + # Columns from compressed_start on belong to the loops below. + mask = mask & (offset < compressed_start) + spill = offset >= WINDOW_SIZE + src = tl.where(spill, offset - WINDOW_SIZE + window_len, offset) + src_mask = tl.where( + spill, + offset < WINDOW_SIZE + spill_len, + offset < window_len, + ) values = tl.load( - decode_swa_indices_ptr + token_idx * decode_swa_stride + offset, - mask=offset < SWA_INDEX_WIDTH, + decode_swa_indices_ptr + token_idx * decode_swa_stride + src, + mask=src_mask, other=-1, ) values = _remap_flashinfer_index(values, swa_block_size, swa_block_span) @@ -1372,12 +1419,27 @@ class BuildFlashinferMixedSparseIndicesKernel( tl.store( sparse_indices_ptr + token_idx * sparse_indices_stride - + SWA_TOTAL_WIDTH + + compressed_start + offset, values, mask=mask, ) + if NONCAUSAL: + # The row's remaining columns, left over by the shifted + # compressed block. + for i in range(0, SWA_TOTAL_WIDTH - WINDOW_SIZE, WINDOW_BLOCK_SIZE): + offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) + tl.store( + sparse_indices_ptr + + token_idx * sparse_indices_stride + + compressed_start + + PADDED_TOP_K + + offset, + tl.full((WINDOW_BLOCK_SIZE,), -1, dtype=tl.int32), + mask=offset < SWA_TOTAL_WIDTH - compressed_start, + ) + if DECODE_COMPRESSED_TOPK == 0: compressed_len = tl.zeros((), dtype=tl.int32) elif not DECODE_COMPRESSED_INDICES_ARE_LOCAL: @@ -1388,7 +1450,9 @@ class BuildFlashinferMixedSparseIndicesKernel( else: compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32) - tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + compressed_len) + tl.store( + sparse_topk_lens_ptr + token_idx, compressed_start + compressed_len + ) return prefill_idx = token_idx - NUM_DECODE_TOKENS diff --git a/vllm/models/deepseek_v4/common/vision.py b/vllm/models/deepseek_v4/common/vision.py index bfc02768c0..cbcf34ffc7 100644 --- a/vllm/models/deepseek_v4/common/vision.py +++ b/vllm/models/deepseek_v4/common/vision.py @@ -38,6 +38,7 @@ from vllm.model_executor.models.vision import ( get_load_balance_assignment, is_vit_use_data_parallel, ) +from vllm.utils.torch_utils import async_tensor_h2d def _compute_vision_cos_sin( @@ -219,8 +220,8 @@ class DeepseekV4ViT(nn.Module): ) -> torch.Tensor: x = self.patch_embed(patches) cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta) - cos = cos.to(device=x.device) - sin = sin.to(device=x.device) + cos = async_tensor_h2d(cos, x.device) + sin = async_tensor_h2d(sin, x.device) for block in self.blocks: x = block(x, cos, sin) return self.norm(x) @@ -279,8 +280,8 @@ def build_packed_vit_metadata( cu_seqlens.append(cu_seqlens[-1] + n) max_seqlen = max(max_seqlen, n) if cos_list: - cos = torch.cat(cos_list).to(device) - sin = torch.cat(sin_list).to(device) + cos = async_tensor_h2d(torch.cat(cos_list), device) + sin = async_tensor_h2d(torch.cat(sin_list), device) else: cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device) sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device) @@ -289,7 +290,7 @@ def build_packed_vit_metadata( return { "vit_cos": cos, "vit_sin": sin, - "cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device), + "cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32), # Read on the host by the attention wrapper; keep on CPU. "max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32), } @@ -327,8 +328,10 @@ def build_packed_merge_metadata( mask_list.append(valid) offset += n_h * n_w if idx_list: - merge_idx = torch.cat(idx_list).to(device) - merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype) + merge_idx = async_tensor_h2d(torch.cat(idx_list), device) + merge_mask = async_tensor_h2d( + torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype + ) else: merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device) merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device) diff --git a/vllm/models/deepseek_v4/compressor.py b/vllm/models/deepseek_v4/compressor.py index 41af0c7660..e6e0fd959e 100644 --- a/vllm/models/deepseek_v4/compressor.py +++ b/vllm/models/deepseek_v4/compressor.py @@ -66,7 +66,7 @@ class CompressorBackend(AttentionBackend): return "CompressorBackend" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(1)] @classmethod diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index 883ef5ecaf..3e83c280f3 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -9,6 +9,8 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context +from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.common.ops import ( build_flashinfer_mixed_sparse_indices, @@ -17,6 +19,8 @@ from vllm.models.deepseek_v4.common.ops import ( from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( compute_fp8_einsum_recipe, deep_gemm_fp8_o_proj, + rope_quant_attn_out, + rope_quant_unsupported_reason, ) from vllm.models.deepseek_v4.sparse_mla import ( DeepseekV4FlashMLAMetadata, @@ -37,6 +41,8 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata +logger = init_logger(__name__) + _FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024 _flashinfer_dsv4_workspace_by_device: dict[torch.device, torch.Tensor] = {} @@ -111,7 +117,7 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [256] @staticmethod @@ -203,7 +209,9 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): def get_padded_num_q_heads(cls, num_heads: int) -> int: return _pad_to_supported_q_heads(num_heads) - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _o_proj( + self, o: torch.Tensor | QuantizedActivation, positions: torch.Tensor + ) -> torch.Tensor: return deep_gemm_fp8_o_proj( o, positions, @@ -219,9 +227,30 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): tma_aligned_scales=self._tma_aligned_scales, ) - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) + def _alloc_attn_out( + self, num_tokens: int, hidden_states: torch.Tensor + ) -> torch.Tensor | QuantizedActivation: + if self._fuse_rope_quant: + return rope_quant_attn_out(num_tokens, hidden_states.device) + return super()._alloc_attn_out(num_tokens, hidden_states) + + def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: + super().__init__(vllm_config, *args, **kwargs) self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe() + # RopeQuant derives each query's RoPE position as seq_len - q_len + i, + # which matches `positions` for DSpark's non-causal draft block too, + # since every call passes the real per-request seq_lens. The one + # exception is a draft block running past max_model_len, whose clamped + # positions it does not reproduce; that only costs those drafts. + reason = rope_quant_unsupported_reason(self) + self._fuse_rope_quant = reason is None + if reason is None: + logger.info_once( + "FLASHINFER_MLA_SPARSE_DSV4 fuses the inverse RoPE and FP8 " + "quant of the attention output." + ) + else: + logger.debug_once("FLASHINFER_MLA_SPARSE_DSV4 RopeQuant off: %s", reason) # Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the # per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``. if self.kv_cache_torch_dtype != torch.float8_e4m3fn: @@ -253,32 +282,39 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, - output: torch.Tensor, + output: torch.Tensor | QuantizedActivation, ) -> None: - # The TRTLLM-gen kernel requires h_q in {64, 128}, so the output buffer - # is allocated at the padded head count while q arrives at the local - # head count; _forward pads q to match before the launcher. - assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], ( - f"output buffer shape {output.shape} incompatible with q shape {q.shape}" - ) - assert output.shape[1] >= q.shape[1], ( - f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}" - ) - # Per-tensor FP8 q produces a bf16 attention output. - expected_output_dtype = ( - torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype - ) - assert output.dtype == expected_output_dtype, ( - f"output dtype {output.dtype} must match expected {expected_output_dtype} " - f"for q dtype {q.dtype}" - ) + if isinstance(output, QuantizedActivation): + assert output.data.shape[0] == q.shape[0] + else: + # The TRTLLM-gen kernel requires h_q in {64, 128}, so the output + # buffer is allocated at the padded head count while q arrives at + # the local head count; _forward pads q to match before the launcher. + assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], ( + f"output buffer shape {output.shape} incompatible with q {q.shape}" + ) + assert output.shape[1] >= q.shape[1], ( + f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}" + ) + # Per-tensor FP8 q produces a bf16 attention output. + expected_output_dtype = ( + torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype + ) + assert output.dtype == expected_output_dtype, ( + f"output dtype {output.dtype} must match expected " + f"{expected_output_dtype} for q dtype {q.dtype}" + ) forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata if attn_metadata is None: # Warmup dummy run: FlashInfer reads the cache directly and lazily # allocates its workspace, so nothing to reserve here. - output.zero_() + if isinstance(output, QuantizedActivation): + output.data.zero_() + output.scale.zero_() + else: + output.zero_() return assert isinstance(attn_metadata, dict) @@ -313,11 +349,12 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): swa_metadata: "DeepseekSparseSWAMetadata", attn_metadata: DeepseekV4FlashMLAMetadata | None, swa_only: bool, + num_rows: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """Build the combined sparse-index tensors for the mixed batch. Returns ``(compressed_kv_cache, seq_lens, sparse_indices, - sparse_topk_lens)``. + sparse_topk_lens)``, sized ``num_rows`` (default: the real tokens). """ num_decodes = swa_metadata.num_decodes num_prefills = swa_metadata.num_prefills @@ -412,6 +449,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): if swa_only else ("c128a" if self.compress_ratio == 128 else "c4a") ) + if num_rows is not None: + cache_key = f"{cache_key}:{num_rows}" cached_sparse = swa_metadata.flashinfer_sparse_index_cache.get(cache_key, None) if cached_sparse is None: swa_block_span = _packed_block_span(swa_k_cache) @@ -439,8 +478,9 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): prefill_right_visible=swa_metadata.prefill_right_visible, # getattr for tests that bypass __init__ via object.__new__. max_image_tokens=getattr(self, "max_image_tokens", 0), + num_rows=num_rows, ) - if cache_key != "c4a": + if not cache_key.startswith("c4a"): swa_metadata.flashinfer_sparse_index_cache[cache_key] = ( sparse_indices, sparse_topk_lens, @@ -457,7 +497,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): swa_metadata: "DeepseekSparseSWAMetadata", attn_metadata: DeepseekV4FlashMLAMetadata | None, swa_only: bool, - output: torch.Tensor, + output: torch.Tensor | QuantizedActivation, ) -> None: assert self.kv_cache_torch_dtype in (torch.bfloat16, torch.float8_e4m3fn) num_decodes = swa_metadata.num_decodes @@ -468,6 +508,11 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): num_tokens = num_decode_tokens + num_prefill_tokens if num_tokens == 0: return + # RopeQuant's output is group-major with a group stride of exactly + # sum_q tokens, so both calls span every row of the (CUDA-graph + # padded) buffers and select their tokens through cum_seq_lens_q + # alone; the kernel never touches rows outside it. + rope_quant = isinstance(output, QuantizedActivation) ( compressed_kv_cache, @@ -480,12 +525,12 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): swa_metadata=swa_metadata, attn_metadata=attn_metadata, swa_only=swa_only, + num_rows=q.shape[0] if rope_quant else None, ) # CUDA graph execution can pad q/output past the scheduled token count; # restrict to the real tokens (the launcher validates sparse indices). - query = q[:num_tokens] - output = output[:num_tokens] + query = q if rope_quant else q[:num_tokens] bmm1_scale: float | torch.Tensor = self.scale bmm2_scale: float | torch.Tensor = 1.0 if self.kv_cache_torch_dtype == torch.float8_e4m3fn: @@ -499,7 +544,9 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): # The TRTLLM-gen sparse-MLA kernel requires h_q in {64, 128}; zero-pad # the query heads to the allocated output head count. Padded heads attend # to the shared KV and are sliced off downstream (output is padded too). - padded_heads = output.shape[1] + padded_heads = ( + output.shape[1] if isinstance(output, torch.Tensor) else self.padded_heads + ) if query.shape[1] < padded_heads: padded_query = query.new_zeros( (query.shape[0], padded_heads, query.shape[2]) @@ -512,49 +559,60 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): query_start_loc_cpu = swa_metadata.query_start_loc_cpu assert query_start_loc is not None and query_start_loc_cpu is not None + def run( + rows: slice, reqs: slice, cum_seq_lens_q: torch.Tensor, max_q_len: int + ) -> None: + if rope_quant: + assert isinstance(output, QuantizedActivation) + rows = slice(None) + out_kwargs = dict( + out=output.data, + dsv4_output_scale=output.scale, + dsv4_inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache, + ) + else: + assert isinstance(output, torch.Tensor) + out_kwargs = dict(out=output[rows]) + flashinfer_trtllm_batch_decode_sparse_mla_dsv4( + query=query[rows], + swa_kv_cache=swa_k_cache, + workspace_buffer=workspace, + sparse_indices=sparse_indices[rows], + sparse_indices_are_storage_offsets=True, + compressed_kv_cache=compressed_kv_cache, + sparse_topk_lens=sparse_topk_lens[rows], + seq_lens=seq_lens[reqs], + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + sinks=self.attn_sink, + cum_seq_lens_q=cum_seq_lens_q, + max_q_len=max_q_len, + **out_kwargs, + ) + # Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for # uniform-q batches, and this avoids flattening mixed batches into one call. if num_decode_tokens > 0: - decode_cu = query_start_loc[: num_decodes + 1] - flashinfer_trtllm_batch_decode_sparse_mla_dsv4( - query=query[:num_decode_tokens], - swa_kv_cache=swa_k_cache, - workspace_buffer=workspace, - sparse_indices=sparse_indices[:num_decode_tokens], - compressed_kv_cache=compressed_kv_cache, - sparse_topk_lens=sparse_topk_lens[:num_decode_tokens], - seq_lens=seq_lens[:num_decodes], - out=output[:num_decode_tokens], - bmm1_scale=bmm1_scale, - bmm2_scale=bmm2_scale, - sinks=self.attn_sink, - cum_seq_lens_q=decode_cu, - max_q_len=swa_metadata.max_decode_query_len, + run( + slice(0, num_decode_tokens), + slice(0, num_decodes), + query_start_loc[: num_decodes + 1], + swa_metadata.max_decode_query_len, ) if num_prefill_tokens > 0: - # The prefill query view re-anchors at offset 0, so rebase the - # cumulative query offsets to start at 0. - prefill_cu = ( - query_start_loc[num_decodes : num_reqs + 1] - - query_start_loc[num_decodes] - ) + prefill_cu = query_start_loc[num_decodes : num_reqs + 1] + if not rope_quant: + # The prefill query view re-anchors at offset 0, so rebase the + # cumulative query offsets to start at 0. + prefill_cu = prefill_cu - query_start_loc[num_decodes] prefill_cu_cpu = query_start_loc_cpu[num_decodes : num_reqs + 1] prefill_lens_cpu = prefill_cu_cpu[1:] - prefill_cu_cpu[:-1] - flashinfer_trtllm_batch_decode_sparse_mla_dsv4( - query=query[num_decode_tokens:num_tokens], - swa_kv_cache=swa_k_cache, - workspace_buffer=workspace, - sparse_indices=sparse_indices[num_decode_tokens:num_tokens], - compressed_kv_cache=compressed_kv_cache, - sparse_topk_lens=sparse_topk_lens[num_decode_tokens:num_tokens], - seq_lens=seq_lens[num_decodes:num_reqs], - out=output[num_decode_tokens:num_tokens], - bmm1_scale=bmm1_scale, - bmm2_scale=bmm2_scale, - sinks=self.attn_sink, - cum_seq_lens_q=prefill_cu, - max_q_len=int(prefill_lens_cpu.max().item()), + run( + slice(num_decode_tokens, num_tokens), + slice(num_decodes, num_reqs), + prefill_cu, + int(prefill_lens_cpu.max().item()), ) diff --git a/vllm/models/deepseek_v4/nvidia/ops/o_proj.py b/vllm/models/deepseek_v4/nvidia/ops/o_proj.py index ec36bf763e..1da4ab0965 100644 --- a/vllm/models/deepseek_v4/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v4/nvidia/ops/o_proj.py @@ -5,11 +5,15 @@ from collections.abc import Callable import torch import torch.nn as nn +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Dynamic128Sym, +) from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import ( fused_inv_rope_fp8_quant, ) from vllm.platforms import current_platform -from vllm.utils.deep_gemm import fp8_einsum +from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size def compute_fp8_einsum_recipe( @@ -29,7 +33,7 @@ def compute_fp8_einsum_recipe( def deep_gemm_fp8_o_proj( - o: torch.Tensor, + o: torch.Tensor | QuantizedActivation, positions: torch.Tensor, cos_sin_cache: torch.Tensor, wo_a: nn.Module, @@ -49,23 +53,31 @@ def deep_gemm_fp8_o_proj( layer selects the recipe at initialization. ``wo_b`` is any callable over the flattened ``z``: the projection module itself, or a wrapper that also reduce-scatters its output (DeepSeek-V4.1 GEMM-RS). + + A QuantizedActivation ``o`` was already rotated and cast by the attention + kernel, one ``wo_a`` group per slot with the live groups first, so only + the padding groups are sliced off. """ use_fp8 = wo_a.weight.dtype == torch.float8_e4m3fn - o_proj_input, o_scale = fused_inv_rope_fp8_quant( - o, - positions, - cos_sin_cache, - n_groups=n_groups, - heads_per_group=heads_per_group, - nope_dim=nope_dim, - rope_dim=rope_dim, - quant_group_size=einsum_recipe[2], - tma_aligned_scales=tma_aligned_scales, - quantize=use_fp8, - ) + if isinstance(o, QuantizedActivation): + assert use_fp8 + o_proj_input, o_scale = o.data[:, :n_groups], o.scale[:, :n_groups] + else: + o_proj_input, o_scale = fused_inv_rope_fp8_quant( + o, + positions, + cos_sin_cache, + n_groups=n_groups, + heads_per_group=heads_per_group, + nope_dim=nope_dim, + rope_dim=rope_dim, + quant_group_size=einsum_recipe[2], + tma_aligned_scales=tma_aligned_scales, + quantize=use_fp8, + ) z = torch.empty( - (o.shape[0], n_groups, o_lora_rank), - device=o.device, + (o_proj_input.shape[0], n_groups, o_lora_rank), + device=o_proj_input.device, dtype=torch.bfloat16, ) if use_fp8: @@ -89,3 +101,58 @@ def deep_gemm_fp8_o_proj( out=z.transpose(0, 1), ) return wo_b(z.flatten(1)) + + +# TRTLLM-gen's DSv4 RopeQuant epilogue (flashinfer-ai/flashinfer#4918) has one +# fixed schedule: 128 query heads in groups of 8 512-wide heads, and FP8 with +# one packed UE8M0 scale per 128 elements. +_ROPE_QUANT_NUM_HEADS = 128 +_ROPE_QUANT_HEADS_PER_GROUP = 8 +_ROPE_QUANT_HEAD_DIM = 512 +_ROPE_QUANT_BLOCK = 128 + + +def rope_quant_unsupported_reason(layer: torch.nn.Module) -> str | None: + """Why ``layer`` cannot take FlashInfer's fused inverse RoPE + FP8 output. + + DSv4 always has 8 heads per ``wo_a`` group at any TP size, and the rest of + the kernel's fixed shape (head/RoPE dims, per-128 einsum scales, FP32 RoPE + cache) is what the SM100 layer uses anyway. What varies is the local head + count and the configs the layer also serves unquantized. + """ + if layer.n_local_heads != _ROPE_QUANT_NUM_HEADS: + # Padding fewer heads up to the kernel's fixed 128 costs more + # attention than the fused epilogue saves. + return f"RopeQuant needs {_ROPE_QUANT_NUM_HEADS} local query heads" + if layer.kv_cache_torch_dtype != torch.float8_e4m3fn: + return "RopeQuant needs the per-tensor FP8 KV cache" + if layer.wo_a.weight.dtype != torch.float8_e4m3fn: + return "RopeQuant needs an FP8 wo_a" + return None + + +def rope_quant_attn_out(num_tokens: int, device: torch.device) -> QuantizedActivation: + """Allocate the output pair the RopeQuant kernel writes. + + Values are group-major, ``[num_tokens, 16, 8 * 512]`` over physical + ``[group, token, K]``. Scales are MN-major packed UE8M0, one int32 per + head over its four 128-wide blocks. Both are what ``fp8_einsum`` consumes. + """ + n_groups = _ROPE_QUANT_NUM_HEADS // _ROPE_QUANT_HEADS_PER_GROUP + group_width = _ROPE_QUANT_HEADS_PER_GROUP * _ROPE_QUANT_HEAD_DIM + data = torch.empty( + (n_groups, num_tokens, group_width), dtype=torch.float8_e4m3fn, device=device + ).transpose(0, 1) + aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize) + scale = torch.empty( + (n_groups, group_width // (_ROPE_QUANT_BLOCK * 4), aligned), + dtype=torch.int32, + device=device, + ).permute(2, 0, 1)[:num_tokens] + return QuantizedActivation( + data=data, + scale=scale, + orig_dtype=torch.bfloat16, + orig_shape=data.shape, + quant_key=kFp8Dynamic128Sym, + ) diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py index 2ffedc97d3..be8846fc79 100644 --- a/vllm/models/deepseek_v4/quant_config.py +++ b/vllm/models/deepseek_v4/quant_config.py @@ -135,15 +135,15 @@ class DeepseekV4FP8Config(Fp8Config): def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> QuantizationMethods | None: + # Quark checkpoints are always handled by QuarkConfig. + if ( + isinstance(hf_quant_cfg, dict) + and hf_quant_cfg.get("quant_method") == "quark" + ): + return None if not ( isinstance(hf_quant_cfg, dict) - and ( - hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") - or ( - hf_quant_cfg.get("quant_method") == "quark" - and cls._is_quark_mxfp4_ocp(hf_quant_cfg) - ) - ) + and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") ): return None model_type = getattr(hf_config, "model_type", None) diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index a4000183a0..cdab7d23ae 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -57,7 +57,7 @@ class DeepseekV4SparseMLABackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [256] @staticmethod diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 1383bac820..0cad301cb6 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -895,6 +895,11 @@ def _make_deepseek_v4_weights_mapper( # renames the engram fp8 table but not its scale; route the # scale explicitly to the same module. re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv", + # Quark exports spell the same tensor ``embed.weight_scale``, + # which the ``\.scale$`` rules never match. + re.compile( + r"(engram\.embed)\.weight_scale$" + ): r"\1_tokens.weight_scale_inv", re.compile(r"\.scale$"): f".{linear_scale_name}", } else: @@ -908,6 +913,11 @@ def _make_deepseek_v4_weights_mapper( ): r"\1.weight_scale_inv", # Same engram reroute as the fp4 branch above. re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv", + # Quark exports spell the same tensor ``embed.weight_scale``, + # which the ``\.scale$`` rules never match. + re.compile( + r"(engram\.embed)\.weight_scale$" + ): r"\1_tokens.weight_scale_inv", re.compile(r"\.scale$"): f".{linear_scale_name}", } return WeightsMapper( diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index b7a67e85bd..e1588b55e7 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -13,6 +13,8 @@ from vllm.distributed import ( ) from vllm.forward_context import get_forward_context from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic from vllm.models.deepseek_v41.attention import ( DeepseekV4Attention, _replace_layer_index, @@ -37,6 +39,9 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( build_ragged_indices_from_dense, rocm_inv_rope_einsum, + rocm_inverse_rope_mxfp8_rows, + rocm_inverse_rope_rows_, + rocm_mxfp8_wo_a_bmm, rocm_sparse_attn_decode, rocm_sparse_attn_prefill, ) @@ -662,7 +667,40 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): transpose_scale=False, ) - def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _alloc_attn_out( + self, num_tokens: int, hidden_states: torch.Tensor + ) -> torch.Tensor | QuantizedActivation: + if not _ON_GFX950: + return super()._alloc_attn_out(num_tokens, hidden_states) + # wo_a's MXFP8 input: the decode reduce writes it directly, prefill + # rows are rotated and quantized after their bf16 attention. + width = self.n_local_heads * self.head_dim + data = torch.empty( + (num_tokens, width), dtype=torch.float8_e4m3fn, device=hidden_states.device + ) + scale = torch.empty( + (num_tokens, width // 32), dtype=torch.uint8, device=hidden_states.device + ) + return QuantizedActivation( + data=data, + scale=scale, + orig_dtype=hidden_states.dtype, + orig_shape=data.shape, + quant_key=kMxfp8Dynamic, + ) + + def _o_proj( + self, attn_out: torch.Tensor | QuantizedActivation, positions: torch.Tensor + ) -> torch.Tensor: + if isinstance(attn_out, QuantizedActivation): + z = rocm_mxfp8_wo_a_bmm( + attn_out.data, + attn_out.scale, + self.wo_a, + self.n_local_groups, + self.o_lora_rank, + ) + return self._wo_b_after_wo_a(z) o = attn_out[:, : self.n_local_heads, :] # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. z = rocm_inv_rope_einsum( @@ -673,27 +711,31 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self.n_local_groups, self.o_lora_rank, self.wo_a, + inverse_rope=False, ) - zf = z.flatten(1) + return self._wo_b_after_wo_a(z.flatten(1)) + + def _wo_b_after_wo_a(self, zf: torch.Tensor) -> torch.Tensor: if self._wo_b_scale is not None and zf.dim() == 2: - result = self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True) - else: - result = self.wo_b(zf) - return result + return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True) + return self.wo_b(zf) def forward_mqa( self, q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, - output: torch.Tensor, + output: torch.Tensor | QuantizedActivation, ) -> None: - assert output.shape == q.shape, ( - f"output buffer shape {output.shape} must match q shape {q.shape}" - ) - assert output.dtype == q.dtype, ( - f"output buffer dtype {output.dtype} must match q dtype {q.dtype}" - ) + mxfp8_out = output if isinstance(output, QuantizedActivation) else None + if mxfp8_out is None: + assert isinstance(output, torch.Tensor) + assert output.shape == q.shape, ( + f"output buffer shape {output.shape} must match q shape {q.shape}" + ) + assert output.dtype == q.dtype, ( + f"output buffer dtype {output.dtype} must match q dtype {q.dtype}" + ) forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata @@ -711,9 +753,13 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): ) M = N + self.window_size + self.max_num_batched_tokens current_workspace_manager().get_simultaneous( - ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), + *self._prefill_workspace_shapes(M, self.max_num_batched_tokens, q) ) - output.zero_() + if mxfp8_out is None: + output.zero_() + else: + mxfp8_out.data.zero_() + mxfp8_out.scale.zero_() return assert isinstance(attn_metadata, dict) @@ -743,19 +789,63 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): positions=positions[num_decode_tokens:], compressed_k_cache=self_kv_cache, swa_k_cache=swa_kv_cache, - output=output[num_decode_tokens:], + output=( + output[num_decode_tokens:] + if isinstance(output, torch.Tensor) + else None + ), attn_metadata=rocm_metadata, swa_metadata=swa_metadata, + mxfp8_out=( + None + if mxfp8_out is None + else ( + mxfp8_out.data[num_decode_tokens:], + mxfp8_out.scale[num_decode_tokens:], + ) + ), ) + if mxfp8_out is not None: + # The reduce epilogue rotates and quantizes every decode row and + # the prefill path already did its own, so nothing is owed here. + if num_decodes > 0: + self._forward_decode( + q=q[:num_decode_tokens], + positions=positions[:num_decode_tokens], + kv_cache=self_kv_cache, + swa_metadata=swa_metadata, + attn_metadata=rocm_metadata, + swa_only=swa_only, + output=None, + output_mxfp8=( + mxfp8_out.data[:num_decode_tokens], + mxfp8_out.scale[:num_decode_tokens], + ), + ) + return + assert isinstance(output, torch.Tensor) + rotated = 0 if num_decodes > 0: - self._forward_decode( + rotated = self._forward_decode( q=q[:num_decode_tokens], + positions=positions[:num_decode_tokens], kv_cache=self_kv_cache, swa_metadata=swa_metadata, attn_metadata=rocm_metadata, swa_only=swa_only, output=output[:num_decode_tokens], ) + # Only the decode reduce rotates its own rows, and only the leading + # `rotated` of them; prefill rows and any decode path that did not + # fuse still owe the standalone pass. Settle that here rather than in + # _o_proj: the split is batch-dependent and _o_proj runs compiled, + # where such a value freezes at its trace-time value. + rocm_inverse_rope_rows_( + output[rotated:, : self.n_local_heads, :], + positions[rotated:], + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + ) def _decode_topk_ragged( self, @@ -796,12 +886,15 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): def _forward_decode( self, q: torch.Tensor, + positions: torch.Tensor, kv_cache: torch.Tensor | None, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, attn_metadata: DeepseekV4FlashMLAMetadata | None, swa_only: bool, - output: torch.Tensor, - ) -> None: + output: torch.Tensor | None, + output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> int: + """Returns how many leading rows the decode epilogue inverse-RoPE'd.""" num_decodes = swa_metadata.num_decodes num_decode_tokens = swa_metadata.num_decode_tokens @@ -820,7 +913,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): num_decode_tokens=num_decode_tokens, ) - rocm_sparse_attn_decode( + return rocm_sparse_attn_decode( q=q, kv_cache=kv_cache, swa_k_cache=self.swa_cache_layer.kv_cache, @@ -839,6 +932,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): nope_head_dim=self.nope_head_dim, rope_head_dim=self.rope_head_dim, output=output, + output_mxfp8=output_mxfp8, + inv_rope_positions=positions, + inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache, extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free( self.kv_cache_dtype, self._has_kv_transfer, @@ -852,10 +948,16 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): positions: torch.Tensor, compressed_k_cache: torch.Tensor | None, swa_k_cache: torch.Tensor, - output: torch.Tensor, + output: torch.Tensor | None, attn_metadata: DeepseekV4FlashMLAMetadata | None, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + mxfp8_out: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> None: + """Sparse prefill into bf16 ``output``, or into ``mxfp8_out``. + + With ``mxfp8_out`` the rows go to a bf16 workspace first and are then + inverse-RoPE'd and quantized into it in one pass. + """ swa_only = attn_metadata is None num_prefills = swa_metadata.num_prefills @@ -891,10 +993,13 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self.PREFILL_CHUNK_SIZE ) - workspace_manager = current_workspace_manager() - kv = workspace_manager.get_simultaneous( - ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), - )[0] + workspace = current_workspace_manager().get_simultaneous( + *self._prefill_workspace_shapes(M, num_prefill_tokens, q) + ) + kv = workspace[0] + if mxfp8_out is not None: + output = workspace[1] + assert output is not None for chunk_idx in range(num_chunks): chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills) @@ -959,3 +1064,31 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): attn_sink=self.attn_sink, output=output[query_start:query_end], ) + if mxfp8_out is not None: + rocm_inverse_rope_mxfp8_rows( + output, + positions[:num_prefill_tokens], + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + mxfp8_out[0][:num_prefill_tokens], + mxfp8_out[1][:num_prefill_tokens], + ) + + def _prefill_workspace_shapes( + self, M: int, num_prefill_tokens: int, q: torch.Tensor + ) -> list[tuple[tuple[int, ...], torch.dtype]]: + """The prefill gather buffer, plus bf16 rows for the MXFP8 output. + + One ``get_simultaneous`` call: separate calls alias the same memory. + """ + shapes: list[tuple[tuple[int, ...], torch.dtype]] = [ + ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16) + ] + if _ON_GFX950: + shapes.append( + ( + (num_prefill_tokens, self.n_local_heads, self.head_dim), + torch.bfloat16, + ) + ) + return shapes diff --git a/vllm/models/deepseek_v41/amd/vl_model.py b/vllm/models/deepseek_v41/amd/vl_model.py index 3c57df2ad2..5ed045029b 100644 --- a/vllm/models/deepseek_v41/amd/vl_model.py +++ b/vllm/models/deepseek_v41/amd/vl_model.py @@ -79,20 +79,22 @@ class DeepseekV4VLImagePixelInputs(TensorSchema): types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})] +_VL_PREFIX_MAPPING: dict[str, str | None] = { + "layers.": "language_model.model.layers.", + "embed.": "language_model.model.embed.", + "norm.": "language_model.model.norm.", + "hc_head": "language_model.model.hc_head", + "mtp.": "language_model.model.mtp.", +} + + def _make_deepseek_v4_vl_weights_mapper( expert_dtype: str, linear_scale_name: str ) -> WeightsMapper: """Text-checkpoint mapping rules re-rooted under ``language_model.``.""" base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name) - orig_to_new_prefix: dict[str, str | None] = { - "layers.": "language_model.model.layers.", - "embed.": "language_model.model.embed.", - "norm.": "language_model.model.norm.", - "hc_head": "language_model.model.hc_head", - "mtp.": "language_model.model.mtp.", - } return WeightsMapper( - orig_to_new_prefix=orig_to_new_prefix, + orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), orig_to_new_regex=base.orig_to_new_regex, orig_to_new_suffix={ "head.weight": "language_model.lm_head.weight", @@ -131,6 +133,22 @@ class DeepseekV41ForCausalLM( supports_encoder_tp_data = True + # Both of these are read off the *class* by + # ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs, + # to translate a quantization config's checkpoint-style module names into + # runtime prefixes. ``__init__`` later overwrites the instance + # ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the + # prefix rules matter here. Without the class attributes, per-layer specs + # naming ``layers.N.attn.wq_a`` never match this wrapper's + # ``language_model.model.layers.N.attn.wq_a`` and silently fall back to + # the global spec. + packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), + orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"}, + ) + # The MoE router needs raw token ids to detect image-span tokens # (all carrying image_token_id, see common/mm_preprocess.py) and apply # bias_vl. diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index d0e69fb358..6d961f0055 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -697,7 +697,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): else: gemm_rs.warn_incompatible_projection() - def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor: + def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor: """Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result. Every ``_o_proj`` implementation projects through this so the decoder @@ -707,7 +707,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): """ if self.gemm_rs is None: return self.wo_b(z) - if self.gemm_rs.should_run(z): + if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z): return self.gemm_rs.apply(z, self.wo_b) # Small batches stay on the unfused path, which is faster there. return sp_reduce_scatter(self.wo_b(z)) diff --git a/vllm/models/deepseek_v41/common/engram.py b/vllm/models/deepseek_v41/common/engram.py index c415bbaa6e..8cd93547e6 100644 --- a/vllm/models/deepseek_v41/common/engram.py +++ b/vllm/models/deepseek_v41/common/engram.py @@ -47,7 +47,7 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.logger import init_logger -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.utils import set_weight_attrs from vllm.triton_utils import tl, triton @@ -923,13 +923,17 @@ class Engram(nn.Module): # suffix rule. self.embed_tokens = self._create_embedding(layout, layer_hash_index) n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads - self.wkv = ReplicatedLinear( + # Without sequence parallelism every TP rank holds every token, so + # shard the output columns instead of replicating the projection. + self.wkv = ColumnParallelLinear( n_hash_cols * layout.head_dim, self.dim * (self.hc_mult + 1), bias=False, + gather_output=True, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wkv", + disable_tp=use_sequence_parallel, ) self.q_weight = nn.Parameter( torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16), diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index 02fb0d5539..ad03c32945 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -1266,7 +1266,7 @@ def build_flashinfer_mixed_sparse_indices( def _remap_flashinfer_index(values, block_size, block_span): # FlashInfer's DSv4 kernel indexes sparse KV by physical token stride, so # packed pages (#44577) need block*block_size+off -> block*block_span+off. - # TODO: remove once flashinfer-ai/flashinfer#3856 is fixed. + # The launcher declares these indices with sparse_indices_are_storage_offsets=True. is_valid = values >= 0 safe_values = tl.where(is_valid, values, 0) values = (safe_values // block_size) * block_span diff --git a/vllm/models/deepseek_v41/common/ops/query_quant.py b/vllm/models/deepseek_v41/common/ops/query_quant.py index 4894a615bc..6094d1980c 100644 --- a/vllm/models/deepseek_v41/common/ops/query_quant.py +++ b/vllm/models/deepseek_v41/common/ops/query_quant.py @@ -3,7 +3,10 @@ import torch -from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -102,6 +105,7 @@ def fused_q_kv_rmsnorm_quant( ) if tokens: block = triton.next_power_of_2(max(q_size, kv_size)) + launch_pdl = current_platform.is_arch_support_pdl() _q_kv_norm_quant_kernel[(padded_tokens, 2)]( qr, kv, @@ -117,8 +121,9 @@ def fused_q_kv_rmsnorm_quant( q_size, kv_size, block, - current_platform.is_arch_support_pdl(), + launch_pdl, num_warps=8 if block >= 2048 else 4, + launch_pdl=launch_pdl, ) return QuantizedActivation(qo, scales, qr.dtype, qr.shape, kMxfp8Dynamic), kvo @@ -135,7 +140,7 @@ def can_fuse_query_quant(linears: list[torch.nn.Module]) -> bool: # QuantKey does not encode scale layout; restrict this producer to the # consumers that accept F8_128x4 swizzled scales. return all( - getattr(linear, "input_quant_key", None) == kMxfp8Dynamic + get_input_quant_key(linear) == kMxfp8Dynamic and type(getattr(getattr(linear, "quant_method", None), "kernel", None)) in (FlashInferCutedslMxfp8LinearKernel, FlashInferCutlassMxfp8LinearKernel) for linear in linears diff --git a/vllm/models/deepseek_v41/common/vl_cudagraph.py b/vllm/models/deepseek_v41/common/vl_cudagraph.py index e2a718b745..bd8ed5a3fd 100644 --- a/vllm/models/deepseek_v41/common/vl_cudagraph.py +++ b/vllm/models/deepseek_v41/common/vl_cudagraph.py @@ -25,6 +25,7 @@ from vllm.models.deepseek_v4.common.vision import ( build_packed_merge_metadata, build_packed_vit_metadata, ) +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphCaptureInputs, EncoderCudaGraphConfig, @@ -370,18 +371,23 @@ class DeepseekV4VLEncoderCudaGraphMixin: r = self.config.vision_downsample_ratio vit_grid = self._get_grid_list(batch_mm_kwargs, "vit_grid") llm_grid = self._get_grid_list(batch_mm_kwargs, "llm_grid") - types = batch_mm_kwargs["types"].to(aligner_out.device) + types = batch_mm_kwargs["types"] n_rows = sum(-(-h // r) * (-(-w // r)) for h, w in vit_grid) span_lens = [lh * (lw + 1) + 2 for lh, lw in llm_grid] - # Batched span assembly: one masked fill per role across all items. - dtype = aligner_out.dtype - span = aligner_out.new_empty(sum(span_lens), self.config.hidden_size) - span[types == IMAGE] = aligner_out[:n_rows] - span[types == IMAGE_START] = self.image_start.to(dtype) - span[types == IMAGE_END] = self.image_end.to(dtype) - span[types == IMAGE_NEW_LINE] = self.image_newline.to(dtype) + # Batched span assembly: one gather over [aligner rows, START, + # NEW_LINE, END] for all items. The index is built from the host-side + # ``types`` so that neither the copy nor the gather syncs. + is_image = types == IMAGE + src_idx = torch.where( + is_image, + is_image.cumsum(0) - 1, + n_rows + (types == IMAGE_NEW_LINE).long() + 2 * (types == IMAGE_END).long(), + ) + delims = torch.stack([self.image_start, self.image_newline, self.image_end]) + src = torch.cat([aligner_out[:n_rows], delims.to(aligner_out.dtype)]) + span = src[async_tensor_h2d(src_idx, aligner_out.device)] # Freshly allocated, so later replays cannot clobber the results. offset = 0 diff --git a/vllm/models/deepseek_v41/compressor.py b/vllm/models/deepseek_v41/compressor.py index 44440fcb61..42335ae84b 100644 --- a/vllm/models/deepseek_v41/compressor.py +++ b/vllm/models/deepseek_v41/compressor.py @@ -37,7 +37,7 @@ class CompressorBackend(AttentionBackend): return "CompressorBackend" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(1)] @classmethod diff --git a/vllm/models/deepseek_v41/nvidia/engram.py b/vllm/models/deepseek_v41/nvidia/engram.py index 7dd820fee4..7d1e0f8a4b 100644 --- a/vllm/models/deepseek_v41/nvidia/engram.py +++ b/vllm/models/deepseek_v41/nvidia/engram.py @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch.""" +import ctypes import mmap +import os import shutil import tempfile import weakref @@ -20,6 +22,10 @@ from vllm.distributed import ( get_tensor_model_parallel_rank, tensor_model_parallel_all_gather, ) +from vllm.distributed.device_communicators.shm_broadcast import ( + SHM_PATH, + check_shm_free_space, +) from vllm.distributed.parallel_state import GroupCoordinator from vllm.forward_context import get_forward_context from vllm.logger import init_logger @@ -42,6 +48,41 @@ from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor logger = init_logger(__name__) +def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None: + """Register prefaulted huge pages, or return None for pinned-memory fallback.""" + try: + mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) + mapping.madvise(mmap.MADV_HUGEPAGE) + except OSError as exc: + logger.warning( + "Engram huge-page allocation failed (%s); using pinned memory.", exc + ) + return None + + owner = np.frombuffer(mapping, dtype=np.uint8) + # Fault the pages in before CUDA pins them, so they can be huge. + owner[:: mmap.PAGESIZE] = 0 + tensor = torch.from_numpy(owner) + pointer = tensor.data_ptr() + result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) + if result.value != 0: + logger.warning( + "Engram cudaHostRegister failed (%s); using pinned memory.", result + ) + return None + # Tensor views retain owner; its finalizer retains mapping until unregister. + finalizer = weakref.finalize( + owner, DPSharedEngramStorage._unregister, mapping, pointer + ) + finalizer.atexit = False # type: ignore[misc] + if not tensor.is_pinned(): + logger.warning( + "CUDA did not recognize Engram registration; using pinned memory." + ) + return None + return tensor + + def engram_head_shard_rank() -> int: """This rank's slot among the hash-head shards of one engram table. @@ -92,6 +133,13 @@ def gather_engram_hashes( def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: """Whether co-located DP replicas exist and /dev/shm can hold the full tables.""" + if not os.path.isdir(SHM_PATH): + logger.warning_once( + "%s is not mounted; storing the offloaded Engram tables per rank " + "instead of sharing them across DP replicas.", + SHM_PATH, + ) + return False if get_engram_dp_size() == 1: logger.warning_once( "Engram DP replicas are not co-located on one node; " @@ -101,11 +149,12 @@ def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: num_bytes = sum(layout.num_embeddings) * ( layout.head_dim + layout.head_dim // block_size ) - if shutil.disk_usage("/dev/shm").total < num_bytes: + if shutil.disk_usage(SHM_PATH).total < num_bytes: logger.warning_once( - "Sharing Engram tables across DP replicas needs %.1f GiB of /dev/shm " + "Sharing Engram tables across DP replicas needs %.1f GiB of %s " "(--shm-size or --ipc=host); sharding them across replicas instead.", num_bytes / 1024**3, + SHM_PATH, ) return False return True @@ -126,20 +175,14 @@ class DPSharedEngramStorage: def _allocate(self, num_bytes: int) -> torch.Tensor: """Map and register one physical allocation across a node-local DP group.""" - from vllm.distributed.device_communicators.shm_broadcast import ( - check_shm_free_space, - ) - group = self.group with ExitStack() as stack: - path, error = None, None + path = error = None if group.rank_in_group == 0: try: check_shm_free_space(num_bytes) backing_file = stack.enter_context( - tempfile.NamedTemporaryFile( - prefix="vllm_engram_", dir="/dev/shm" - ) + tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH) ) backing_file.truncate(num_bytes) path = backing_file.name @@ -151,57 +194,42 @@ class DPSharedEngramStorage: "Engram shared-memory creation failed on EDP rank 0: " + error ) - mapping = owner = tensor = finalizer = None - stage = "open" + mapping = owner = tensor = None try: - try: - with open(path, "r+b") as file: - stage = "mmap" - mapping = mmap.mmap( - file.fileno(), num_bytes, flags=mmap.MAP_SHARED - ) - stage = "cudaHostRegister" - owner = np.frombuffer(mapping, dtype=np.uint8) - pointer = owner.ctypes.data - tensor = torch.from_numpy(owner) - result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) - if result.value != 0: - raise RuntimeError(f"cudaHostRegister failed: {result}") - finalizer = weakref.finalize( - owner, self._unregister, mapping, pointer - ) - finalizer.atexit = False # type: ignore[misc] - # The UVA helper otherwise allocates a private pinned copy. - if not tensor.is_pinned(): - raise RuntimeError( - "CUDA did not recognize the shared Engram registration" - ) - except Exception as exc: - error = f"{stage}: {type(exc).__name__}: {exc}" - - errors: list[str | None] = [None] * group.world_size - # Also fences peer mappings before the leader unlinks the file. - torch.distributed.all_gather_object( - errors, error, group=group.cpu_group - ) - failures = "; ".join( - f"EDP rank {rank}: {error}" - for rank, error in enumerate(errors) - if error is not None - ) - if failures: + with open(path, "r+b") as file: + mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED) + owner = np.frombuffer(mapping, dtype=np.uint8) + pointer = owner.ctypes.data + tensor = torch.from_numpy(owner) + result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) + if result.value != 0: + raise RuntimeError(f"cudaHostRegister failed: {result}") + finalizer = weakref.finalize(owner, self._unregister, mapping, pointer) + finalizer.atexit = False # type: ignore[misc] + # The UVA helper otherwise allocates a private pinned copy. + if not tensor.is_pinned(): raise RuntimeError( - "Engram shared-memory initialization failed: " + failures + "cudaHostRegister did not pin the shared Engram mapping" ) - assert tensor is not None - return tensor - except Exception: - if finalizer is not None: - finalizer() - tensor = owner = None - if mapping is not None: - mapping.close() - raise + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + + # Also fences peer mappings before the leader unlinks the file. + errors: list[str | None] = [None] * group.world_size + torch.distributed.all_gather_object(errors, error, group=group.cpu_group) + failures = "; ".join( + f"EDP rank {rank}: {error}" + for rank, error in enumerate(errors) + if error is not None + ) + if failures: + # Dropping the owner runs the finalizer; the mapping then unmaps. + del tensor, owner, mapping + raise RuntimeError( + "Engram shared-memory initialization failed: " + failures + ) + assert tensor is not None + return tensor @staticmethod def _unregister(mapping: mmap.mmap, pointer: int) -> None: @@ -249,9 +277,12 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding): block_size: int = 32, cpu_offload: bool = False, dp_shared_memory: bool = False, + use_thp: bool = False, ) -> None: self.cpu_offload = cpu_offload self.dp_shared_memory = dp_shared_memory + self.use_thp = use_thp + self._packed: torch.Tensor | None = None self.dp_size = get_engram_dp_size() if dp_shared_memory: if not cpu_offload: @@ -300,6 +331,17 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding): return storage.weight, storage.weight_scale_inv if not self.cpu_offload: return super()._allocate_weights() + if self.use_thp: + weight_bytes = self.part_num_embeddings * self.dim + packed = _allocate_huge_page_storage( + weight_bytes + weight_bytes // self.block_size + ) + if packed is not None: + self._packed = packed + return ( + packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim), + packed[weight_bytes:].view(-1, self.dim // self.block_size), + ) # Model initialization may be inside a CUDA device context. return ( torch.empty( @@ -318,6 +360,19 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding): ), ) + def collapse_huge_pages(self) -> None: + """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.""" + if self._packed is None: + return + addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes + libc = ctypes.CDLL(None, use_errno=True) + libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + if libc.madvise(addr, num_bytes, 25) != 0: + logger.warning( + "Engram MADV_COLLAPSE failed; keeping existing pages: %s", + os.strerror(ctypes.get_errno()), + ) + def _storage(self) -> tuple[torch.Tensor, torch.Tensor]: if self._shared_memory is not None: return self._shared_memory.get_views(self.weight, self.weight_scale_inv) @@ -373,6 +428,15 @@ class Engram(BaseEngram): """NVIDIA Engram with asynchronous offload and node-local DP lookup.""" _prefetch_stream: torch.cuda.Stream | None = None + _prefetch_done: torch.cuda.Event | None = None + + def __init__( + self, *args, prefetch_stream: torch.cuda.Stream | None = None, **kwargs + ) -> None: + # Layers sharing one stream serialize their offloaded lookups, so they + # take turns instead of jointly starving decoder compute of SMs. + self._prefetch_stream = prefetch_stream + super().__init__(*args, **kwargs) def _create_embedding( self, layout: EngramLayout, layer_hash_index: int @@ -385,12 +449,17 @@ class Engram(BaseEngram): tuple(size for order in layout.primes[layer_hash_index] for size in order), cpu_offload=engram_config.cpu_offload, dp_shared_memory=bool(engram_config.dp_shared_memory), + use_thp=engram_config.use_thp, ) def _init_staging(self, max_tokens: int, head_dim: int) -> None: super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim) - if self.embed_tokens.cpu_offload: + if not self.embed_tokens.cpu_offload: + self._prefetch_stream = None + return + if self._prefetch_stream is None: self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device) + self._prefetch_done = torch.cuda.Event() def prepare_embeddings(self, hash_ids: torch.Tensor) -> None: """Prefetch local shared rows or the DP group's gathered hash IDs.""" @@ -410,14 +479,17 @@ class Engram(BaseEngram): hash_ids.record_stream(stream) with torch.cuda.stream(stream): self.embed_tokens.lookup(hash_ids, rows, background=True) + assert self._prefetch_done is not None + self._prefetch_done.record(stream) @eager_break_during_capture - def _finish_prefetch(self, stream: torch.cuda.Stream) -> None: - torch.cuda.current_stream().wait_stream(stream) + def _finish_prefetch(self, event: torch.cuda.Event) -> None: + # Wait for this layer's lookup only, not for later layers on the stream. + torch.cuda.current_stream().wait_event(event) def _ready_rows(self, num_tokens: int) -> torch.Tensor: - if self._prefetch_stream is not None: - self._finish_prefetch(self._prefetch_stream) + if self._prefetch_done is not None: + self._finish_prefetch(self._prefetch_done) if self.embed_tokens.dp_size > 1: slot = engram_gathered_num_tokens() staged = self.staged_rows[: slot * self.embed_tokens.dp_size] diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 485cdd9ebd..0e02b486a5 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -9,15 +9,16 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( build_flashinfer_mixed_sparse_indices, compute_global_topk_indices_and_lens, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, DeepseekV4SparseMLABackend, @@ -113,7 +114,7 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [128] @staticmethod @@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) # Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the # per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``. if self.kv_cache_torch_dtype != torch.float8_e4m3fn: @@ -560,6 +548,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): swa_kv_cache=swa_k_cache, workspace_buffer=workspace, sparse_indices=sparse_indices[:num_decode_tokens], + sparse_indices_are_storage_offsets=True, compressed_kv_cache=compressed_kv_cache, sparse_topk_lens=decode_topk_lens, seq_lens=decode_seq_lens, @@ -585,6 +574,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): swa_kv_cache=swa_k_cache, workspace_buffer=workspace, sparse_indices=sparse_indices[num_decode_tokens:num_tokens], + sparse_indices_are_storage_offsets=True, compressed_kv_cache=compressed_kv_cache, sparse_topk_lens=sparse_topk_lens[num_decode_tokens:num_tokens], seq_lens=seq_lens[num_decodes:num_reqs], @@ -621,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: super().__init__(vllm_config, *args, **kwargs) diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index 96dde36c61..ce3047ceca 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast import torch from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( combine_topk_swa_indices, compute_global_topk_indices_and_lens, dequantize_and_gather_k_cache, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, @@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index 22674bf705..9e7a35f0fc 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -33,6 +33,14 @@ from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import ( + TrtLlmMxfp4ExpertsModular, +) +from vllm.model_executor.layers.fused_moe.moe_output import ( + MoEOutput, + UnfinalizedMoEOutput, +) +from vllm.model_executor.layers.fused_moe.runner.moe_runner import _unpack from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -40,7 +48,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + drop_checkpoint_cache, +) from vllm.model_executor.models.interfaces import ( EagleModelMixin, MixtureOfExperts, @@ -91,6 +102,7 @@ from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask from .engram import Engram, can_share_engram_tables, gather_engram_hashes from .ops.mhc import ( MHC_OVERLAP_MAX_TOKENS, + init_mhc_all_reduce, mhc_pre_delayed_overlap, mhc_shifted_post_pre, supports_mhc_all_reduce, @@ -132,6 +144,48 @@ class DeepseekV4MoE(DeepseekV4MoEBase): image_sentinel_lo=IMAGE_SENTINEL_BASE_ID, ) + def defer_finalize(self) -> None: + """Leave the routed top-k reduction to the next fused all-reduce + mHC. + + Only the modular TRTLLM MXFP4 experts can stop after GEMM2; the fused + kernel takes as many tokens as the mHC overlap path. + """ + moe_config = self.experts.moe_config + experts_cls = getattr( + self.experts.routed_experts.quant_method, "experts_cls", None + ) + if ( + experts_cls is TrtLlmMxfp4ExpertsModular + and self.experts.routed_scaling_factor == 1.0 + ): + moe_config.defer_moe_finalize(MHC_OVERLAP_MAX_TOKENS) + if moe_config.use_deferred_moe_finalize: + logger.info_once( + "DSV4.1 mHC: MoE top-k finalize fused into the all-reduce for " + "up to %d tokens.", + MHC_OVERLAP_MAX_TOKENS, + ) + + def defers_finalize(self, num_tokens: int) -> bool: + return ( + not self.use_mega_moe + and self.experts.moe_config.should_defer_moe_finalize(num_tokens) + ) + + def forward_unfinalized( + self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None + ) -> MoEOutput: + """``forward`` with the routed top-k reduction and all-reduce left open.""" + # The runner's custom op returns tensors only, so run its body directly. + shared_output, routed = _unpack( + self.experts._forward_impl( + hidden_states, hidden_states, hidden_states, input_ids + ) + ) + assert shared_output is not None + assert isinstance(routed, UnfinalizedMoEOutput) + return MoEOutput(routed=routed, shared_output=shared_output) + def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: """Pick the CUDA sparse-MLA attention class for the configured backend. @@ -226,6 +280,7 @@ class DeepseekV4DecoderLayer(nn.Module): aux_stream_list: list[torch.cuda.Stream] | None = None, candidate_block_buffer: torch.Tensor | None = None, engram_layout: EngramLayout | None = None, + engram_prefetch_stream: torch.cuda.Stream | None = None, run_gemm_rs: bool = False, mhc_stream: torch.cuda.Stream | None = None, fuse_mhc_all_reduce: bool = False, @@ -249,6 +304,7 @@ class DeepseekV4DecoderLayer(nn.Module): engram_layout.layer_ids.index(layer_id), use_sequence_parallel=self.use_sequence_parallel, prefix=f"{prefix}.engram", + prefetch_stream=engram_prefetch_stream, ) self.rms_norm_eps = config.rms_norm_eps @@ -389,7 +445,7 @@ class DeepseekV4DecoderLayer(nn.Module): def forward( self, - x: torch.Tensor, + x: torch.Tensor | MoEOutput, positions: torch.Tensor, input_ids: torch.Tensor | None, pre_mix: torch.Tensor | None = None, @@ -402,7 +458,7 @@ class DeepseekV4DecoderLayer(nn.Module): capture_previous_aux: bool = False, mega_gate_metadata: MegaGateRoutingMetadata | None = None, ) -> tuple[ - torch.Tensor, + torch.Tensor | MoEOutput, torch.Tensor, torch.Tensor, torch.Tensor, @@ -427,6 +483,7 @@ class DeepseekV4DecoderLayer(nn.Module): # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. if residual is None: + assert isinstance(x, torch.Tensor) if x.dim() == 2: # First layer: the stream is the embedding broadcast to hc # copies and the identity pre-mix selects copy 0. @@ -466,7 +523,9 @@ class DeepseekV4DecoderLayer(nn.Module): # Engram injection happens between the previous sublayer's post # and this block's pre, on the full hc stream, so the mix # coefficients see the injected stream. The injection also keeps - # the post out of the pre-norm GEMM's fused prologue. + # the post out of the pre-norm GEMM's fused prologue. The layer + # before an engram layer finalizes its own MoE. + assert isinstance(x, torch.Tensor) if self.fuse_mhc_all_reduce: x = tensor_model_parallel_all_reduce(x) previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix) @@ -547,7 +606,11 @@ class DeepseekV4DecoderLayer(nn.Module): stream=mhc_stream, reduce_results=self.fuse_mhc_all_reduce, ) - x = self.ffn(x, input_ids, mega_gate_metadata) + if self.fuse_mhc_all_reduce and self.ffn.defers_finalize(x.shape[0]): + # The next layer's first boundary finalizes it with the all-reduce. + x = self.ffn.forward_unfinalized(x, input_ids) + else: + x = self.ffn(x, input_ids, mega_gate_metadata) if mhc_stream is not None: torch.cuda.current_stream().wait_stream(mhc_stream) return x, residual, post_mix, res_mix, ffn_pre, previous_aux @@ -586,6 +649,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce( vllm_config ) + if self.fuse_mhc_all_reduce: + init_mhc_all_reduce(vllm_config) # Reserved topk indices buffer for all Indexer layers to reuse. self.topk_indices_buffer = torch.empty( @@ -621,6 +686,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): self.engram_layout = EngramLayout.from_config(config) engram_config = vllm_config.engram_config + self.vllm_config = vllm_config + engram_prefetch_stream = ( + torch.cuda.Stream() + if self.engram_layout is not None + and engram_config is not None + and engram_config.cpu_offload + else None + ) if ( self.engram_layout is not None and engram_config is not None @@ -628,6 +701,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): ): engram_config.dp_shared_memory = can_share_engram_tables(self.engram_layout) + if self.engram_layout is not None and engram_config and engram_config.use_thp: + # Release old checkpoint cache before allocating the Engram host tables. + model_config = vllm_config.model_config + drop_checkpoint_cache( + model_config.model_weights or model_config.model, + revision=model_config.revision, + cache_dir=vllm_config.load_config.download_dir, + ) + # GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP # ranks to belong to one NVLink domain. Collective: run before layers. self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel) @@ -641,12 +723,23 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): aux_stream_list=aux_stream_list, candidate_block_buffer=self.candidate_block_buffer, engram_layout=self.engram_layout, + engram_prefetch_stream=engram_prefetch_stream, run_gemm_rs=self.run_gemm_rs, mhc_stream=mhc_stream, fuse_mhc_all_reduce=self.fuse_mhc_all_reduce, ), prefix=f"{prefix}.layers", ) + if self.fuse_mhc_all_reduce: + # A MoE's top-k finalize folds into the next layer's first mHC + # boundary, which the last local layer lacks and an engram layer + # replaces with its own all-reduce. + local_layers = list(islice(self.layers, self.start_layer, self.end_layer)) + for layer, successor in zip(local_layers, local_layers[1:]): + assert isinstance(layer, DeepseekV4DecoderLayer) + assert isinstance(successor, DeepseekV4DecoderLayer) + if successor.engram is None: + layer.ffn.defer_finalize() # The n-gram hash needs a slot-keyed rolling store of compressed ids # (chunked prefill / decode lookback); key it off the first local @@ -851,6 +944,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if self.use_sequence_parallel: previous_aux = sp_all_gather(previous_aux)[:full_num_tokens] aux_hidden_by_layer[idx] = previous_aux + # Without a successor boundary, the last layer finalized its own MoE. + assert isinstance(hidden_states, torch.Tensor) if layer is not None: # The last layer has no successor to fold its post into. if self.fuse_mhc_all_reduce: @@ -1329,6 +1424,20 @@ class DeepseekV41LLMForCausalLM( loader = AutoWeightsLoader(self) loaded_params = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) self.process_weights_after_loading() + config = self.model.vllm_config + if config.engram_config and config.engram_config.use_thp: + # Loading weights refills the file cache; release it before MADV_COLLAPSE. + model_config = config.model_config + drop_checkpoint_cache( + model_config.model_weights or model_config.model, + revision=model_config.revision, + cache_dir=config.load_config.download_dir, + ) + for layer in islice( + self.model.layers, self.model.start_layer, self.model.end_layer + ): + if layer.engram is not None: + layer.engram.embed_tokens.collapse_huge_pages() return loaded_params def process_weights_after_loading(self) -> None: diff --git a/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/__init__.py b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/__init__.py new file mode 100644 index 0000000000..5e126a42d6 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""CuTe DSL kernels for DSV4.1 mHC.""" + +from .all_reduce_mhc import AllReduceMHC + +__all__ = ["AllReduceMHC"] diff --git a/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/all_reduce_mhc.py b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/all_reduce_mhc.py new file mode 100644 index 0000000000..473ad28102 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/all_reduce_mhc.py @@ -0,0 +1,1134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from FlashInfer's low-latency MNNVL CuTe DSL all-reduce +# (flashinfer/comm/mnnvl_cutedsl/kernel_ll, flashinfer-ai/flashinfer@139af6f6). + +"""Lamport all-reduce fused with DSV4.1's mHC post, collapse and RMSNorm. + +FlashInfer's LL kernels, copied: a publish kernel (or its MoE finalize +variant) multicasts each rank's contribution into a three-generation Lamport +mailbox, and a PDL-chained consumer spins until every rank's fragment lands, +reduces it and normalizes. The consumer's residual add and copy are replaced by +the mHC post-mix of the hc residual streams and their pre-mix collapse, which +the RMSNorm then normalizes. The finalize variant also reads FP32 routing +weights. +""" + +from __future__ import annotations + +import math + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +import torch.distributed as dist +import torch.distributed._symmetric_memory as symm_mem +from cutlass import BFloat16, Float32, Int32, Int64, Uint32 +from cutlass.cute.runtime import make_fake_compact_tensor + +from vllm.distributed import get_tp_group + +from .primitives import ( + NEGATIVE_ZERO_BF16_BITS, + QUAD_BF16, + VEC_BF16, + WARP_SIZE, + bf16x4_to_packed_u32x2, + bf16x8_to_packed_u32x4, + current_cu_stream, + f32_to_bf16_bits, + fragment_has_negative_zero, + load_global_bf16_as_f32, + load_global_u32x2, + load_global_u32x4, + load_volatile_u32, + make_fake_dynamic_compact_tensor, + map_shared_to_peer, + packed_u32x2_to_bf16x4, + packed_u32x4_to_bf16x8, + sanitize_negative_zero_u32x2, + sanitize_negative_zero_u32x4, + shuffle_sync_idx_u32, + stmc_bf16x2, + stmc_bf16x4, + stmc_bf16x8, + store_global_u32, + store_global_u32x4, + store_lamport_sentinel_u32x4, + store_shared_cluster_f32, + to_cute, + to_cute_dynamic, +) + +LAMPORT_GENERATIONS = 3 +NEXT_STAGE = 0 +ACTIVE_STAGE = 1 + + +@cute.jit +def _group_leader_block_sum( + value: Float32, + warp_sums: cute.Tensor, + warps: cutlass.Constexpr[int], + leader_stride: cutlass.Constexpr[int], +) -> Float32: + lane = cute.arch.lane_idx() + warp = cute.arch.warp_idx() + for offset in cutlass.range_constexpr(1, WARP_SIZE): + if cutlass.const_expr(offset >= leader_stride and (offset & (offset - 1)) == 0): + value = value + cute.arch.shuffle_sync_bfly( + value, + offset=offset, + mask=-1, + mask_and_clamp=31, + ) + if lane == 0: + cute.arch.store((warp_sums + warp).llvm_ptr, value) + cute.arch.barrier() + + result = Float32(0.0) + if warp == 0: + if lane < Int32(warps): + result = cute.arch.load( + (warp_sums + lane).llvm_ptr, + Float32, + ) + result = cute.arch.warp_reduction_sum(result) + if lane == 0: + cute.arch.store(warp_sums.llvm_ptr, result) + cute.arch.barrier() + return cute.arch.load(warp_sums.llvm_ptr, Float32) + + +class _SharedOnlyPublishDeviceKernel: + def __init__( + self, + *, + hidden: int, + tp: int, + rank: int, + capacity_m: int, + elements_per_thread: int, + threads: int, + release_before_store: bool, + enable_pdl: bool, + ) -> None: + if elements_per_thread not in (1, QUAD_BF16, VEC_BF16): + raise ValueError("elements_per_thread must be 1, 4, or 8") + if hidden <= 0 or hidden % elements_per_thread: + raise ValueError("hidden must divide evenly across thread fragments") + self.hidden = hidden + self.tp = tp + self.rank = rank + self.capacity_m = capacity_m + self.elements_per_thread = elements_per_thread + self.threads = threads + self.fragments = hidden // elements_per_thread + self.ctas_per_token = math.ceil(self.fragments / threads) + self.release_before_store = release_before_store + self.enable_pdl = enable_pdl + + @cute.jit + def __call__( + self, + shared_output: cute.Tensor, + stage_state: cute.Tensor, + contribution_mailbox_multicast_address: Int64, + m: Int32, + stream: cuda.CUstream, + ) -> None: + self.kernel( + shared_output, + stage_state, + contribution_mailbox_multicast_address, + ).launch( + grid=(m * self.ctas_per_token, 1, 1), + block=(self.threads, 1, 1), + stream=stream, + use_pdl=self.enable_pdl, + ) + + @cute.kernel + def kernel( + self, + shared_output: cute.Tensor, + stage_state: cute.Tensor, + contribution_mailbox_multicast_address: Int64, + ) -> None: + block, _, _ = cute.arch.block_idx() + tidx, _, _ = cute.arch.thread_idx() + token = block // self.ctas_per_token + cta_in_token = block % self.ctas_per_token + fragment = cta_in_token * self.threads + tidx + + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_wait() + + if cutlass.const_expr(self.elements_per_thread == 1): + bits = Uint32(0) + if fragment < self.fragments: + element = Int64(token) * self.hidden + Int64(fragment) + bits = f32_to_bf16_bits( + load_global_bf16_as_f32( + Int64((shared_output.iterator + element).toint()) + ) + ) + if bits == Uint32(NEGATIVE_ZERO_BF16_BITS): + bits = Uint32(0) + partner_bits = shuffle_sync_idx_u32( + bits, + Int32(cute.arch.lane_idx() | Int32(1)), + ) + packed_word = bits | (partner_bits << Uint32(16)) + elif cutlass.const_expr(self.elements_per_thread == QUAD_BF16): + packed_words = cute.make_rmem_tensor(cute.make_layout((2,)), Uint32) + packed_words.fill(Uint32(0)) + if fragment < self.fragments: + element = Int64(token) * self.hidden + Int64(fragment) * QUAD_BF16 + pointer = cute.make_ptr( + BFloat16, + (shared_output.iterator + element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=8, + ) + packed_words.store( + sanitize_negative_zero_u32x2(load_global_u32x2(pointer)) + ) + else: + packed_words = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32) + packed_words.fill(Uint32(0)) + if fragment < self.fragments: + element = Int64(token) * self.hidden + Int64(fragment) * VEC_BF16 + pointer = cute.make_ptr( + BFloat16, + (shared_output.iterator + element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=16, + ) + packed_words.store( + sanitize_negative_zero_u32x4(load_global_u32x4(pointer)) + ) + + stage = load_volatile_u32(stage_state.iterator + NEXT_STAGE) + if cutlass.const_expr(self.release_before_store): + if block == 0 and tidx == 0: + store_global_u32( + stage_state.iterator + ACTIVE_STAGE, + stage, + ) + cute.arch.barrier() + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_launch_dependents() + + if cutlass.const_expr(self.elements_per_thread == 1): + if (cute.arch.lane_idx() & Int32(1)) == Int32( + 0 + ) and fragment < self.fragments: + mailbox_element = ( + (Int64(stage) * self.tp + self.rank) * self.capacity_m + + Int64(token) + ) * self.hidden + Int64(fragment) + stmc_bf16x2( + contribution_mailbox_multicast_address + mailbox_element * 2, + packed_word, + ) + elif cutlass.const_expr(self.elements_per_thread == QUAD_BF16): + if fragment < self.fragments: + mailbox_element = ( + (Int64(stage) * self.tp + self.rank) * self.capacity_m + + Int64(token) + ) * self.hidden + Int64(fragment) * QUAD_BF16 + stmc_bf16x4( + contribution_mailbox_multicast_address + mailbox_element * 2, + packed_words, + ) + else: + if fragment < self.fragments: + mailbox_element = ( + (Int64(stage) * self.tp + self.rank) * self.capacity_m + + Int64(token) + ) * self.hidden + Int64(fragment) * VEC_BF16 + stmc_bf16x8( + contribution_mailbox_multicast_address + mailbox_element * 2, + packed_words, + ) + + if cutlass.const_expr(not self.release_before_store): + if block == 0 and tidx == 0: + store_global_u32( + stage_state.iterator + ACTIVE_STAGE, + stage, + ) + cute.arch.barrier() + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_launch_dependents() + + +class _QuadFinalizePublishDeviceKernel: + def __init__( + self, + *, + hidden: int, + top_k: int, + tp: int, + rank: int, + capacity_m: int, + threads: int, + routed_scaling_factor: float, + include_shared_expert: bool, + load_shared_expert_before_pdl: bool, + enable_pdl: bool, + prefetch_group: int, + fp32_weights: bool = False, + ) -> None: + if hidden <= 0 or hidden % QUAD_BF16: + raise ValueError("hidden must be a positive multiple of 4") + self.hidden = hidden + self.top_k = top_k + self.tp = tp + self.rank = rank + self.capacity_m = capacity_m + self.threads = threads + self.routed_scaling_factor = routed_scaling_factor + self.fragments = hidden // QUAD_BF16 + self.ctas_per_token = math.ceil(self.fragments / threads) + self.include_shared_expert = include_shared_expert + self.load_shared_expert_before_pdl = load_shared_expert_before_pdl + self.enable_pdl = enable_pdl + self.prefetch_group = prefetch_group + self.prefetch_groups = (top_k + prefetch_group - 1) // prefetch_group + self.fp32_weights = fp32_weights + + def smem_size_in_bytes(self) -> int: + return self.top_k * 8 + + @cute.jit + def __call__( + self, + routed_output: cute.Tensor, + expert_weights: cute.Tensor, + permuted_indices: cute.Tensor, + shared_output: cute.Tensor, + stage_state: cute.Tensor, + contribution_mailbox_multicast_address: Int64, + m: Int32, + stream: cuda.CUstream, + ) -> None: + self.kernel( + routed_output, + expert_weights, + permuted_indices, + shared_output, + stage_state, + contribution_mailbox_multicast_address, + ).launch( + grid=(m * self.ctas_per_token, 1, 1), + block=(self.threads, 1, 1), + smem=self.smem_size_in_bytes(), + stream=stream, + use_pdl=self.enable_pdl, + ) + + @cute.kernel + def kernel( + self, + routed_output: cute.Tensor, + expert_weights: cute.Tensor, + permuted_indices: cute.Tensor, + shared_output: cute.Tensor, + stage_state: cute.Tensor, + contribution_mailbox_multicast_address: Int64, + ) -> None: + block, _, _ = cute.arch.block_idx() + tidx, _, _ = cute.arch.thread_idx() + token = block // self.ctas_per_token + cta_in_token = block % self.ctas_per_token + fragment = cta_in_token * self.threads + tidx + + smem = cutlass.utils.SmemAllocator() + staged_indices = smem.allocate_array(Int32, self.top_k) + staged_weights = smem.allocate_array(Float32, self.top_k) + metadata_index = Int32(tidx) + while metadata_index < Int32(self.top_k): + element = Int64(token) * self.top_k + Int64(metadata_index) + row = cute.arch.load( + (permuted_indices.iterator + element).llvm_ptr, + Int32, + ) + if cutlass.const_expr(self.fp32_weights): + weight = cute.arch.load( + (expert_weights.iterator + element).llvm_ptr, + Float32, + ) + else: + weight = load_global_bf16_as_f32( + Int64((expert_weights.iterator + element).toint()) + ) + if cutlass.const_expr(self.routed_scaling_factor != 1.0): + weight = weight * Float32(self.routed_scaling_factor) + if row == Int32(-1): + weight = Float32(0.0) + cute.arch.store( + (staged_indices + metadata_index).llvm_ptr, + row, + ) + cute.arch.store( + (staged_weights + metadata_index).llvm_ptr, + weight, + ) + metadata_index = metadata_index + self.threads + cute.arch.barrier() + + if cutlass.const_expr(self.include_shared_expert): + shared_values = cute.make_rmem_tensor( + cute.make_layout((QUAD_BF16,)), BFloat16 + ) + shared_values.fill(BFloat16(0.0)) + if cutlass.const_expr(self.load_shared_expert_before_pdl): # noqa: SIM102 + if fragment < self.fragments: + shared_element = ( + Int64(token) * self.hidden + Int64(fragment) * QUAD_BF16 + ) + shared_pointer = cute.make_ptr( + BFloat16, + (shared_output.iterator + shared_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=8, + ) + shared_values.store( + packed_u32x2_to_bf16x4(load_global_u32x2(shared_pointer)) + ) + + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_wait() + + if cutlass.const_expr( # noqa: SIM102 + self.include_shared_expert and not self.load_shared_expert_before_pdl + ): + if fragment < self.fragments: + shared_element = ( + Int64(token) * self.hidden + Int64(fragment) * QUAD_BF16 + ) + shared_pointer = cute.make_ptr( + BFloat16, + (shared_output.iterator + shared_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=8, + ) + shared_values.store( + packed_u32x2_to_bf16x4(load_global_u32x2(shared_pointer)) + ) + + stage = load_volatile_u32(stage_state.iterator + NEXT_STAGE) + if fragment < self.fragments: + accumulator = cute.make_rmem_tensor(cute.make_layout((QUAD_BF16,)), Float32) + accumulator.fill(Float32(0.0)) + if cutlass.const_expr(self.prefetch_group == 1): + for k in cutlass.range_constexpr(self.top_k): + row = cute.arch.load((staged_indices + k).llvm_ptr, Int32) + weight = cute.arch.load( + (staged_weights + k).llvm_ptr, + Float32, + ) + if row != Int32(-1): + source_element = ( + Int64(row) * self.hidden + Int64(fragment) * QUAD_BF16 + ) + source_pointer = cute.make_ptr( + BFloat16, + (routed_output.iterator + source_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=8, + ) + accumulator.store( + accumulator.load() + + packed_u32x2_to_bf16x4( + load_global_u32x2(source_pointer) + ).to(Float32) + * weight + ) + else: + inputs = cute.make_rmem_tensor( + cute.make_layout((self.prefetch_group, 2)), + Uint32, + ) + inputs.fill(Uint32(0)) + for group in cutlass.range_constexpr(self.prefetch_groups): + for item in cutlass.range_constexpr(self.prefetch_group): + k = group * self.prefetch_group + item + if cutlass.const_expr(k < self.top_k): + row = cute.arch.load( + (staged_indices + k).llvm_ptr, + Int32, + ) + for word in cutlass.range_constexpr(2): + inputs[item, word] = Uint32(0) + if row != Int32(-1): + source_element = ( + Int64(row) * self.hidden + + Int64(fragment) * QUAD_BF16 + ) + source_pointer = cute.make_ptr( + BFloat16, + (routed_output.iterator + source_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=8, + ) + source = load_global_u32x2(source_pointer) + for word in cutlass.range_constexpr(2): + inputs[item, word] = source[word] + for item in cutlass.range_constexpr(self.prefetch_group): + k = group * self.prefetch_group + item + if cutlass.const_expr(k < self.top_k): + source = cute.make_rmem_tensor( + cute.make_layout((2,)), + Uint32, + ) + for word in cutlass.range_constexpr(2): + source[word] = inputs[item, word] + accumulator.store( + accumulator.load() + + packed_u32x2_to_bf16x4(source.load()).to(Float32) + * cute.arch.load( + (staged_weights + k).llvm_ptr, + Float32, + ) + ) + result = accumulator.load() + if cutlass.const_expr(self.include_shared_expert): + result = result + shared_values.load().to(Float32) + result_packed = bf16x4_to_packed_u32x2(result.to(BFloat16)) + packed = sanitize_negative_zero_u32x2(result_packed) + mailbox_element = ( + (Int64(stage) * self.tp + self.rank) * self.capacity_m + Int64(token) + ) * self.hidden + Int64(fragment) * QUAD_BF16 + stmc_bf16x4( + contribution_mailbox_multicast_address + mailbox_element * 2, + packed, + ) + + if block == 0 and tidx == 0: + store_global_u32( + stage_state.iterator + ACTIVE_STAGE, + stage, + ) + cute.arch.barrier() + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_launch_dependents() + + +class _LamportMHCDeviceKernel: + """FlashInfer's ``_LamportResidualRMSNormDeviceKernel`` with mHC. + + The residual add and residual copy become the mHC post-mix of the hc + streams, stored as the residual output, and their pre-mix collapse, which + the unchanged RMSNorm normalizes. + """ + + def __init__( + self, + *, + hidden: int, + hc: int, + tp: int, + capacity_m: int, + cluster_size: int, + rank_lanes: int, + threads: int, + enable_pdl: bool, + ) -> None: + if rank_lanes not in (1, 2, 4, 8): + raise ValueError("rank_lanes must be 1, 2, 4, or 8") + if tp % rank_lanes: + raise ValueError("tp must be divisible by rank_lanes") + if threads <= 0 or threads % WARP_SIZE or threads % rank_lanes: + raise ValueError("threads must be a positive warp and rank-lane multiple") + if hidden <= 0 or hidden % VEC_BF16: + raise ValueError("hidden must be a positive multiple of 8") + self.hidden = hidden + self.hc = hc + self.tp = tp + self.capacity_m = capacity_m + self.cluster_size = cluster_size + self.rank_lanes = rank_lanes + self.threads = threads + self.enable_pdl = enable_pdl + self.fragments = hidden // VEC_BF16 + self.groups_per_cta = threads // rank_lanes + self.fragment_stride = cluster_size * self.groups_per_cta + self.trips = math.ceil(self.fragments / self.fragment_stride) + self.warps = threads // WARP_SIZE + self.rank_waves = tp // rank_lanes + + def smem_size_in_bytes(self) -> int: + return (self.warps + self.cluster_size) * 4 + + @cute.jit + def __call__( + self, + contribution_mailbox: cute.Tensor, + residual_source: cute.Tensor, + post: cute.Tensor, + comb: cute.Tensor, + pre: cute.Tensor, + gamma: cute.Tensor, + residual_output: cute.Tensor, + norm_output: cute.Tensor, + stage_state: cute.Tensor, + eps: Float32, + m: Int32, + stream: cuda.CUstream, + ) -> None: + self.kernel( + contribution_mailbox, + residual_source, + post, + comb, + pre, + gamma, + residual_output, + norm_output, + stage_state, + eps, + ).launch( + grid=(m, self.cluster_size, 1), + block=(self.threads, 1, 1), + cluster=(1, self.cluster_size, 1), + smem=self.smem_size_in_bytes(), + stream=stream, + use_pdl=self.enable_pdl, + ) + + @cute.kernel + def kernel( + self, + contribution_mailbox: cute.Tensor, + residual_source: cute.Tensor, + post: cute.Tensor, + comb: cute.Tensor, + pre: cute.Tensor, + gamma: cute.Tensor, + residual_output: cute.Tensor, + norm_output: cute.Tensor, + stage_state: cute.Tensor, + eps: Float32, + ) -> None: + hc = self.hc + tidx, _, _ = cute.arch.thread_idx() + token, _, _ = cute.arch.block_idx() + cluster_rank = cute.arch.make_warp_uniform(cute.arch.block_idx_in_cluster()) + rank_lane = tidx % self.rank_lanes + group = tidx // self.rank_lanes + base_fragment = cluster_rank * self.groups_per_cta + group + + prenorm_fragments = cute.make_rmem_tensor( + cute.make_layout( + (self.trips, VEC_BF16), + stride=(VEC_BF16, 1), + ), + BFloat16, + ) + prenorm_fragments.fill(BFloat16(0.0)) + gamma_fragments = cute.make_rmem_tensor( + cute.make_layout( + (self.trips, VEC_BF16), + stride=(VEC_BF16, 1), + ), + BFloat16, + ) + gamma_fragments.fill(BFloat16(0.0)) + residual_fragments = cute.make_rmem_tensor( + cute.make_layout( + (self.trips, hc, VEC_BF16), + stride=(hc * VEC_BF16, VEC_BF16, 1), + ), + BFloat16, + ) + residual_fragments.fill(BFloat16(0.0)) + + for trip in cutlass.range_constexpr(self.trips): + fragment = base_fragment + trip * self.fragment_stride + if fragment < self.fragments and rank_lane == 0: + gamma_element = Int64(fragment) * VEC_BF16 + gamma_pointer = cute.make_ptr( + BFloat16, + (gamma.iterator + gamma_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=16, + ) + gamma_fragments[trip, None].store( + packed_u32x4_to_bf16x8(load_global_u32x4(gamma_pointer)) + ) + for source in cutlass.range_constexpr(hc): + residual_element = ( + Int64(token) * hc + source + ) * self.hidden + gamma_element + residual_pointer = cute.make_ptr( + BFloat16, + (residual_source.iterator + residual_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=16, + ) + residual_fragments[trip, source, None].store( + packed_u32x4_to_bf16x8(load_global_u32x4(residual_pointer)) + ) + post_mix = cute.make_rmem_tensor(cute.make_layout((hc,)), Float32) + pre_mix = cute.make_rmem_tensor(cute.make_layout((hc,)), Float32) + comb_mix = cute.make_rmem_tensor( + cute.make_layout((hc, hc), stride=(hc, 1)), Float32 + ) + for target in cutlass.range_constexpr(hc): + mix_index = Int64(token) * hc + target + post_mix[target] = cute.arch.load( + (post.iterator + mix_index).llvm_ptr, Float32 + ) + pre_mix[target] = cute.arch.load( + (pre.iterator + mix_index).llvm_ptr, Float32 + ) + for source in cutlass.range_constexpr(hc): + comb_index = (Int64(token) * hc + source) * hc + target + comb_mix[source, target] = cute.arch.load( + (comb.iterator + comb_index).llvm_ptr, Float32 + ) + + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_wait() + + active_stage = load_volatile_u32(stage_state.iterator + ACTIVE_STAGE) + for trip in cutlass.range_constexpr(self.trips): + fragment = base_fragment + trip * self.fragment_stride + lane_packed = cute.make_rmem_tensor( + cute.make_layout((self.rank_waves, 4)), + Uint32, + ) + lane_packed.fill(Uint32(0)) + dirty = fragment < self.fragments + while dirty: + dirty = False + for wave in cutlass.range_constexpr(self.rank_waves): + source_rank = wave * self.rank_lanes + rank_lane + if fragment < self.fragments: + source_element = ( + (Int64(active_stage) * self.tp + Int64(source_rank)) + * self.capacity_m + + Int64(token) + ) * self.hidden + Int64(fragment) * VEC_BF16 + source_pointer = cute.make_ptr( + BFloat16, + (contribution_mailbox.iterator + source_element).llvm_ptr, + cute.AddressSpace.gmem, + assumed_align=16, + ) + packed = load_global_u32x4( + source_pointer, + volatile=True, + ) + dirty = dirty | fragment_has_negative_zero(packed) + for word in cutlass.range_constexpr(4): + lane_packed[wave, word] = packed[word] + + lane_sum = cute.make_rmem_tensor(cute.make_layout((VEC_BF16,)), Float32) + lane_sum.fill(Float32(0.0)) + for wave in cutlass.range_constexpr(self.rank_waves): + packed = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32) + for word in cutlass.range_constexpr(4): + packed[word] = lane_packed[wave, word] + lane_sum.store( + lane_sum.load() + packed_u32x4_to_bf16x8(packed.load()).to(Float32) + ) + for offset in cutlass.range_constexpr(1, 5): + if cutlass.const_expr(offset < self.rank_lanes and offset in (1, 2, 4)): + for element in cutlass.range_constexpr(VEC_BF16): + lane_sum[element] = lane_sum[ + element + ] + cute.arch.shuffle_sync_bfly( + lane_sum[element], + offset=offset, + mask=-1, + mask_and_clamp=31, + ) + + if fragment < self.fragments and rank_lane == 0: + # The all-reduced sublayer output is BF16, as the unfused + # collective returns it, before mHC post mixes it in. + reduced = lane_sum.load().to(BFloat16).to(Float32) + collapse = cute.make_rmem_tensor(cute.make_layout((VEC_BF16,)), Float32) + collapse.fill(Float32(0.0)) + for target in cutlass.range_constexpr(hc): + mixed = reduced * post_mix[target] + for source in cutlass.range_constexpr(hc): + mixed = ( + mixed + + residual_fragments[trip, source, None].load().to(Float32) + * comb_mix[source, target] + ) + mixed_bf16 = mixed.to(BFloat16) + output_element = (Int64(token) * hc + target) * self.hidden + Int64( + fragment + ) * VEC_BF16 + store_global_u32x4( + Int64((residual_output.iterator + output_element).toint()), + bf16x8_to_packed_u32x4(mixed_bf16), + ) + collapse.store( + collapse.load() + mixed_bf16.to(Float32) * pre_mix[target] + ) + prenorm_fragments[trip, None].store(collapse.load().to(BFloat16)) + + if token == 0 and cluster_rank == 0 and tidx == 0: + store_global_u32( + stage_state.iterator + NEXT_STAGE, + (active_stage + Uint32(1)) % Uint32(LAMPORT_GENERATIONS), + ) + if cutlass.const_expr(self.enable_pdl): + cute.arch.griddepcontrol_launch_dependents() + + for trip in cutlass.range_constexpr(self.trips): + fragment = base_fragment + trip * self.fragment_stride + for wave in cutlass.range_constexpr(self.rank_waves): + source_rank = wave * self.rank_lanes + rank_lane + if fragment < self.fragments: + source_element = ( + (Int64(active_stage) * self.tp + Int64(source_rank)) + * self.capacity_m + + Int64(token) + ) * self.hidden + Int64(fragment) * VEC_BF16 + store_lamport_sentinel_u32x4( + Int64((contribution_mailbox.iterator + source_element).toint()) + ) + + thread_sum = Float32(0.0) + for trip in cutlass.range_constexpr(self.trips): + fragment = base_fragment + trip * self.fragment_stride + if fragment < self.fragments and rank_lane == 0: + values = prenorm_fragments[trip, None].load().to(Float32) + thread_sum = thread_sum + (values * values).reduce( + cute.ReductionOp.ADD, + init_val=Float32(0.0), + reduction_profile=0, + ) + + smem = cutlass.utils.SmemAllocator() + warp_sums = smem.allocate_array(Float32, self.warps) + cluster_sums = smem.allocate_array(Float32, self.cluster_size) + cta_sum = _group_leader_block_sum( + thread_sum, + warp_sums, + self.warps, + self.rank_lanes, + ) + if tidx < self.cluster_size: + local_slot = cluster_sums + cluster_rank + remote_slot = map_shared_to_peer(local_slot, Int32(tidx)) + store_shared_cluster_f32(remote_slot, cta_sum) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + full_sum = Float32(0.0) + for peer in cutlass.range_constexpr(self.cluster_size): + full_sum = full_sum + cute.arch.load( + (cluster_sums + peer).llvm_ptr, + Float32, + ) + inv_rms = cute.math.rsqrt( + full_sum / Float32(self.hidden) + eps, + fastmath=True, + ) + for trip in cutlass.range_constexpr(self.trips): + fragment = base_fragment + trip * self.fragment_stride + if fragment < self.fragments and rank_lane == 0: + gamma_values = gamma_fragments[trip, None].load().to(Float32) + result = ( + prenorm_fragments[trip, None].load().to(Float32) + * inv_rms + * gamma_values + ).to(BFloat16) + output_element = Int64(token) * self.hidden + Int64(fragment) * VEC_BF16 + store_global_u32x4( + Int64((norm_output.iterator + output_element).toint()), + bf16x8_to_packed_u32x4(result), + ) + + +class AllReduceMHC: + """Lamport TP all-reduce fused with mHC post, collapse and RMSNorm. + + ``__call__`` reduces ``x`` across the TP group, writes the post-mixed hc + streams and the normalized collapse, and leaves the next mixes to the + caller. ``finalize`` does the same for an unfinalized MoE output, folding + the top-k reduction and the shared-expert add into the publish. Each + instance owns a Lamport mailbox; construction is collective over the TP + group. + """ + + def __init__( + self, + *, + hidden_size: int, + hc_mult: int, + max_num_tokens: int, + top_k: int, + device: torch.device, + ) -> None: + group = get_tp_group().device_group + self.hidden_size = hidden = hidden_size + self.hc_mult = hc = hc_mult + self.capacity = capacity = max_num_tokens + self.top_k = top_k + tp = dist.get_world_size(group) + rank = dist.get_rank(group) + device = torch.device(device) + + # FlashInfer's LL_ALL_REDUCE_GB300_TP{4,8}_H5120 and + # LL_FINALIZE_..._H5120_K* presets: a cluster of 5 x 128 threads covers + # a 640-fragment token in one fully used trip, and the finalize stages + # all top_k rows at once. + with torch.accelerator.device_index(device.index): + self._publish = cute.compile( + _SharedOnlyPublishDeviceKernel( + hidden=hidden, + tp=tp, + rank=rank, + capacity_m=capacity, + elements_per_thread=VEC_BF16, + threads=128, + release_before_store=False, + enable_pdl=True, + ), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hidden + ), + make_fake_compact_tensor(Int32, (2,), assumed_align=4), + Int64(0), + Int32(capacity), + current_cu_stream(), + ) + self._finalize_publish = cute.compile( + _QuadFinalizePublishDeviceKernel( + hidden=hidden, + top_k=top_k, + tp=tp, + rank=rank, + capacity_m=capacity, + threads=128, + # DSV4.1's router folds the scale into the weights. + routed_scaling_factor=1.0, + include_shared_expert=True, + load_shared_expert_before_pdl=False, + enable_pdl=True, + prefetch_group=top_k, + fp32_weights=True, + ), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hidden + ), + make_fake_dynamic_compact_tensor( + Float32, alignment=4, divisibility=top_k + ), + make_fake_dynamic_compact_tensor( + Int32, alignment=4, divisibility=top_k + ), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hidden + ), + make_fake_compact_tensor(Int32, (2,), assumed_align=4), + Int64(0), + Int32(capacity), + current_cu_stream(), + ) + self._collective = cute.compile( + _LamportMHCDeviceKernel( + hidden=hidden, + hc=hc, + tp=tp, + capacity_m=capacity, + cluster_size=5, + rank_lanes=1, + threads=128, + enable_pdl=True, + ), + make_fake_compact_tensor( + BFloat16, + (LAMPORT_GENERATIONS * tp * capacity * hidden,), + assumed_align=16, + ), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hc * hidden + ), + make_fake_dynamic_compact_tensor(Float32, alignment=4, divisibility=hc), + make_fake_dynamic_compact_tensor( + Float32, alignment=4, divisibility=hc * hc + ), + make_fake_dynamic_compact_tensor(Float32, alignment=4, divisibility=hc), + make_fake_compact_tensor(BFloat16, (hidden,), assumed_align=16), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hc * hidden + ), + make_fake_dynamic_compact_tensor( + BFloat16, alignment=16, divisibility=hidden + ), + make_fake_compact_tensor(Int32, (2,), assumed_align=4), + Float32(0.0), + Int32(capacity), + current_cu_stream(), + ) + + self._mailbox = symm_mem.empty( + (LAMPORT_GENERATIONS, tp, capacity, hidden), + dtype=torch.bfloat16, + device=device, + ) + self._mailbox_handle = symm_mem.rendezvous(self._mailbox, group) + multicast = int(self._mailbox_handle.multicast_ptr or 0) + if not multicast or multicast % 16: + raise RuntimeError("NVLink multicast mapping is unavailable") + self._multicast = multicast + # Every slot starts as the Lamport sentinel, BF16 negative zero. + self._mailbox.view(torch.int16).fill_(-32768) + self._stage_state = torch.zeros(2, dtype=torch.int32, device=device) + torch.accelerator.synchronize() + dist.barrier(group=group) + + def __call__( + self, + x: torch.Tensor, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + pre: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Return (post-mixed residual streams, normalized layer input). + + Args: + x: [M, hidden] BF16 TP-partial sublayer output. + residual: [M, hc, hidden] BF16 residual streams. + post: [M, hc(, 1)] FP32 post-mix. + comb: [M, hc, hc] FP32 residual mix, indexed [source, target]. + pre: [M, hc] FP32 carried pre-mix for the collapse. + norm_weight: [hidden] BF16 RMSNorm weight. + norm_eps: RMSNorm epsilon. + + """ + m = x.shape[0] + self._check(m, residual, post, comb, pre) + if x.shape != (m, self.hidden_size): + raise ValueError("unexpected x shape") + self._publish( + to_cute_dynamic(x.flatten(), 16, divisibility=self.hidden_size), + to_cute(self._stage_state, 4), + Int64(self._multicast), + Int32(m), + current_cu_stream(), + ) + return self._collect(residual, post, comb, pre, norm_weight, norm_eps, m) + + def finalize( + self, + gemm2_permuted: torch.Tensor, + expert_weights: torch.Tensor, + expanded_idx_to_permuted_idx: torch.Tensor, + shared_output: torch.Tensor, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + pre: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + """``__call__`` for a MoE output left unfinalized. + + Args: + gemm2_permuted: [rows, hidden] BF16 unweighted GEMM2 output in the + MoE's permuted order. + expert_weights: [M, top_k] FP32 routing weights, already scaled. + expanded_idx_to_permuted_idx: [M, top_k] int32 row of each route, + -1 for an expert this rank does not hold. + shared_output: [M, hidden] BF16 TP-partial shared-expert output. + residual: [M, hc, hidden] BF16 residual streams. + post: [M, hc(, 1)] FP32 post-mix. + comb: [M, hc, hc] FP32 residual mix, indexed [source, target]. + pre: [M, hc] FP32 carried pre-mix for the collapse. + norm_weight: [hidden] BF16 RMSNorm weight. + norm_eps: RMSNorm epsilon. + + """ + m = shared_output.shape[0] + hidden, top_k = self.hidden_size, self.top_k + self._check(m, residual, post, comb, pre) + if gemm2_permuted.dim() != 2 or gemm2_permuted.shape[1] != hidden: + raise ValueError("gemm2_permuted must be [rows, hidden]") + if shared_output.shape != (m, hidden): + raise ValueError("unexpected shared_output shape") + if expert_weights.dtype != torch.float32 or expert_weights.numel() != m * top_k: + raise ValueError("expert_weights must be FP32 [M, top_k]") + if ( + expanded_idx_to_permuted_idx.dtype != torch.int32 + or expanded_idx_to_permuted_idx.numel() != m * top_k + ): + raise ValueError("expanded_idx_to_permuted_idx must be int32 [M, top_k]") + self._finalize_publish( + to_cute_dynamic(gemm2_permuted.flatten(), 16, divisibility=hidden), + to_cute_dynamic(expert_weights.flatten(), 4, divisibility=top_k), + to_cute_dynamic( + expanded_idx_to_permuted_idx.flatten(), 4, divisibility=top_k + ), + to_cute_dynamic(shared_output.flatten(), 16, divisibility=hidden), + to_cute(self._stage_state, 4), + Int64(self._multicast), + Int32(m), + current_cu_stream(), + ) + return self._collect(residual, post, comb, pre, norm_weight, norm_eps, m) + + def _check( + self, + m: int, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + pre: torch.Tensor, + ) -> None: + hc = self.hc_mult + if not 1 <= m <= self.capacity: + raise ValueError(f"M={m} is outside [1, {self.capacity}]") + if residual.shape != (m, hc, self.hidden_size): + raise ValueError("unexpected residual shape") + if ( + post.numel() != m * hc + or pre.numel() != m * hc + or comb.numel() != m * hc * hc + ): + raise ValueError("unexpected mix shapes") + + def _collect( + self, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + pre: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + m: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + hidden, hc = self.hidden_size, self.hc_mult + residual_output = torch.empty_like(residual) + layer_input = torch.empty( + (m, hidden), dtype=torch.bfloat16, device=residual.device + ) + self._collective( + to_cute(self._mailbox.flatten(), 16), + to_cute_dynamic(residual.flatten(), 16, divisibility=hc * hidden), + to_cute_dynamic(post.flatten(), 4, divisibility=hc), + to_cute_dynamic(comb.flatten(), 4, divisibility=hc * hc), + to_cute_dynamic(pre.flatten(), 4, divisibility=hc), + to_cute(norm_weight, 16), + to_cute_dynamic(residual_output.flatten(), 16, divisibility=hc * hidden), + to_cute_dynamic(layer_input.flatten(), 16, divisibility=hidden), + to_cute(self._stage_state, 4), + Float32(norm_eps), + Int32(m), + current_cu_stream(), + ) + return residual_output, layer_input diff --git a/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/primitives.py b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/primitives.py new file mode 100644 index 0000000000..16d5f4721a --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/cute_dsl/primitives.py @@ -0,0 +1,396 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from FlashInfer's flashinfer/comm/mnnvl_cutedsl/cute_dsl_primitives.py +# and runtime.py (flashinfer-ai/flashinfer@139af6f6). + +"""PTX building blocks and launch helpers for the DSV4.1 LL all-reduce.""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass import Float32, Int32, Int64, Uint16, Uint32 +from cutlass._mlir import ir +from cutlass._mlir.dialects import llvm, vector +from cutlass.cute.runtime import from_dlpack, make_fake_compact_tensor +from cutlass.cutlass_dsl import T, dsl_user_op + +WARP_SIZE = 32 +VEC_BF16 = 8 +QUAD_BF16 = 4 +NEGATIVE_ZERO_BF16_BITS = 0x8000 +NEGATIVE_ZERO_BF16_PAIR = 0x80008000 + + +def _asm(result, operands, text, constraints, *, side_effects, loc, ip): + return llvm.inline_asm( + result, + operands, + text, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def load_global_u32x4( + pointer: cute.Pointer, + *, + volatile: cutlass.Constexpr[bool] = False, + loc=None, + ip=None, +): + address = pointer.toint(loc=loc, ip=ip) + opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32" + loaded = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [address.ir_value(loc=loc, ip=ip)], + f"{opcode} {{$0, $1, $2, $3}}, [$4];", + "=r,=r,=r,=r,l", + side_effects=volatile, + loc=loc, + ip=ip, + ) + packed = vector.from_elements( + ir.VectorType.get([4], T.i32(), loc=loc), + [ + llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip) + for index in range(4) + ], + loc=loc, + ip=ip, + ) + return cute.TensorSSA(packed, 4, Uint32) + + +@dsl_user_op +def load_global_u32x2(pointer: cute.Pointer, *, loc=None, ip=None): + address = pointer.toint(loc=loc, ip=ip) + loaded = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [address.ir_value(loc=loc, ip=ip)], + "ld.global.v2.u32 {$0, $1}, [$2];", + "=r,=r,l", + side_effects=False, + loc=loc, + ip=ip, + ) + packed = vector.from_elements( + ir.VectorType.get([2], T.i32(), loc=loc), + [ + llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip) + for index in range(2) + ], + loc=loc, + ip=ip, + ) + return cute.TensorSSA(packed, 2, Uint32) + + +@dsl_user_op +def load_global_bf16_as_f32(address: Int64, *, loc=None, ip=None) -> Float32: + return Float32( + _asm( + T.f32(), + [address.ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b16 bits;\n\tld.global.b16 bits, [$1];\n\t" + "cvt.f32.bf16 $0, bits;\n\t}", + "=f,l", + side_effects=False, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def store_global_u32x4(address: Int64, packed, *, loc=None, ip=None) -> None: + words = [packed[index].ir_value(loc=loc, ip=ip) for index in range(4)] + _asm( + None, + [address.ir_value(loc=loc, ip=ip), *words], + "st.global.v4.u32 [$0], {$1, $2, $3, $4};", + "l,r,r,r,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def store_lamport_sentinel_u32x4(address: Int64, *, loc=None, ip=None) -> None: + sentinel = Uint32(NEGATIVE_ZERO_BF16_PAIR).ir_value(loc=loc, ip=ip) + _asm( + None, + [address.ir_value(loc=loc, ip=ip), sentinel, sentinel, sentinel, sentinel], + "st.global.v4.u32 [$0], {$1, $2, $3, $4};", + "l,r,r,r,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def load_volatile_u32(pointer: cute.Pointer, *, loc=None, ip=None) -> Uint32: + address = pointer.toint(loc=loc, ip=ip) + return Uint32( + _asm( + T.i32(), + [address.ir_value(loc=loc, ip=ip)], + "ld.volatile.global.u32 $0, [$1];", + "=r,l", + side_effects=True, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def store_global_u32( + pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None +) -> None: + address = pointer.toint(loc=loc, ip=ip) + _asm( + None, + [address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)], + "st.global.u32 [$0], $1;", + "l,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def packed_u32x4_to_bf16x8(packed, *, loc=None, ip=None): + values = llvm.bitcast( + ir.VectorType.get([VEC_BF16], cutlass.BFloat16.mlir_type, loc=loc), + packed.ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + return cute.TensorSSA(values, VEC_BF16, cutlass.BFloat16) + + +@dsl_user_op +def packed_u32x2_to_bf16x4(packed, *, loc=None, ip=None): + values = llvm.bitcast( + ir.VectorType.get([QUAD_BF16], cutlass.BFloat16.mlir_type, loc=loc), + packed.ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + return cute.TensorSSA(values, QUAD_BF16, cutlass.BFloat16) + + +@dsl_user_op +def bf16x4_to_packed_u32x2(values, *, loc=None, ip=None): + packed = llvm.bitcast( + ir.VectorType.get([2], T.i32(), loc=loc), + values.ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + return cute.TensorSSA(packed, 2, Uint32) + + +@dsl_user_op +def bf16x8_to_packed_u32x4(values, *, loc=None, ip=None): + packed = llvm.bitcast( + ir.VectorType.get([4], T.i32(), loc=loc), + values.ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + return cute.TensorSSA(packed, 4, Uint32) + + +@cute.jit +def sanitize_negative_zero_u32(word: Uint32) -> Uint32: + low = Uint16(word & Uint32(0xFFFF)) + high = Uint16(word >> Uint32(16)) + if low == Uint16(NEGATIVE_ZERO_BF16_BITS): + word = word & Uint32(0xFFFF0000) + if high == Uint16(NEGATIVE_ZERO_BF16_BITS): + word = word & Uint32(0x0000FFFF) + return word + + +@cute.jit +def sanitize_negative_zero_u32x4(packed): + sanitized = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32) + for index in cutlass.range_constexpr(4): + sanitized[index] = sanitize_negative_zero_u32(packed[index]) + return sanitized.load() + + +@cute.jit +def sanitize_negative_zero_u32x2(packed): + sanitized = cute.make_rmem_tensor(cute.make_layout((2,)), Uint32) + for index in cutlass.range_constexpr(2): + sanitized[index] = sanitize_negative_zero_u32(packed[index]) + return sanitized.load() + + +@cute.jit +def fragment_has_negative_zero(packed): + dirty = False + for index in cutlass.range_constexpr(4): + word = packed[index] + dirty = ( + dirty + | (Uint16(word & Uint32(0xFFFF)) == Uint16(NEGATIVE_ZERO_BF16_BITS)) + | (Uint16(word >> Uint32(16)) == Uint16(NEGATIVE_ZERO_BF16_BITS)) + ) + return dirty + + +@dsl_user_op +def map_shared_to_peer( + smem_pointer: cute.Pointer, peer_rank: Int32, *, loc=None, ip=None +) -> Int32: + address = smem_pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip) + return Int32( + _asm( + T.i32(), + [address, peer_rank.ir_value(loc=loc, ip=ip)], + "mapa.shared::cluster.u32 $0, $1, $2;", + "=r,r,r", + side_effects=False, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def store_shared_cluster_f32( + remote_address: Int32, value: Float32, *, loc=None, ip=None +) -> None: + _asm( + None, + [remote_address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)], + "st.shared::cluster.f32 [$0], $1;", + "r,f", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def f32_to_bf16_bits(value: Float32, *, loc=None, ip=None) -> Uint32: + return Uint32( + _asm( + T.i32(), + [value.ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b16 bits;\n\tcvt.rn.bf16.f32 bits, $1;\n\t" + "cvt.u32.u16 $0, bits;\n\t}", + "=r,f", + side_effects=False, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def shuffle_sync_idx_u32( + value: Uint32, source_lane: Int32, *, loc=None, ip=None +) -> Uint32: + return Uint32( + _asm( + T.i32(), + [value.ir_value(loc=loc, ip=ip), source_lane.ir_value(loc=loc, ip=ip)], + "shfl.sync.idx.b32 $0, $1, $2, 0x1f, 0xffffffff;", + "=r,r,r", + # Preserve full-warp execution across later divergent consumers. + side_effects=True, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def stmc_bf16x2(address: Int64, packed: Uint32, *, loc=None, ip=None) -> None: + _asm( + None, + [address.ir_value(loc=loc, ip=ip), packed.ir_value(loc=loc, ip=ip)], + "multimem.st.relaxed.sys.global.bf16x2 [$0], $1;", + "l,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def stmc_bf16x4(address: Int64, values, *, loc=None, ip=None) -> None: + words = [values[index].ir_value(loc=loc, ip=ip) for index in range(2)] + _asm( + None, + [address.ir_value(loc=loc, ip=ip), *words], + "multimem.st.relaxed.sys.global.v2.bf16x2 [$0], {$1, $2};", + "l,r,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def stmc_bf16x8(address: Int64, values, *, loc=None, ip=None) -> None: + words = [values[index].ir_value(loc=loc, ip=ip) for index in range(4)] + _asm( + None, + [address.ir_value(loc=loc, ip=ip), *words], + "multimem.st.relaxed.sys.global.v4.bf16x2 [$0], {$1, $2, $3, $4};", + "l,r,r,r,r", + side_effects=True, + loc=loc, + ip=ip, + ) + + +class _GraphSafeDLPack: + __slots__ = ("tensor",) + + def __init__(self, tensor: torch.Tensor) -> None: + self.tensor = tensor + + def __dlpack__(self, stream=None): + # stream=-1 skips producer sync; CuTe launches on the current stream, + # which is the capturing one under CUDA graphs. + return self.tensor.__dlpack__(stream=-1) + + def __dlpack_device__(self): + return self.tensor.__dlpack_device__() + + +def to_cute(tensor: torch.Tensor, alignment: int) -> cute.Tensor: + return from_dlpack(_GraphSafeDLPack(tensor.detach()), assumed_align=alignment) + + +def to_cute_dynamic( + tensor: torch.Tensor, alignment: int, *, divisibility: int +) -> cute.Tensor: + return to_cute(tensor, alignment).mark_compact_shape_dynamic( + mode=0, divisibility=divisibility + ) + + +def make_fake_dynamic_compact_tensor(dtype, *, alignment: int, divisibility: int): + return make_fake_compact_tensor( + dtype, (cute.sym_int32(divisibility=divisibility),), assumed_align=alignment + ) + + +def current_cu_stream() -> cuda.CUstream: + return cuda.CUstream(torch.cuda.current_stream().cuda_stream) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py new file mode 100644 index 0000000000..b195d4b849 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization.""" + +from dataclasses import dataclass +from typing import Any + +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +import torch +from cuda.bindings.driver import CUstream +from cutlass import ( + BFloat16, + Float8E4M3FN, + Float32, + Int32, + Int64, + Uint8, + Uint32, + Uint64, +) +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import make_fake_tensor +from cutlass.cutlass_dsl import dsl_user_op + +from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy +from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, +) + +# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last +# _ROPE_DIM columns carry RoPE. +_HEAD_DIM = 512 +_ROPE_DIM = 64 + + +@dsl_user_op +def _ue8m0_rp(value, *, loc=None, ip=None): + result = llvm.inline_asm( + Uint32.mlir_type, + [value.ir_value(loc=loc, ip=ip)], + "{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; " + "cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }", + "=r,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Uint32(result) + + +@dsl_user_op +def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None): + """Preserve multiply/FMA rounding when RoPE loads are hoisted.""" + result = llvm.inline_asm( + Float32.mlir_type, + [v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)], + "{ .reg .f32 product; mul.rn.f32 product, $3, $4; " + + ( + "neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }" + if subtract + else "fma.rn.f32 $0, $1, $2, product; }" + ), + "=f,f,f,f,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Float32(result) + + +@cute.jit +def _scale(amax: Float32): + exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0)) + inv = recast_val((Uint32(254) - exponent) << 23, Float32) + if exponent == 0: + inv = Float32(0) + return exponent, inv + + +class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + # T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank). + tokens_per_tile: int + token_tiles: int + full_tiles: bool + n_groups: int + heads_per_group: int + o_lora_rank: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + tokens = compile_key.tokens_per_tile + token_tiles = compile_key.token_tiles + full_tiles = compile_key.full_tiles + acc_cols = 1 << (tokens - 1).bit_length() + tile_m = max(8, acc_cols) + tile_n = 128 + tile_k = 128 + # tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction. + mma_k = 32 + # Each group's heads split K across one cluster, one head per CTA. + heads_per_group = compile_key.heads_per_group + tiles_per_group = compile_key.o_lora_rank // tile_n + n_tiles = compile_key.n_groups * tiles_per_group + # Each CTA keeps one whole head of K in SMEM, one TMA box per stage. + num_stages = _HEAD_DIM // tile_k + w_stage_bytes = tile_n * tile_k + nope_dim = _HEAD_DIM - _ROPE_DIM + threads_per_token = _HEAD_DIM // 4 + sf_col = max(16, tile_m) + tmem_cols = sf_col * 2 + threads = min(acc_cols * threads_per_token, 512) + tokens_per_iter = threads // threads_per_token + + @cute.kernel + def device_kernel( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cpasync.TmaInfo, + ws: cpasync.TmaInfo, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + ): + total_tokens = num_tokens + if cutlass.const_expr(full_tiles): + total_tokens = Int32(tokens * token_tiles) + tid, _, _ = cute.arch.thread_idx() + bid, token_tile, _ = cute.arch.block_idx() + token_start = Int32(0) + if cutlass.const_expr(token_tiles > 1): + token_start = token_tile * tokens + warp = cute.arch.make_warp_uniform(tid // 32) + lane = tid % 32 + split = bid % heads_per_group + tile = bid // heads_per_group + group = tile // tiles_per_group + smem = utils.SmemAllocator() + sW = smem.allocate_tensor( + Float8E4M3FN, + w.smem_layout.outer, + byte_alignment=128, + swizzle=w.smem_layout.inner, + ) + sX = smem.allocate_tensor( + Float8E4M3FN, + cute.make_layout( + (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k) + ), + byte_alignment=1024, + swizzle=cute.make_swizzle(3, 4, 3), + ) + # tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 + + # (r // 32) * 4, one 4-byte cell of MX-block scales per row. + sW_SF = smem.allocate_tensor( + Int32, + cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)), + 128, + ) + sW_SF_raw = smem.allocate_tensor( + Int32, cute.make_layout((tile_n, num_stages)), 128 + ) + sX_SF = smem.allocate_tensor( + Uint8, + cute.make_layout( + ((32, 4), tile_k // mma_k, num_stages), + stride=((16, 4), 1, tile_n * 4), + ), + 128, + ) + copy_fp8x4 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32 + ) + partial = smem.allocate_tensor( + Float32, + cute.make_layout( + (tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group) + ), + 128, + ) + mbar_loaded = smem.allocate_array(Int64, num_stages) + mbar_ws_loaded = smem.allocate_array(Int64, 1) + mbar_done = smem.allocate_array(Int64, 1) + mbar_reduced = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + if tid == 0: + for stage in cutlass.range_constexpr(num_stages): + # Wait for weight TMA and this stage's quantization threads. + cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4) + cute.arch.mbarrier_init(mbar_ws_loaded, 1) + cute.arch.mbarrier_init(mbar_done, 1) + cute.arch.mbarrier_init(mbar_reduced, 1) + cute.arch.mbarrier_init_fence() + # Local users of the barriers (warp 0's TMA below) need the init too. + cute.arch.sync_threads() + # Paired with cluster_wait() below: peers must see mbar_reduced + # initialized before their st.async; the split hides the barrier. + cute.arch.cluster_arrive_relaxed() + if warp == 0: + # Raw DeepGEMM MN-major scales; the MMA warp transposes them to + # the UTCCP order, which TMA cannot scatter at 4-byte granularity. + ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages)) + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4) + simple_tma_copy( + ws.atom, + ws_tiles[None, (tile % tiles_per_group, split, group)], + sW_SF_raw, + mbar_ws_loaded, + ) + tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k)) + for stage in cutlass.range_constexpr(num_stages): + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes) + simple_tma_copy( + w.atom, + tiles[None, (tile, split * num_stages + stage)], + sW[None, None, stage], + mbar_loaded + stage, + ) + # x and positions come from the PDL predecessor; everything above + # only touches weights. Padded sX rows only feed unread MMA columns. + cute.arch.griddepcontrol_wait() + # TMEM is allocated at runtime, and a PDL predecessor on this SM may + # hold it until it exits; allocating earlier can deadlock. + if warp == 0: + cute.arch.alloc_tmem(tmem_cols, taddr) + cute.arch.relinquish_tmem_alloc_permit() + copy_x = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64 + ) + n_iters = cute.ceil_div(tokens, tokens_per_iter) + qid = (tid + 32) % threads_per_token + k = qid * 4 + # Tail rows repeat the last input; their output stores are masked. + # Issue every token's x load before any math so their latencies overlap. + x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_src = cute.local_tile( + x[ + cute.min(token_start + token, total_tokens - 1), + group * heads_per_group + split, + None, + ], + (4,), + (qid,), + ) + cute.copy(copy_x, x_src, x_bf16[None, iteration]) + # Overlap the RoPE loads across tokens before consuming any of them. + cos_reg = cute.make_rmem_tensor((2, n_iters), Float32) + sin_reg = cute.make_rmem_tensor((2, n_iters), Float32) + if cutlass.const_expr(n_iters > 1): + copy_rope = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 + ) + if k >= nope_dim: + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if ( + cutlass.const_expr(tokens % tokens_per_iter == 0) + or token < tokens + ): + src_token = cute.min(token_start + token, total_tokens - 1) + pos = positions[src_token] + freq = (k - nope_dim) // 2 + rope_row = rope[pos, None] + cos_src = cute.local_tile(rope_row, (2,), (freq // 2,)) + cute.copy(copy_rope, cos_src, cos_reg[None, iteration]) + sin_tile = (freq + _ROPE_DIM // 2) // 2 + sin_src = cute.local_tile(rope_row, (2,), (sin_tile,)) + cute.copy(copy_rope, sin_src, sin_reg[None, iteration]) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_f32 = cute.make_rmem_tensor((4,), Float32) + x_f32.store(x_bf16[None, iteration].load().to(Float32)) + if k >= nope_dim: + # Inverse RoPE on the interleaved pairs (k, k+1) and + # (k+2, k+3); each rope row is cos || sin. + for pair in cutlass.range_constexpr(2): + if cutlass.const_expr(n_iters > 1): + cos = cos_reg[pair, iteration] + sin = sin_reg[pair, iteration] + else: + pos = positions[ + cute.min(token_start + token, total_tokens - 1) + ] + freq = (k - nope_dim) // 2 + pair + cos = Float32(rope[pos, freq]) + sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) + even, odd = x_f32[2 * pair], x_f32[2 * pair + 1] + x_f32[2 * pair] = _rope_fma(even, cos, odd, sin) + x_f32[2 * pair + 1] = _rope_fma( + odd, cos, even, sin, subtract=True + ) + amax = cute.arch.fmax( + cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])), + cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])), + ) + amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) + exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) + x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN) + x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN)) + cute.copy( + copy_fp8x4, + x_fp8, + cute.local_tile( + sX[token, None, k // tile_k], (4,), (qid % 32,) + ), + ) + if lane % 8 == 0: + sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent) + cute.arch.fence_proxy("async.shared", space="cta") + mbarrier.arrive(mbar_loaded + qid // 32, order="release") + if warp == 0: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + # sW / sX are 128B-swizzled K-major; one stage's K fills the + # swizzle atom, so the leading byte offset is unused. + sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0) + # Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix + # (16 B units at bit 32); bit 46 is the SM100 descriptor version. + sf_sbo = 8 * 16 + sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46)) + # M = tile_n weight rows (A), N = tile_m token rows (B). + idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m) + # sW_SF's layout is the UTCCP order, so a plain copy transposes; + # fence the generic-proxy writes before tcgen05.cp reads them. + cute.arch.mbarrier_wait(mbar_ws_loaded, 0) + for stage in cutlass.range_constexpr(num_stages): + for i in cutlass.range_constexpr(4): + sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage] + cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_warp() + for stage in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_wait(mbar_loaded + stage, 0) + _tcgen05.fence_after_thread_sync() + if cutlass.const_expr(stage == num_stages - 1): + with cute.arch.elect_one(): + cute.arch.griddepcontrol_launch_dependents() + adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4) + bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4) + _tcgen05.cp( + base + sf_col, + sfdesc | (sW_SF[None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + _tcgen05.cp( + base + sf_col + 4, + sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + # Advance mma_k FP8 bytes (16 B units) and pick MX block kk of + # each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29). + for kk in cutlass.range_constexpr(tile_k // mma_k): + _tcgen05.mma_mxfp8( + base, + adesc + kk * mma_k // 16, + bdesc + kk * mma_k // 16, + idesc + ((kk << 4) | (kk << 29)), + base + sf_col, + base + sf_col + 4, + stage > 0 or kk > 0, + ) + _tcgen05.commit(mbar_done) + # Only the MMA warp waits; barrier 2 releases the TMEM readers. + cute.arch.mbarrier_wait(mbar_done, 0) + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + # Tokens t with t % heads_per_group == split arrive from every peer + # (this CTA included), tile_n Float32 rows each. + if tid == 0: + owned = cute.ceil_div(tokens - split, heads_per_group) + mbarrier.arrive_expect_tx( + mbar_reduced, owned * tile_n * heads_per_group * 4 + ) + # Pairs with cluster_arrive_relaxed() above: every peer's + # mbar_reduced is initialized before the st.async below. + cute.arch.cluster_wait() + # quack's process-wide const_expr-if rewrite can leave these unbound + # after the dynamic path above; bind them so the regions join. + amax, exponent, inv = Float32(0), Uint32(0), Float32(0) + if tid < tile_n: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + _tcgen05.fence_after_thread_sync() + acc = cute.make_rmem_tensor(acc_cols, Float32) + if cutlass.const_expr(tokens == 1): + acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) + else: + acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols)) + _tcgen05.wait_ld() + for token in cutlass.range_constexpr(tokens): + ptr = cute.domain_offset( + (tid, token // heads_per_group, split), partial + ).iterator + cute.arch.store_async_dsmem( + ptr, + recast_val(acc[token], Int32), + mbar_reduced, + token % heads_per_group, + ) + if split < tokens: + # One warp waits for the peers' bytes; named barrier 1 (0 is + # sync_threads) releases the other tile_n epilogue threads. + if warp == 0: + cute.arch.mbarrier_wait(mbar_reduced, 0) + cute.arch.barrier(barrier_id=1, number_of_threads=tile_n) + for i in cutlass.range_constexpr( + cute.ceil_div(tokens, heads_per_group) + ): + token = split + i * heads_per_group + if cutlass.const_expr( + full_tiles + and ( + tokens <= heads_per_group + or tokens % heads_per_group == 0 + ) + ) or (token < tokens and token_start + token < total_tokens): + value = Float32(0) + for peer in cutlass.range_constexpr(heads_per_group): + value += partial[tid, i, peer] + value = Float32(BFloat16(value)) + amax = cute.arch.warp_redux_sync(value, "max", abs=True) + exponent, inv = _scale(amax) + q[token_start + token, tile * tile_n + tid] = Float8E4M3FN( + value * inv + ) + if lane == 0: + row = token_start + token + offset = row % 32 * 16 + row // 32 * 4 + warp + qs[tile * 512 + offset] = Uint8(exponent) + # Every scale tile owns its padding, with no second memset kernel. + for i in cutlass.range_constexpr(4): + offset = i * tile_n + tid + row = offset // 16 + (offset % 16 // 4) * 32 + if token_tile == 0 and split == 0 and row >= total_tokens: + qs[tile * 512 + offset] = Uint8(0) + # Each receiver drained all writes into its own inbox. Only local + # TMEM readers must finish before deallocation; no peer reads SMEM. + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + if warp == 0: + cute.arch.dealloc_tmem( + cute.make_ptr( + Float32, + cute.make_tensor(taddr, cute.make_layout(1))[0], + cute.AddressSpace.tmem, + ), + tmem_cols, + ) + + @cute.jit + def host_entrypoint( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cute.Tensor, + ws: cute.Tensor, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + stream: CUstream, + ): + layout = cute.make_composed_layout( + cute.make_swizzle(3, 4, 3), + 0, + cute.make_layout( + (tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes) + ), + ) + tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + w, + layout, + (tile_n, tile_k), + ) + ws_tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])), + cute.make_layout((tile_n, num_stages)), + (tile_n, num_stages), + ) + device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch( + grid=(n_tiles * heads_per_group, token_tiles, 1), + block=(threads, 1, 1), + cluster=(heads_per_group, 1, 1), + stream=stream, + use_pdl=True, + ) + + return host_entrypoint + + def dispatch( # type: ignore[override] + self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> CompileKey: + token_tiles = (tokens + 31) // 32 + # Host integer arithmetic: balance tiles in multiples of four tokens. + tile_tokens = ( + tokens + if token_tiles == 1 + else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4 + ) + return self.CompileKey( + tokens_per_tile=tile_tokens, + token_tiles=token_tiles, + full_tiles=tokens % tile_tokens == 0, + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def get_warmup_keys( + self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> list[CompileKey]: + # Trace all accepted token counts and deduplicate shared tile shapes. + return self._trace_dispatch(self.dispatch)( + tokens=WarmupIntRange(1, max_tokens + 1), + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: + tokens = ( + compile_key.tokens_per_tile * compile_key.token_tiles + if compile_key.full_tiles + else cute.sym_int() + ) + groups = compile_key.n_groups + rank = compile_key.o_lora_rank + n = groups * rank + k = compile_key.heads_per_group * _HEAD_DIM + return ( + make_fake_tensor( + BFloat16, + (tokens, groups * compile_key.heads_per_group, _HEAD_DIM), + (cute.sym_int(divisibility=8), _HEAD_DIM, 1), + assumed_align=16, + ), + make_fake_tensor(Int64, (tokens,), (1,)), + make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)), + make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16), + make_fake_tensor( + Int32, + (groups, rank, k // 128), + (rank * k // 128, 1, rank), + assumed_align=16, + ), + make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), + make_fake_tensor(Uint8, (128 * n // 32,), (1,)), + Int32(0), + ) + + @kernel_launcher + def __call__( + self, + *, + x: torch.Tensor, + positions: torch.Tensor, + rope: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]: + """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes.""" + tokens = x.shape[0] + groups, rank = weight_scale.shape[:2] + n = groups * rank + q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8) + launch_args = ( + x, + positions, + rope, + weight.view(n, -1), + weight_scale, + q, + scales, + tokens, + ) + compile_key = self.dispatch( + tokens=tokens, + n_groups=groups, + heads_per_group=x.shape[1] // groups, + o_lora_rank=rank, + ) + return compile_key, launch_args, (q, scales) + + +_FUSED_WO_A_KERNEL = FusedWoAKernel() diff --git a/vllm/models/deepseek_v41/nvidia/ops/mhc.py b/vllm/models/deepseek_v41/nvidia/ops/mhc.py index 88288ead4d..edfdbd1ee4 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/mhc.py +++ b/vllm/models/deepseek_v41/nvidia/ops/mhc.py @@ -9,10 +9,15 @@ import torch from vllm.config import VllmConfig from vllm.distributed import get_tp_group +from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( mhc_fused_post_pre_delayed_tilelang, mhc_post_tilelang, ) +from vllm.model_executor.layers.fused_moe.moe_output import ( + MoEOutput, + UnfinalizedMoEOutput, +) from vllm.platforms import current_platform from vllm.utils.deep_gemm import is_deep_gemm_supported @@ -24,9 +29,17 @@ from .mega_mhc import ( if TYPE_CHECKING: from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator + from .cute_dsl import AllReduceMHC + +logger = init_logger(__name__) + # GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress. MHC_OVERLAP_MAX_TOKENS = 16 +# The fused all-reduce + mHC kernel of the overlap path, built once by +# init_mhc_all_reduce. +_all_reduce_mhc: "AllReduceMHC | None" = None + def supports_mhc_overlap(vllm_config: VllmConfig) -> bool: """Check kernel requirements and safety of sharing the coefficient stream.""" @@ -53,10 +66,35 @@ def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool: or config.hc_mult != 4 ): return False + # The custom all-reduce's MNNVL buffers succeed exactly when NVLink + # multicast does, which the fused kernel's own mailbox needs too. comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr) +def init_mhc_all_reduce(vllm_config: VllmConfig) -> None: + """Build the fused all-reduce + mHC kernel the overlap path reduces with. + + Collective over the TP group, so every rank must call it, and only when + ``supports_mhc_all_reduce`` holds. + """ + global _all_reduce_mhc + from .cute_dsl import AllReduceMHC + + config = vllm_config.model_config.hf_config + _all_reduce_mhc = AllReduceMHC( + hidden_size=config.hidden_size, + hc_mult=config.hc_mult, + max_num_tokens=MHC_OVERLAP_MAX_TOKENS, + top_k=config.num_experts_per_tok, + device=current_platform.current_device(), + ) + logger.info_once( + "DSV4.1 mHC: CuTe DSL Lamport all-reduce fused with mHC for up to %d tokens.", + MHC_OVERLAP_MAX_TOKENS, + ) + + def mhc_pre_delayed_overlap( residual: torch.Tensor, fn: torch.Tensor, @@ -151,7 +189,7 @@ def mhc_pre_delayed_overlap( def mhc_shifted_post_pre( - x: torch.Tensor, + x: torch.Tensor | MoEOutput, residual: torch.Tensor, post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, @@ -175,39 +213,55 @@ def mhc_shifted_post_pre( ]: """Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang. + A MoE output left unfinalized is finalized inside the fused all-reduce. When stream is supplied, join it before consuming the returned coefficients. """ layer_input = None - if reduce_results: - tp = get_tp_group() + if isinstance(x, MoEOutput): + assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called" + assert pre_mix is not None and norm_weight is not None + routed = x.routed + assert isinstance(routed, UnfinalizedMoEOutput) + assert x.shared_output is not None + residual, layer_input = _all_reduce_mhc.finalize( + routed.gemm2_permuted, + routed.expert_weights, + routed.expanded_idx_to_permuted_idx, + x.shared_output, + residual, + post_layer_mix, + comb_res_mix, + pre_mix, + norm_weight, + norm_eps, + ) + if stream is None: + # Outside FULL graphs, compute the next mixes inline. + stream = torch.cuda.current_stream() + elif reduce_results: if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS: - comm = cast("CudaCommunicator", tp.device_communicator).ca_comm - assert comm is not None and comm.mnnvl_lamport_epochs is not None - output = torch.empty_like(residual) - layer_input = torch.empty_like(x) - torch.ops._C_custom_ar.all_reduce_mhc( + assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called" + assert pre_mix is not None and norm_weight is not None + residual, layer_input = _all_reduce_mhc( x, residual, post_layer_mix, comb_res_mix, pre_mix, norm_weight, - output, - layer_input, - comm.mnnvl_lamport_ag_local_ptr, - comm.mnnvl_lamport_ag_multicast_ptr, - comm.mnnvl_lamport_epochs[0], - comm.rank, - comm.mnnvl_buffer_size, norm_eps, ) - residual = output else: - x = tp.all_reduce(x) + x = get_tp_group().all_reduce(x) if stream is not None: if layer_input is None: + assert isinstance(x, torch.Tensor) residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix) - aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1]) + aux = ( + residual.mean(dim=1) + if capture_aux + else residual.new_empty(0, residual.shape[-1]) + ) pre_outputs = mhc_pre_delayed_overlap( residual, fn, @@ -226,6 +280,7 @@ def mhc_shifted_post_pre( ) return residual, *pre_outputs, aux + assert isinstance(x, torch.Tensor) if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux): assert pre_mix is not None and norm_weight is not None outputs = mhc_shifted_post_pre_deep_gemm( diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py new file mode 100644 index 0000000000..4a315ddc9b --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 output projection with a small-batch SM100/SM103 fusion.""" + +import torch +from torch import nn + +from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( + FlashInferCutedslMxfp8LinearKernel, +) +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj +from vllm.platforms import current_platform + + +def _fused_wo_a_max_tokens(layer: nn.Module) -> int: + max_tokens = getattr(layer, "_fused_wo_a_token_limit", None) + if max_tokens is None: + # Additional token tiles only pay off for smaller grids. + ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) + max_tokens = 96 if ctas_per_tile <= 128 else 32 + layer._fused_wo_a_token_limit = max_tokens + return max_tokens + + +def _can_fuse_wo_a(layer: nn.Module) -> bool: + """Whether the attention layer matches the fused WO-A kernel's layout.""" + method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method) + return ( + current_platform.is_device_capability_family(100) + # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly. + and layer._einsum_recipe == (1, 1, 32) + and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic + and isinstance( + getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel + ) + # A group's heads form one portable (<= 8 CTA) cluster. + and layer.n_local_heads // layer.n_local_groups <= 8 + and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) + and layer.o_lora_rank % 128 == 0 + ) + + +def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: + """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" + # Draft models load outside the warmup registry; their layers share the + # target's attention shapes, so the target's registration covers them too. + if _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + _FUSED_WO_A_KERNEL.register_warmup( + max_tokens=_fused_wo_a_max_tokens(layer), + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + o_lora_rank=layer.o_lora_rank, + ) + + +def dsv41_o_proj( + layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor +) -> torch.Tensor: + """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches.""" + o = attn_out[:, : layer.n_local_heads, :] + cos_sin_cache = layer.rotary_emb.cos_sin_cache + if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + q, scales = _FUSED_WO_A_KERNEL( + x=o, + positions=positions, + rope=cos_sin_cache, + weight=layer.wo_a.weight, + weight_scale=layer.wo_a.weight_scale, + ) + return layer._wo_b_proj( + QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic) + ) + return deep_gemm_fp8_o_proj( + o, + positions, + cos_sin_cache, + layer.wo_a, + layer._wo_b_proj, + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + nope_dim=layer.nope_head_dim, + rope_dim=layer.rope_head_dim, + o_lora_rank=layer.o_lora_rank, + einsum_recipe=layer._einsum_recipe, + tma_aligned_scales=layer._tma_aligned_scales, + ) diff --git a/vllm/models/deepseek_v41/quant_config.py b/vllm/models/deepseek_v41/quant_config.py index c8361b2da5..bed60ae758 100644 --- a/vllm/models/deepseek_v41/quant_config.py +++ b/vllm/models/deepseek_v41/quant_config.py @@ -139,15 +139,17 @@ class DeepseekV4FP8Config(Fp8Config): def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> QuantizationMethods | None: + # Quark checkpoints are always handled by QuarkConfig, which knows + # how to dispatch each per-layer scheme (MXFP4, MXFP8, etc.). + # ``from_config`` below rewrites the config into a single global FP8 + # scheme, which would discard per-layer specs — so never claim Quark. + if isinstance(hf_quant_cfg, dict) and ( + hf_quant_cfg.get("quant_method") == "quark" + ): + return None if not ( isinstance(hf_quant_cfg, dict) - and ( - hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") - or ( - hf_quant_cfg.get("quant_method") == "quark" - and cls._is_quark_mxfp4_ocp(hf_quant_cfg) - ) - ) + and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") ): return None model_type = getattr(hf_config, "model_type", None) diff --git a/vllm/models/deepseek_v41/sparse_mla.py b/vllm/models/deepseek_v41/sparse_mla.py index 8db6d4ed2b..1130eec090 100644 --- a/vllm/models/deepseek_v41/sparse_mla.py +++ b/vllm/models/deepseek_v41/sparse_mla.py @@ -86,7 +86,7 @@ class DeepseekV4SparseMLABackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64 if current_platform.is_device_capability_family(90) else 128] @staticmethod @@ -255,7 +255,7 @@ class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend): return "FLASHMLA_MEGA_ATTN_DSV41" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [128] @classmethod diff --git a/vllm/models/dots3_note/nvidia/attention.py b/vllm/models/dots3_note/nvidia/attention.py index 27928b6e6f..561a3bfd5c 100644 --- a/vllm/models/dots3_note/nvidia/attention.py +++ b/vllm/models/dots3_note/nvidia/attention.py @@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> Dots3NoteDecodeMetadata: - del max_seq_len, query_start_loc_device, num_decode_tokens + del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0]) return Dots3NoteDecodeMetadata( block_table=block_table_tensor, diff --git a/vllm/models/dots3_note/nvidia/vision.py b/vllm/models/dots3_note/nvidia/vision.py index 4aba14d3c6..774d888301 100644 --- a/vllm/models/dots3_note/nvidia/vision.py +++ b/vllm/models/dots3_note/nvidia/vision.py @@ -9,7 +9,7 @@ import torch import torch.nn.functional as F from torch import nn from torch.nn import LayerNorm -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.modeling_utils import PreTrainedModel from vllm.third_party.deep_gemm import per_block_cast_to_fp8 @@ -27,7 +27,7 @@ from .vision_attention import ( from .vision_moe import note_vision_fused_moe_fp8 -class DotsMoEVitConfig(PretrainedConfig): +class DotsMoEVitConfig(PreTrainedConfig): model_type: str = "dots_moe_vit" def __init__( diff --git a/vllm/models/glm5next/amd/ops/kpool_compress.py b/vllm/models/glm5next/amd/ops/kpool_compress.py index 97d8314c09..766950f332 100644 --- a/vllm/models/glm5next/amd/ops/kpool_compress.py +++ b/vllm/models/glm5next/amd/ops/kpool_compress.py @@ -327,37 +327,38 @@ def _kpool_tail_seed_kernel( KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, KPOOL: tl.constexpr, + RING: tl.constexpr, BLOCK_D: tl.constexpr, ): - """Copy token ``i``'s raw K + gate into its request's tail block. + """Copy token ``i``'s raw K + gate into its request's tail ring. Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL ahead belongs to a different tail block (or is past the batch / padding, - slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is - ``tail[block, {0:K, 1:score}, pos % KPOOL, :]``. + slot < 0). ``tslot = block * RING + pos % RING``; the destination is + ``tail[block, {0:K, 1:score}, pos % RING, :]``. """ i = tl.program_id(0) t = tl.load(tslot_ptr + i).to(tl.int64) if t < 0: return - blk = t // KPOOL # t >= 0 here, so trunc == floor + blk = t // RING # t >= 0 here, so trunc == floor ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to( tl.int64 ) # Match the torch semantics exactly: a negative ahead slot floors to a # block id that differs from every real block -> token is in the tail. # Only divide non-negative slots (Triton int div truncates, torch floors). - if ahead >= 0 and ahead // KPOOL == blk: + if ahead >= 0 and ahead // RING == blk: return offs = tl.arange(0, BLOCK_D) m = offs < HEAD_DIM block_base = blk * TAIL_BLOCK_ELEMS - base = block_base + (t % KPOOL) * HEAD_DIM + base = block_base + (t % RING) * HEAD_DIM k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m) s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m) tl.store(tail_ptr + base + offs, k, mask=m) tl.store( - tail_ptr + block_base + KPOOL_HEAD + (t % KPOOL) * HEAD_DIM + offs, s, mask=m + tail_ptr + block_base + KPOOL_HEAD + (t % RING) * HEAD_DIM + offs, s, mask=m ) @@ -385,6 +386,7 @@ def kpool_seed_tail_cache( KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, KPOOL=kpool, + RING=tail_kv_cache.shape[2], BLOCK_D=triton.next_power_of_2(head_dim), ) @@ -412,6 +414,7 @@ def _kpool_decode_update_batched_kernel( PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, + RING: tl.constexpr, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, @@ -428,10 +431,7 @@ def _kpool_decode_update_batched_kernel( the tail-ring slots that tokens t < t* (same request) just stashed in this same invocation. ``tl.range`` iterates sequentially within the program, so those stashes are visible to the later completion read. Cross-request - programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE - (the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one - completion can occur per request per call, but the ordered loop is correct - for any NEXT_N. + programs are independent (distinct tail blocks). RING >= POOL_SIZE. """ req = tl.program_id(0) offs = tl.arange(0, BLOCK_D) @@ -445,7 +445,7 @@ def _kpool_decode_update_batched_kernel( pos_valid = (cache_loc >= 0) & (pos >= 0) slot = safe_pos % POOL_SIZE - phys_slot = safe_pos % POOL_SIZE + phys_slot = safe_pos % RING # Derive the tail block from THIS token's tail_slot (the request's block # is constant across a pool, but a padded / invalid entry carries a @@ -453,7 +453,7 @@ def _kpool_decode_update_batched_kernel( # token's base address). Clamp so an invalid entry can never form an # out-of-bounds base; the accesses below are gated on pos_valid anyway. tail_slot = tl.load(tail_slot_mapping_ptr + idx) - block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE + block = tl.maximum(tail_slot, 0).to(tl.int64) // RING block_base = block * TAIL_BLOCK_ELEMS # The tail-ring stash must run for EVERY real token, so it is gated on @@ -481,7 +481,7 @@ def _kpool_decode_update_batched_kernel( max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -499,7 +499,7 @@ def _kpool_decode_update_batched_kernel( denom = tl.full((BLOCK_D,), 0.0, tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -609,7 +609,8 @@ def kpool_decode_update_and_maybe_write_cache_batched( return assert tail_kv_cache.ndim == 4 assert tail_kv_cache.shape[1] == 2 - assert tail_kv_cache.shape[2] == pool_size + ring = tail_kv_cache.shape[2] + assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size) assert tail_kv_cache.shape[3] == head_dim assert tail_kv_cache.dtype == torch.bfloat16 assert key.ndim == 3 and key.shape[2] == head_dim @@ -658,6 +659,7 @@ def kpool_decode_update_and_maybe_write_cache_batched( PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf.stride(0), POOL_SIZE=pool_size, + RING=ring, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 4f776e8a8e..25f51117a8 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -34,6 +34,7 @@ from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool from vllm.platforms import current_platform from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES +from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec logger = init_logger(__name__) @@ -160,7 +161,7 @@ class Glm5NextTailCache(DeepseekV32IndexerCache): """Paged circular buffer for the kpool indexer's in-progress (tail) pool. Holds the trailing incomplete pool's raw K + gate score: one block of - ``index_kpool`` slots per request, overwritten in place by ``pos % kpool`` + ``ring`` slots per request, overwritten in place by ``pos % ring`` as decode/spec-decode advances. Prefill seeds it (instead of discarding the tail raw K+gate); the connector transfers it across PD; decode reads it to compress the boundary pool correctly. ``KpoolTailSpec`` / @@ -190,13 +191,24 @@ class Glm5NextTailCache(DeepseekV32IndexerCache): def get_kv_cache_spec(self, vllm_config: VllmConfig): # The two head slots form [K, gate score] in the generic # [block, head, state, content] cache view. + # Drafts are stashed before acceptance. With a one-pool ring, the + # drafts behind a rejected pool-completing draft overwrite the keys + # read by its redo. + span = self._index_kpool + vllm_config.num_speculative_tokens + ring = self._index_kpool * next_power_of_2(cdiv(span, self._index_kpool)) + # ring must divide the attention block size (a multiple of 128). + assert self.cache_config.block_size % ring == 0, ( + f"Glm5NextTailCache: cache_config.block_size " + f"({self.cache_config.block_size}) must be a multiple of the " + f"tail ring ({ring})" + ) return KpoolTailSpec( - block_size=self._index_kpool, + block_size=ring, num_kv_heads=2, head_size=self.head_dim, head_size_v=0, dtype=torch.bfloat16, - sliding_window=self._index_kpool, + sliding_window=ring, ) def get_attn_backend(self): diff --git a/vllm/models/glm5next/common/mtp.py b/vllm/models/glm5next/common/mtp.py index 33a5402b8f..8361a053b8 100644 --- a/vllm/models/glm5next/common/mtp.py +++ b/vllm/models/glm5next/common/mtp.py @@ -42,7 +42,6 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): assert vllm_config.speculative_config is not None config = vllm_config.speculative_config.draft_model_config.hf_config self.config = config - quant_config = vllm_config.quant_config self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -66,7 +65,9 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): device=current_platform.device_type, ) self.shared_head = SharedHead( - config=config, prefix=prefix, quant_config=quant_config + config=config, + prefix=prefix, + defer_lm_head=True, ) # MTP layers sit past the base model's hidden layers; parse the index # from the prefix (e.g. "...layers.32") so the decoder builds an MLA diff --git a/vllm/models/glm5next/nvidia/ops/kpool_compress.py b/vllm/models/glm5next/nvidia/ops/kpool_compress.py index 416d3b5c48..db496d300b 100644 --- a/vllm/models/glm5next/nvidia/ops/kpool_compress.py +++ b/vllm/models/glm5next/nvidia/ops/kpool_compress.py @@ -387,14 +387,15 @@ def _kpool_tail_seed_kernel( KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, KPOOL: tl.constexpr, + RING: tl.constexpr, BLOCK_D: tl.constexpr, ): - """Copy token ``i``'s raw K + gate into its request's tail block. + """Copy token ``i``'s raw K + gate into its request's tail ring. Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL ahead belongs to a different tail block (or is past the batch / padding, - slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is - ``tail[block, {0:K, 1:score}, pos % KPOOL, :]``. + slot < 0). ``tslot = block * RING + pos % RING``; the destination is + ``tail[block, {0:K, 1:score}, pos % RING, :]``. The tail cache aliases the indexer cache with the indexer's (padded) block stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` / @@ -405,18 +406,18 @@ def _kpool_tail_seed_kernel( t = tl.load(tslot_ptr + i).to(tl.int64) if t < 0: return - blk = t // KPOOL # t >= 0 here, so trunc == floor + blk = t // RING # t >= 0 here, so trunc == floor ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to( tl.int64 ) # Match the torch semantics exactly: a negative ahead slot floors to a # block id that differs from every real block -> token is in the tail. # Only divide non-negative slots (Triton int div truncates, torch floors). - if ahead >= 0 and ahead // KPOOL == blk: + if ahead >= 0 and ahead // RING == blk: return offs = tl.arange(0, BLOCK_D) m = offs < HEAD_DIM - base = blk * TAIL_BLOCK_ELEMS + (t % KPOOL) * HEAD_DIM + base = blk * TAIL_BLOCK_ELEMS + (t % RING) * HEAD_DIM k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m) s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m) tl.store(tail_ptr + base + offs, k, mask=m) @@ -449,6 +450,7 @@ def kpool_seed_tail_cache( KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, KPOOL=kpool, + RING=tail_kv_cache.shape[2], BLOCK_D=triton.next_power_of_2(head_dim), ) @@ -476,6 +478,7 @@ def _kpool_decode_update_batched_kernel( PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, + RING: tl.constexpr, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, @@ -490,10 +493,7 @@ def _kpool_decode_update_batched_kernel( the tail-ring slots that tokens t < t* (same request) just stashed in this same invocation. ``tl.range`` iterates sequentially within the program, so those stashes are visible to the later completion read. Cross-request - programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE - (the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one - completion can occur per request per call, but the ordered loop is correct - for any NEXT_N. + programs are independent (distinct tail blocks). RING >= POOL_SIZE. """ req = tl.program_id(0) offs = tl.arange(0, BLOCK_D) @@ -507,7 +507,7 @@ def _kpool_decode_update_batched_kernel( pos_valid = (cache_loc >= 0) & (pos >= 0) slot = safe_pos % POOL_SIZE - phys_slot = safe_pos % POOL_SIZE + phys_slot = safe_pos % RING # Derive the tail block from THIS token's tail_slot (the request's block # is constant across a pool, but a padded / invalid entry carries a @@ -515,7 +515,7 @@ def _kpool_decode_update_batched_kernel( # token's base address). Clamp so an invalid entry can never form an # out-of-bounds base; the accesses below are gated on pos_valid anyway. tail_slot = tl.load(tail_slot_mapping_ptr + idx) - block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE + block = tl.maximum(tail_slot, 0).to(tl.int64) // RING block_base = block * TAIL_BLOCK_ELEMS # The tail-ring stash must run for EVERY real token, so it is gated on @@ -543,7 +543,7 @@ def _kpool_decode_update_batched_kernel( max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -561,7 +561,7 @@ def _kpool_decode_update_batched_kernel( denom = tl.full((BLOCK_D,), 0.0, tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -671,7 +671,8 @@ def kpool_decode_update_and_maybe_write_cache_batched( return assert tail_kv_cache.ndim == 4 assert tail_kv_cache.shape[1] == 2 - assert tail_kv_cache.shape[2] == pool_size + ring = tail_kv_cache.shape[2] + assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size) assert tail_kv_cache.shape[3] == head_dim assert tail_kv_cache.dtype == torch.bfloat16 assert key.ndim == 3 and key.shape[2] == head_dim @@ -717,6 +718,7 @@ def kpool_decode_update_and_maybe_write_cache_batched( PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf.stride(0), POOL_SIZE=pool_size, + RING=ring, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index c80e442015..00f51cc6a7 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -558,6 +558,7 @@ def sparse_attn_indexer_kpool( decode_metadata.schedule_metadata, max_model_len=max_pool_len, clean_logits=False, + indices=decode_metadata.indices, ) num_rows = logits.shape[0] # kpool: logits are pool-granular -> select topk_tokens//kpool pools, diff --git a/vllm/models/hy_v4/amd/model.py b/vllm/models/hy_v4/amd/model.py index 753ac92a38..772e4e5b74 100644 --- a/vllm/models/hy_v4/amd/model.py +++ b/vllm/models/hy_v4/amd/model.py @@ -30,7 +30,7 @@ from itertools import islice import regex as re import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -75,7 +75,7 @@ from ..nvidia.moe import HYV4FeedForward, HYV4MoEFused logger = init_logger(__name__) -def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig: +def _normalize_hyv4_config(config: PreTrainedConfig) -> PreTrainedConfig: """Populate the aliases consumed by the shared MoE implementation.""" config.router_scaling_factor = config.routed_scaling_factor config.num_experts = config.n_routed_experts @@ -95,7 +95,7 @@ class HYV4DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vllm_config: VllmConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, diff --git a/vllm/models/hy_v4/nvidia/attention.py b/vllm/models/hy_v4/nvidia/attention.py index 7f963cc707..0e7fb61173 100644 --- a/vllm/models/hy_v4/nvidia/attention.py +++ b/vllm/models/hy_v4/nvidia/attention.py @@ -18,7 +18,7 @@ from typing import cast import regex as re import torch from torch import nn -from transformers import DeepseekV2Config, DeepseekV3Config, PretrainedConfig +from transformers import DeepseekV2Config, DeepseekV3Config, PreTrainedConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config @@ -51,7 +51,7 @@ _SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention" _WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)") -def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]: +def compute_skip_topk_layers(config: PreTrainedConfig) -> set[int]: """Return the backbone layers that reuse a previous layer's top-k indices. A "shared" indexer layer performs sparse attention with the indices computed @@ -280,7 +280,7 @@ class HYV4MLAAttention(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, num_heads: int, qk_nope_head_dim: int, diff --git a/vllm/models/hy_v4/nvidia/hc.py b/vllm/models/hy_v4/nvidia/hc.py index 5c7fcc931e..617b361e75 100644 --- a/vllm/models/hy_v4/nvidia/hc.py +++ b/vllm/models/hy_v4/nvidia/hc.py @@ -17,7 +17,7 @@ requires restructuring the decoder-layer forward scheduling. import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.model_executor.layers.hpc import HpcIHCHead, HpcIHCPost, HpcIHCPre from vllm.model_executor.layers.linear import ReplicatedLinear @@ -40,7 +40,7 @@ class HYV4HCPreLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_dim: int, hc_mult: int = 4, magnitude: float = 2.0, @@ -163,7 +163,7 @@ class HYV4HCPostLayer(nn.Module): y[n, i, d] = post[n, i] * x[n, d] + residual[n, i, d] """ - def __init__(self, config: PretrainedConfig): + def __init__(self, config: PreTrainedConfig): super().__init__() self.config = config @@ -213,7 +213,7 @@ class HYV4HCHeadLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, hidden_size: int, hc_mult: int = 4, hc_eps: float = 1e-6, @@ -301,7 +301,7 @@ class HYV4HCLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_idx: int, init_std: float = 6e-3, base_noise_std: float = 0.0, diff --git a/vllm/models/hy_v4/nvidia/model.py b/vllm/models/hy_v4/nvidia/model.py index b9988da377..e9313989fa 100644 --- a/vllm/models/hy_v4/nvidia/model.py +++ b/vllm/models/hy_v4/nvidia/model.py @@ -30,7 +30,7 @@ from itertools import islice import regex as re import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, VllmConfig from vllm.distributed import ( @@ -74,7 +74,7 @@ from .moe import HYV4FeedForward, HYV4MoEFused logger = init_logger(__name__) -def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig: +def _normalize_hyv4_config(config: PreTrainedConfig) -> PreTrainedConfig: """Populate the aliases consumed by the shared MoE implementation.""" config.router_scaling_factor = config.routed_scaling_factor config.num_experts = config.n_routed_experts @@ -94,7 +94,7 @@ class HYV4DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, vllm_config: VllmConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, diff --git a/vllm/models/hy_v4/nvidia/moe.py b/vllm/models/hy_v4/nvidia/moe.py index be8e49f21e..4c5907f765 100644 --- a/vllm/models/hy_v4/nvidia/moe.py +++ b/vllm/models/hy_v4/nvidia/moe.py @@ -4,7 +4,7 @@ import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import get_current_vllm_config from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size @@ -87,7 +87,7 @@ class HYV4MoEFused(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", enable_eplb: bool = False, diff --git a/vllm/models/hy_v4/nvidia/mtp.py b/vllm/models/hy_v4/nvidia/mtp.py index 6c9ea11e81..8683777cf4 100644 --- a/vllm/models/hy_v4/nvidia/mtp.py +++ b/vllm/models/hy_v4/nvidia/mtp.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Iterable import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import ( @@ -59,7 +59,7 @@ _MTP_QUANT_EXCLUSION_ATTRS = ("ignored_layers", "exclude_modules") def _get_spec_layer_idx_from_weight_name( - config: PretrainedConfig, weight_name: str + config: PreTrainedConfig, weight_name: str ) -> int | None: """Return the MTP layer index a checkpoint weight belongs to, or None. @@ -147,7 +147,7 @@ def _prepare_mtp_fp8_expert_scale( def _create_mtp_quant_config( - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, backbone_quant_config: QuantizationConfig | None = None, ) -> QuantizationConfig | None: """Create the quantization config for the MTP layers. @@ -303,7 +303,7 @@ class HYV4SharedHead(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() @@ -329,7 +329,7 @@ class HYV4MultiTokenPredictorLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, vllm_config: VllmConfig, model_config: ModelConfig, diff --git a/vllm/models/inkling/configs.py b/vllm/models/inkling/configs.py index 2f8a247aec..58ed5ea00a 100644 --- a/vllm/models/inkling/configs.py +++ b/vllm/models/inkling/configs.py @@ -8,10 +8,10 @@ from dataclasses import dataclass, field from typing import Any, ClassVar, Literal, cast import torch -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class InklingModelConfig(PretrainedConfig): +class InklingModelConfig(PreTrainedConfig): model_type = "inkling_model" keys_to_ignore_at_inference = ["past_key_values"] @@ -215,7 +215,7 @@ class InklingModelConfig(PretrainedConfig): return TMLConvCacheParams(shape=shape, layers=self.conv_layer_ids, dtype=dtype) -class InklingAudioConfig(PretrainedConfig): +class InklingAudioConfig(PreTrainedConfig): model_type = "inkling_audio_model" def __init__( @@ -255,7 +255,7 @@ class InklingAudioConfig(PretrainedConfig): super().__init__(**kwargs) -class InklingVisionConfig(PretrainedConfig): +class InklingVisionConfig(PreTrainedConfig): model_type = "inkling_vision_model" def __init__( @@ -295,10 +295,10 @@ class InklingVisionConfig(PretrainedConfig): super().__init__(**kwargs) -class InklingMMConfig(PretrainedConfig): +class InklingMMConfig(PreTrainedConfig): model_type = "inkling_mm_model" keys_to_ignore_at_inference = ["past_key_values"] - sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = { + sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = { "text_config": InklingModelConfig, "audio_config": InklingAudioConfig, "vision_config": InklingVisionConfig, diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index ed631aea7e..d98d16521f 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -438,6 +438,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1) ) spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1] + uniform_sequence_length = m.uniform_spec_sequence_length + # Token padding can make the packed tensor larger than the + # request-local metadata represented by this length. + if ( + uniform_sequence_length is not None + and q_spec.shape[1] != m.num_spec_decodes * uniform_sequence_length + ): + uniform_sequence_length = None # Spec-only batches write directly into core_attn_out. spec_out = ( core_attn_out[:, : q_spec.shape[1]] @@ -457,6 +465,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): cu_seqlens=spec_cu_seqlens, ssm_state_indices=spec_state_indices_tensor, num_accepted_tokens=num_accepted_tokens, + uniform_sequence_length=uniform_sequence_length, out=spec_out, ) diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py index 77ed09d489..15f446e2eb 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py @@ -16,9 +16,10 @@ # - OOB-mask correctness fix: present (all tl.load use mask=..., other=0). # Validated on gfx950: no core-dump, gsm8k 94.1%. # -# AMD-specific deltas vs the NVIDIA copy: NONE yet (byte-identical). Keep in sync -# with the NVIDIA copy on FLA updates; any divergence should be an intentional, -# documented gfx950-specific change (a #869-style AMD-only fix). +# AMD-specific deltas vs the NVIDIA copy: +# - fused recurrent gated speculative-decode launch geometry is tuned for +# gfx950. +# Keep the copies in sync on FLA updates outside documented platform tuning. from .chunk import ( chunk_kda, diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py index db519fb6f0..9db2912458 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py @@ -164,15 +164,24 @@ def fused_recurrent_kda_fwd_kernel( APPLY_BETA_SIGMOID: tl.constexpr, HAS_DT_BIAS: tl.constexpr, USE_LOWER_BOUND: tl.constexpr, + IS_SINGLE_SEQUENCE: tl.constexpr, + SEQUENCE_LENGTH: tl.constexpr, num_stages: tl.constexpr, ): pid = tl.program_id(0) i_v = pid % tl.cdiv(V, BV) i_nh = pid // tl.cdiv(V, BV) - i_n, i_h = i_nh // H, i_nh % H - bos = tl.load(cu_seqlens + i_n).to(tl.int64) - eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64) - sequence_length = eos - bos + if IS_SINGLE_SEQUENCE: + i_n, i_h = 0, i_nh + else: + i_n, i_h = i_nh // H, i_nh % H + if SEQUENCE_LENGTH: + bos = i_n * SEQUENCE_LENGTH + sequence_length = SEQUENCE_LENGTH + else: + bos = tl.load(cu_seqlens + i_n).to(tl.int64) + eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64) + sequence_length = eos - bos if sequence_length == 0: return @@ -208,6 +217,14 @@ def fused_recurrent_kda_fwd_kernel( p_v = v + bos * stride_qkv_token + i_h * V + o_v p_g = g + bos * stride_g_token + i_h * K + o_k p_beta = beta + bos * stride_beta_token + i_h + if USE_GATE_IN_KERNEL: + b_a = exp(tl.load(A_log + i_h).to(tl.float32)) + if HAS_DT_BIAS: + b_bias = tl.load( + dt_bias + i_h * K + o_k, + mask=m_k, + other=0.0, + ).to(tl.float32) for i_t in tl.range(0, sequence_length, num_stages=num_stages): b_q = tl.load(p_q, mask=m_k, other=0.0, eviction_policy="evict_last").to( tl.float32 @@ -219,8 +236,8 @@ def fused_recurrent_kda_fwd_kernel( tl.float32 ) if USE_QK_L2NORM_IN_KERNEL: - b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6) - b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) + b_q *= tl.rsqrt(tl.sum(b_q * b_q) + 1e-6) + b_k *= tl.rsqrt(tl.sum(b_k * b_k) + 1e-6) b_q *= scale b_gate = tl.load( @@ -231,13 +248,7 @@ def fused_recurrent_kda_fwd_kernel( ).to(tl.float32) if USE_GATE_IN_KERNEL: if HAS_DT_BIAS: - b_bias = tl.load( - dt_bias + i_h * K + o_k, - mask=m_k, - other=0.0, - ).to(tl.float32) b_gate += b_bias - b_a = exp(tl.load(A_log + i_h).to(tl.float32)) if USE_LOWER_BOUND: b_gate = lower_bound * tl.sigmoid(b_a * b_gate) else: @@ -288,6 +299,60 @@ def fused_recurrent_kda_fwd_kernel( p_out += stride_out_token +def _get_static_sequence_length( + total_tokens: int, + head_sequences: int, + is_single_sequence: bool, + uniform_sequence_length: int | None, +) -> int: + if is_single_sequence: + return total_tokens + # MI355X performance gate: 192 head-sequences is 16 sequences at H=12. + # Static loops win through 16; dynamic loops win at 32 for short queries. + if ( + uniform_sequence_length is not None + and uniform_sequence_length > 1 + and head_sequences <= 192 + ): + return uniform_sequence_length + return 0 + + +def _select_kda_launch_config( + use_gate_in_kernel: bool, + is_spec_decoding: bool, + head_sequences: int, + static_sequence_length: int, + has_multi_token_sequence: bool, +) -> tuple[int, int, int]: + if static_sequence_length: + BV = 4 if head_sequences <= 24 else 8 + if static_sequence_length == 1: + num_stages = 2 + elif static_sequence_length <= 3: + num_stages = 8 if head_sequences <= 24 else 4 + elif static_sequence_length <= 6: + num_stages = 6 + elif static_sequence_length <= 10: + num_stages = 10 + else: + num_stages = 12 + return BV, 1, num_stages + + if use_gate_in_kernel and is_spec_decoding: + if head_sequences <= 24: + return 4, 1, 4 if has_multi_token_sequence else 2 + if has_multi_token_sequence and head_sequences <= 48: + return 8, 1, 4 + if has_multi_token_sequence: + return 8, 1, 2 + return 32, 4, 2 + + if use_gate_in_kernel: + return 32, 4, 2 + return 8, 1, 2 + + def fused_recurrent_kda_fwd( q: torch.Tensor, k: torch.Tensor, @@ -300,6 +365,7 @@ def fused_recurrent_kda_fwd( cu_seqlens: torch.Tensor | None = None, ssm_state_indices: torch.Tensor | None = None, num_accepted_tokens: torch.Tensor | None = None, + uniform_sequence_length: int | None = None, use_qk_l2norm_in_kernel: bool = True, A_log: torch.Tensor | None = None, dt_bias: torch.Tensor | None = None, @@ -336,6 +402,10 @@ def fused_recurrent_kda_fwd( assert num_accepted_tokens is None else: assert ssm_state_indices.stride(1) == 1 + if uniform_sequence_length is not None: + assert num_accepted_tokens is not None + assert uniform_sequence_length > 0 + assert N * uniform_sequence_length == T assert cu_seqlens.is_contiguous() if use_gate_in_kernel: assert A_log is not None and A_log.is_contiguous() @@ -344,8 +414,22 @@ def fused_recurrent_kda_fwd( if scale is None: scale = K**-0.5 - BV = 32 if use_gate_in_kernel else 8 - num_warps = 4 if use_gate_in_kernel else 1 + is_spec_decoding = num_accepted_tokens is not None + is_single_sequence = is_spec_decoding and N == 1 + head_sequences = H * N + static_sequence_length = _get_static_sequence_length( + total_tokens=T, + head_sequences=head_sequences, + is_single_sequence=is_single_sequence, + uniform_sequence_length=uniform_sequence_length, + ) + BV, num_warps, num_stages = _select_kda_launch_config( + use_gate_in_kernel=use_gate_in_kernel, + is_spec_decoding=is_spec_decoding, + head_sequences=head_sequences, + static_sequence_length=static_sequence_length, + has_multi_token_sequence=T > N, + ) grid = (cdiv(V, BV) * N * H,) fused_recurrent_kda_fwd_kernel[grid]( q=q, @@ -379,8 +463,10 @@ def fused_recurrent_kda_fwd( USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, USE_GATE_IN_KERNEL=use_gate_in_kernel, APPLY_BETA_SIGMOID=use_beta_sigmoid_in_kernel, + IS_SINGLE_SEQUENCE=is_single_sequence, + SEQUENCE_LENGTH=static_sequence_length, num_warps=num_warps, - num_stages=2, + num_stages=num_stages, ) return out, initial_state @@ -398,6 +484,7 @@ def fused_recurrent_kda( cu_seqlens: torch.Tensor, ssm_state_indices: torch.Tensor, num_accepted_tokens: torch.Tensor | None = None, + uniform_sequence_length: int | None = None, out: torch.Tensor | None = None, fuse_gate: bool | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -433,6 +520,7 @@ def fused_recurrent_kda( cu_seqlens=cu_seqlens, ssm_state_indices=ssm_state_indices, num_accepted_tokens=num_accepted_tokens, + uniform_sequence_length=uniform_sequence_length, use_qk_l2norm_in_kernel=True, A_log=A_log if fuse_gate else None, dt_bias=dt_bias if fuse_gate else None, diff --git a/vllm/models/kimi_k3/nvidia/dspark_mla.py b/vllm/models/kimi_k3/nvidia/dspark_mla.py index 87f9e94d9b..0baff884cc 100644 --- a/vllm/models/kimi_k3/nvidia/dspark_mla.py +++ b/vllm/models/kimi_k3/nvidia/dspark_mla.py @@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import ( ReplicatedLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead +from vllm.model_executor.models.qwen3_dspark import ( + DSparkConfidenceHead, + DSparkMarkovHead, +) from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, @@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module): self.config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(self.config, "enable_confidence_head", False): + with_markov = getattr(self.config, "confidence_head_with_markov", False) + input_dim = self.config.hidden_size + ( + self.config.markov_rank if with_markov else 0 + ) + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) self._max_num_context_tokens = ( vllm_config.scheduler_config.max_num_batched_tokens ) @@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module): assert vllm_config.speculative_config is not None self.draft_model_config = vllm_config.speculative_config.draft_model_config self.config = self.draft_model_config.hf_config + if getattr(self.config, "enable_confidence_head", False): + # The class mapper drops the head as training-only; keep it for drafts + # that actually built one, since adaptive verification reads it. + self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"confidence_head": "confidence_head"} + ) target_layer_num = vllm_config.model_config.get_total_num_hidden_layers() self.model = K3DSparkModel( vllm_config=vllm_config, @@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module): def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: return self.model.markov_head.bias(markov_embed, self.logits_processor) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" + assert self.model.confidence_head is not None + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) # read: 1. all weights. 2. context kv weights diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index d1c2b31921..2562171109 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d -from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionBackend, GDNAttentionMetadata, @@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata): class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): + # Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every + # builder, and KDA reads per-request offsets off device within a fixed k+1 window, + # so one k+1 graph replays any 1..k+1 mix. + _cudagraph_support = AttentionCGSupport.ALWAYS mamba_aligned_state_indices: torch.Tensor | None = None def __init__( @@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): query_start_loc_cpu.diff() > 0 ) spec_sequence_masks_cpu |= active_decode_mask_cpu + # Adaptive's cost-table profiler probes eager dummy rows longer than + # RecoverSSM's num_spec+1 activation capacity; route them out instead + # of tripping the capacity check below. + spec_sequence_masks_cpu &= ( + query_start_loc_cpu.diff() <= self.num_spec + 1 + ) # Native KDA can use its regular decode path when no draft token # was scheduled. RecoverSSM must preserve its extended conv window. if ( @@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend): @staticmethod def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]: return KimiK3KDAMetadataBuilder + + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # Pure spec-decode reads its plan from DEVICE offsets, so CPU/device + # mismatch is fine; the mixed branch still assumes CPU totals match. + return True diff --git a/vllm/models/kimi_k3/nvidia/latent_moe_runner.py b/vllm/models/kimi_k3/nvidia/latent_moe_runner.py index 8eedb5ba89..8678dcbfe0 100644 --- a/vllm/models/kimi_k3/nvidia/latent_moe_runner.py +++ b/vllm/models/kimi_k3/nvidia/latent_moe_runner.py @@ -115,10 +115,11 @@ class LatentMoERunner(MoERunner): experts_cls = getattr(self._quant_method, "experts_cls", None) # The kernel instance is built after weight loading, while the tail # must register its CuTeDSL warmup units during runner construction. - self.moe_config.defer_moe_finalize = ( + if ( experts_cls is TrtLlmMxfp4ExpertsMonolithic or experts_cls is TrtLlmNvFp4ExpertsMonolithic - ) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded + ) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded: + self.moe_config.defer_moe_finalize() from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import ( KimiK3LatentMoETailOp, ) @@ -136,20 +137,13 @@ class LatentMoERunner(MoERunner): rms_eps=norm.variance_epsilon, ) self._k3_latent_moe_tail_op = op - self.moe_config.defer_moe_finalize_max_num_tokens = ( - op.contract.max_num_tokens - if self.moe_config.use_deferred_moe_finalize - else -1 - ) if self.moe_config.use_deferred_moe_finalize: + self.moe_config.limit_deferred_moe_finalize(op.contract.max_num_tokens) logger.info_once( "K3 latent-MoE tail fusion with deferred top-k finalization " "is enabled for up to %d tokens.", op.contract.max_num_tokens, ) - else: - self.moe_config.defer_moe_finalize = False - self.moe_config.defer_moe_finalize_max_num_tokens = -1 def _get_zero_residual( self, diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index c13537513f..188f72e660 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -58,6 +58,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( accumulate_mla_context_chunk, init_mla_context_partial, neutralize_empty_context_partials, + split_kv_b_proj, ) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.layernorm import RMSNorm @@ -73,9 +74,6 @@ from vllm.model_executor.layers.quantization import ( QuantizationConfig, resolve_quant_method, ) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - get_and_maybe_dequant_weights, -) from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope from vllm.model_executor.utils import replace_parameter from vllm.models.common.ops import fused_q_kv_rmsnorm @@ -441,20 +439,13 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): projected into latent space by ``W_UK_T`` and the attention output is projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V. """ - kv_b_proj_weight = get_and_maybe_dequant_weights( - self.kv_b_proj, out_dtype=act_dtype - ).T - assert kv_b_proj_weight.shape == ( - self.kv_lora_rank, - self.num_local_heads * (self.qk_nope_head_dim + self.v_head_dim), - ), f"{kv_b_proj_weight.shape=}" - kv_b_proj_weight = kv_b_proj_weight.view( + W_UK, W_UV = split_kv_b_proj( + self.kv_b_proj, + act_dtype, self.kv_lora_rank, self.num_local_heads, - self.qk_nope_head_dim + self.v_head_dim, - ) - W_UK, W_UV = kv_b_proj_weight.split( - [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + self.qk_nope_head_dim, + self.v_head_dim, ) # (L, N, V) -> (N, L, V) replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True) diff --git a/vllm/models/minimax_m3/amd/indexer_aiter.py b/vllm/models/minimax_m3/amd/indexer_aiter.py index adf505aa7d..d46db51363 100644 --- a/vllm/models/minimax_m3/amd/indexer_aiter.py +++ b/vllm/models/minimax_m3/amd/indexer_aiter.py @@ -52,7 +52,11 @@ from vllm.models.minimax_m3.common.sparse_attention import ( _minimax_m3_aiter_sparse_pa_requested, ) from vllm.platforms import current_platform -from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata +from vllm.v1.attention.backend import ( + AttentionBackend, + CommonAttentionMetadata, + max_decode_query_len, +) from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.kv_cache_interface import AttentionSpec @@ -97,10 +101,7 @@ def aiter_indexer_max_decode_query_len(vllm_config: VllmConfig) -> int: since that is what the builder splits the batch on and therefore what the decode kernel will actually be handed. """ - spec = vllm_config.speculative_config - if spec is None or spec.num_speculative_tokens is None: - return 1 - return 1 + (2 if spec.parallel_drafting else 1) * spec.num_speculative_tokens + return max_decode_query_len(vllm_config) @cache diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 88651e944a..4871e66a9a 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -21,7 +21,7 @@ from collections.abc import Iterable import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import _custom_ops as ops from vllm import envs @@ -134,7 +134,7 @@ from vllm.v1.kv_cache_interface import ( logger = init_logger(__name__) -def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]: +def _sparse_attention_layer_ids(config: PreTrainedConfig) -> set[int]: """Layer ids whose attention runs the extra sparse "index" branch.""" cfg = getattr(config, "sparse_attention_config", None) if not cfg: @@ -145,7 +145,7 @@ def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]: return {i for i, f in enumerate(freq) if f != 0} -def _sparse_attention_layer_ordinals(config: PretrainedConfig) -> dict[int, int]: +def _sparse_attention_layer_ordinals(config: PreTrainedConfig) -> dict[int, int]: """Map each sparse-attention layer id to its ordinal among sparse layers.""" return { lid: ordinal @@ -153,7 +153,7 @@ def _sparse_attention_layer_ordinals(config: PretrainedConfig) -> dict[int, int] } -def _should_skip_index_topk(config: PretrainedConfig, layer_id: int) -> bool: +def _should_skip_index_topk(config: PreTrainedConfig, layer_id: int) -> bool: """ATOM ``index_topk_freq`` (cross-layer index sharing). Only 1 of every ``index_topk_freq`` sparse-attention layers recomputes the @@ -176,7 +176,7 @@ def _should_skip_index_topk(config: PretrainedConfig, layer_id: int) -> bool: return max(ordinal - offset, 0) % freq != 0 -def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool: +def _is_moe_layer(config: PreTrainedConfig, layer_id: int) -> bool: """Whether this layer's MLP is a sparse MoE block (vs a dense MLP).""" moe_layer_freq = getattr(config, "moe_layer_freq", None) if moe_layer_freq is None: @@ -184,7 +184,7 @@ def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool: return moe_layer_freq[layer_id] != 0 -def _build_rotary_emb(config: PretrainedConfig, head_dim: int): +def _build_rotary_emb(config: PreTrainedConfig, head_dim: int): """Build the (partial NeoX) RoPE, honoring an optional ``rope_scaling`` config. Without scaling the cos/sin cache is sized to ``max_position_embeddings`` @@ -262,7 +262,7 @@ class MiniMaxM3MLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, intermediate_size: int, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, @@ -338,7 +338,7 @@ class MiniMaxM3MoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -488,7 +488,7 @@ class MiniMaxM3Attention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -623,7 +623,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -1264,7 +1264,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): class MiniMaxM3DecoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, prefix: str, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, @@ -1738,7 +1738,7 @@ class MiniMaxM3SparseForConditionalGeneration( with self._mark_tower_model(vllm_config, {"image", "video"}): vision_config = config.vision_config self.vision_tower = MiniMaxVLVisionModel( - config=PretrainedConfig.from_dict(vision_config), + config=PreTrainedConfig.from_dict(vision_config), text_hidden_size=text_hidden_size, projector_hidden_size=projector_hidden_size, quant_config=self.quant_config, diff --git a/vllm/models/minimax_m3/amd/ops/swiglu_oai.py b/vllm/models/minimax_m3/amd/ops/swiglu_oai.py index df7abc0760..4dd2b91475 100644 --- a/vllm/models/minimax_m3/amd/ops/swiglu_oai.py +++ b/vllm/models/minimax_m3/amd/ops/swiglu_oai.py @@ -113,8 +113,10 @@ def _swiglu_oai_quant_kernel( act = gate * tl.sigmoid(alpha * gate) * (up + beta) # [BLOCK_M, 32] fp32 amax = tl.maximum(tl.max(tl.abs(act), axis=1), TINY) # [BLOCK_M] sb = tl.minimum(tl.maximum(tl.ceil(tl.log2(amax / FP8_MAX)) + 127.0, 0.0), 254.0) - descale = tl.exp2(sb - 127.0) - aq = (act / descale[:, None]).to(aq_ptr.dtype.element_ty) + # Computing the E8M0 byte-0 divisor (2^-127) can flush to zero on CDNA. + # Use its finite reciprocal so all-zero blocks do not produce 0/0. + rescale = tl.exp2(127.0 - sb) + aq = (act * rescale[:, None]).to(aq_ptr.dtype.element_ty) tl.store( aq_ptr + offs_m[:, None] * stride_qm + offs_c[None, :] * stride_qn, aq, diff --git a/vllm/models/minimax_m3/common/indexer.py b/vllm/models/minimax_m3/common/indexer.py index ac02bca1fc..2459d4ef85 100644 --- a/vllm/models/minimax_m3/common/indexer.py +++ b/vllm/models/minimax_m3/common/indexer.py @@ -91,7 +91,7 @@ class MiniMaxM3IndexerBackend(AttentionBackend): return [128] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [128] @classmethod diff --git a/vllm/models/minimax_m3/common/sparse_attention.py b/vllm/models/minimax_m3/common/sparse_attention.py index f385f82067..75aa90100c 100644 --- a/vllm/models/minimax_m3/common/sparse_attention.py +++ b/vllm/models/minimax_m3/common/sparse_attention.py @@ -159,7 +159,7 @@ class MiniMaxM3SparseBackend(AttentionBackend): return [128] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # Page size == sparse block size (one sparse block per KV page). return [128] diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py index 23b8b3ed31..ce56a179d0 100644 --- a/vllm/models/minimax_m3/common/vision_tower.py +++ b/vllm/models/minimax_m3/common/vision_tower.py @@ -6,7 +6,7 @@ import numpy as np import torch import torch.nn as nn from einops import rearrange -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils @@ -14,26 +14,23 @@ from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) +from vllm.model_executor.layers.conv import Conv3dLayer from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb +from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope +from vllm.model_executor.layers.rotary_embedding.mrope_vit_setup import ( + vit_mrope_setup, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import ( get_vit_attn_backend, is_vit_use_data_parallel, ) -from vllm.platforms import current_platform - -# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton -# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with -# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide -# when RoPE must be applied per video segment instead of in one launch. -_HIP_MAX_GRID_DIM_Y = 65536 class MiniMaxVLPatchEmbed(nn.Module): @@ -43,7 +40,7 @@ class MiniMaxVLPatchEmbed(nn.Module): and projects each to a hidden-size embedding. """ - def __init__(self, config: PretrainedConfig) -> None: + def __init__(self, config: PreTrainedConfig) -> None: super().__init__() compression = config.img_token_compression_config temporal_patch_size = compression.get("temporal_patch_size", 2) @@ -55,7 +52,7 @@ class MiniMaxVLPatchEmbed(nn.Module): self.num_channels = num_channels self.hidden_size = config.hidden_size - self.patch_embedding = nn.Conv3d( + self.patch_embedding = Conv3dLayer( in_channels=num_channels, out_channels=config.hidden_size, kernel_size=(temporal_patch_size, patch_size, patch_size), @@ -124,56 +121,13 @@ class MiniMaxVLAttention(nn.Module): head_size=self.head_dim, prefix=f"{prefix}.attn", ) - # ApplyRotaryEmb handles the internal cos/sin repeat and partial - # rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax). - # enable_fp32_compute=True runs the rotation in fp32 (q/k upcast, - # fp32 cos/sin), matching the reference ``_minimax_rope_applier``. - self.apply_rotary_emb = ApplyRotaryEmb( - enforce_enable=True, enable_fp32_compute=True - ) - - def _apply_rotary_emb( - self, - qk_reshaped: torch.Tensor, - rotary_cos: torch.Tensor, - rotary_sin: torch.Tensor, - seq_len: int, - rotary_segment_lengths: list[int] | None, - ) -> torch.Tensor: - # Default fast path (all NVIDIA inputs, and ROCm short clips/images): - # a single rotary kernel launch. ``rotary_segment_lengths`` is only - # populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so - # the per-segment path below is ROCm-only and never touches the - # NVIDIA/CUDA code path. - if not current_platform.is_rocm() or rotary_segment_lengths is None: - return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin) - - # ROCm only: the HIP flash-attn Triton rotary kernel fails with - # hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds - # _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128 - # (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of - # 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail. - # Only long videos cross it; since vision_segment_max_frames caps each - # segment at a few frames (<< limit), applying RoPE per segment keeps - # every sub-call in range. Splitting on segment boundaries is - # mathematically exact because rotary_cos/sin are precomputed per token. - # Images and short clips stay on the single-kernel fast path above. - rotary_dim = rotary_cos.shape[-1] * 2 - block_m = 8 if rotary_dim <= 128 else 4 - hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m - if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1: - return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin) - - qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1) - cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0) - sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0) - return torch.cat( - [ - self.apply_rotary_emb(qk_s, cos_s, sin_s) - for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments) - ], - dim=1, - ) + # Partial 3D RoPE geometry: rot_dim is split evenly across t/h/w + # (same formula as MiniMaxVLVisionTransformer; rot_dim may be < + # head_dim). mrope_section is in half-dim units for triton_mrope. + rope_dims = 2 * (self.head_dim // 2) + axis_dim = 2 * ((rope_dims // 3) // 2) + self.rotary_dim = 3 * axis_dim + self.mrope_section = [axis_dim // 2] * 3 def forward( self, @@ -182,33 +136,34 @@ class MiniMaxVLAttention(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: # x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim] x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim) seq_len, batch_size, _ = x_qkv.shape - # Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention - qkv = rearrange( - x_qkv, - "s b (three head d) -> b s three head d", - three=3, - head=self.num_heads_per_partition, - ) - qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d) + qkv = x_qkv.view(seq_len, 3, self.num_heads_per_partition, self.head_dim) + v = qkv[:, 2].unsqueeze(0) # (b=1, N, heads, head_dim) strided view - # Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application. - # rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally - # and rotates only the first 2*half_rot_dim dims, passing the rest through. - qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous() - qk_rotated = self._apply_rotary_emb( - qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths + # In-place partial 3D RoPE on strided q/k views of the packed qkv + # tensor: one kernel launch, no repack/upcast copies. fp32 cos/sin + # make the kernel compute in fp32, matching the reference + # ``_minimax_rope_applier`` precision. + q = qkv[:, 0].reshape(seq_len, -1) + k = qkv[:, 1].reshape(seq_len, -1) + triton_mrope( + q, + k, + rotary_cos, + rotary_sin, + self.mrope_section, + self.head_dim, + self.rotary_dim, + mrope_interleaved=False, + is_neox_style=True, ) - qk_rotated = qk_rotated.view( - 2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim - ) - q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim) + q = q.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim) + k = k.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim) # Flash attention → (b, N, heads, head_dim) context = self.attn( @@ -231,7 +186,7 @@ class MiniMaxVLEncoderLayer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: @@ -271,7 +226,6 @@ class MiniMaxVLEncoderLayer(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: # x: (N, 1, hidden_size) @@ -281,7 +235,6 @@ class MiniMaxVLEncoderLayer(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths, ) residual = x @@ -294,7 +247,7 @@ class MiniMaxVLEncoderLayer(nn.Module): class MiniMaxVLEncoder(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, num_hidden_layers_override: int | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -323,7 +276,6 @@ class MiniMaxVLEncoder(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: for layer in self.layers: @@ -333,7 +285,6 @@ class MiniMaxVLEncoder(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths, ) return x @@ -348,7 +299,7 @@ class MiniMaxVLVisionTransformer(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, num_hidden_layers_override: int | None = None, require_post_norm: bool | None = None, quant_config: QuantizationConfig | None = None, @@ -426,82 +377,6 @@ class MiniMaxVLVisionTransformer(nn.Module): # out_hidden_size needed by run_dp_sharded_mrope_vision_model self.out_hidden_size = embed_dim - # ── RoPE helpers ───────────────────────────────────────────────────── - - def _get_3d_rope_embed( - self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int - ) -> torch.Tensor: - """Compute 3D RoPE frequencies for a single (T, H, W) grid. - - Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers. - Mirrors the reference ``_get_3d_rope_embed`` exactly. - """ - tokens_per_frame = grid_h * grid_w - - tpos_ids = ( - torch.arange(grid_t, device=self.inv_freq_t.device) - .unsqueeze(1) - .expand(-1, tokens_per_frame) - .flatten() - ) - - hpos_ids = ( - torch.arange(grid_h, device=self.inv_freq_h.device) - .unsqueeze(1) - .expand(-1, grid_w) - .reshape( - grid_h // spatial_merge_size, - spatial_merge_size, - grid_w // spatial_merge_size, - spatial_merge_size, - ) - .permute(0, 2, 1, 3) - .unsqueeze(0) - .expand(grid_t, -1, -1, -1, -1) - .flatten() - ) - wpos_ids = ( - torch.arange(grid_w, device=self.inv_freq_w.device) - .unsqueeze(0) - .expand(grid_h, -1) - .reshape( - grid_h // spatial_merge_size, - spatial_merge_size, - grid_w // spatial_merge_size, - spatial_merge_size, - ) - .permute(0, 2, 1, 3) - .unsqueeze(0) - .expand(grid_t, -1, -1, -1, -1) - .flatten() - ) - - max_t = max(grid_t, 1) - max_hw = max(grid_h, grid_w) - - seq_t = torch.arange( - max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype - ) - seq_hw = torch.arange( - max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype - ) - - freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2) - freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2) - freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2) - - return torch.cat( - [freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1 - ) # (T*H*W, half_rot_dim) - - def _get_rope_embed_3d( - self, grid_thw: list[list[int]], spatial_merge_size: int - ) -> torch.Tensor: - embeds = [ - self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw - ] - return torch.cat(embeds, dim=0) # (total_N, half_rot_dim) - # ── Frame-limit helper (mirrors the reference) ─────────────────────── def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]: @@ -556,21 +431,19 @@ class MiniMaxVLVisionTransformer(nn.Module): hidden.device, ) - # 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally - freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size) - freqs = freqs.to(device=hidden.device) - # Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the - # rotation in fp32 to match the reference precision. - rotary_cos, rotary_sin = freqs.cos(), freqs.sin() + # 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for + # triton_mrope from one fused setup kernel. Kept fp32 so the kernel + # computes the rotation in fp32, matching the reference. + rotary_cos, rotary_sin = vit_mrope_setup( + self.inv_freq_t, + self.inv_freq_h, + self.inv_freq_w, + limited, + self.spatial_merge_size, + ) # Encoder expects (N, 1, hidden_size) — add batch dim hidden = hidden.unsqueeze(1) - # On ROCm, the flash_attn Triton rotary kernel can fail with - # hipErrorInvalidValue when seqlen is very large, e.g. 192k video - # tokens; pass per-segment lengths so RoPE can be applied in chunks. - # On other platforms leave it None -> single-kernel fast path, so the - # NVIDIA/CUDA code path is unchanged. - rotary_segment_lengths = lens if current_platform.is_rocm() else None hidden = self.encoder( hidden, @@ -578,7 +451,6 @@ class MiniMaxVLVisionTransformer(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths=sequence_lengths, ) hidden = hidden.squeeze(1) # back to (total_N, hidden_size) @@ -678,7 +550,7 @@ class MiniMaxVLVisionModel(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, text_hidden_size: int, projector_hidden_size: int | None = None, quant_config: QuantizationConfig | None = None, diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 970b06d407..4fb260330f 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -16,7 +16,7 @@ from collections.abc import Iterable import torch from torch import nn -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import _custom_ops as ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture @@ -92,7 +92,7 @@ from vllm.v1.kv_cache_interface import ( ) -def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]: +def _sparse_attention_layer_ids(config: PreTrainedConfig) -> set[int]: """Layer ids whose attention runs the extra sparse "index" branch.""" cfg = getattr(config, "sparse_attention_config", None) if not cfg: @@ -103,7 +103,7 @@ def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]: return {i for i, f in enumerate(freq) if f != 0} -def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool: +def _is_moe_layer(config: PreTrainedConfig, layer_id: int) -> bool: """Whether this layer's MLP is a sparse MoE block (vs a dense MLP).""" moe_layer_freq = getattr(config, "moe_layer_freq", None) if moe_layer_freq is None: @@ -147,7 +147,7 @@ class MiniMaxM3MLP(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, intermediate_size: int, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, @@ -194,7 +194,7 @@ class MiniMaxM3MoE(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, @@ -290,7 +290,7 @@ class MiniMaxM3Attention(nn.Module): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -403,7 +403,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, layer_id: int, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -1130,7 +1130,7 @@ class MiniMaxM3SparseForConditionalGeneration( with self._mark_tower_model(vllm_config, {"image", "video"}): vision_config = config.vision_config self.vision_tower = MiniMaxVLVisionModel( - config=PretrainedConfig.from_dict(vision_config), + config=PreTrainedConfig.from_dict(vision_config), text_hidden_size=text_hidden_size, projector_hidden_size=projector_hidden_size, quant_config=self.quant_config, diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py index 2616a3e9e0..0055821d56 100644 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -11,6 +11,7 @@ from torch import nn from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context +from vllm.logger import init_logger from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.mamba_utils import ( @@ -18,6 +19,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, is_conv_state_dim_first, ) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, +) from vllm.model_executor.models.utils import AutoWeightsLoader from vllm.transformers_utils.configs.qwen4_exp import ( Qwen4ExpTextConfig, @@ -30,7 +34,13 @@ from vllm.v1.attention.backends.short_conv_attn import ( ) from vllm.v1.attention.backends.utils import NULL_BLOCK_ID -from ..common.ple import PLEVocabParallelEmbedding +from ..common.ngram_embedding import ( + Qwen4ExpPLEDeviceEmbedding, + Qwen4ExpPLEEmbeddingMethod, + Qwen4ExpPLEPinnedHostEmbedding, +) + +logger = init_logger(__name__) class Qwen4ExpPLEGroupedNorm(nn.Module): @@ -172,6 +182,10 @@ class Qwen4ExpNGramEmbedding(nn.Module): max_num_reqs: int, prefix: str, layer_name: str, + *, + data_parallel_rank: int = 0, + quant_config: QuantizationConfig | None = None, + params_dtype: torch.dtype | None = None, ) -> None: super().__init__() self.embedding_dim = embedding_dim @@ -224,11 +238,39 @@ class Qwen4ExpNGramEmbedding(nn.Module): ) divisor = int(config.make_ngram_vocab_size_divisible_by) padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor - self.ngram_embedding = PLEVocabParallelEmbedding( + if params_dtype is None: + params_dtype = torch.get_default_dtype() + embedding_prefix = f"{prefix}.ngram_embedding" + embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config( + quant_config, + embedding_prefix, + getattr(config, "ple_embedding_dtype", None), + ) + engram_config = get_current_vllm_config().engram_config + embedding_cls = ( + Qwen4ExpPLEPinnedHostEmbedding + if engram_config is not None and engram_config.cpu_offload + else Qwen4ExpPLEDeviceEmbedding + ) + self.ngram_embedding = embedding_cls( padded_vocab_size, self.head_dim, + params_dtype=params_dtype, padding_size=divisor, - prefix=f"{prefix}.ngram_embedding", + prefix=embedding_prefix, + embedding_method=embedding_quant_method, + num_ngram_heads=self.ngram_heads, + max_total_tokens=max_total_tokens, + data_parallel_rank=data_parallel_rank, + ) + logger.info( + "Initialized AMD PLE embedding %s: quantization_method=%s, " + "weight_dtype=%s, weight_device=%s, pinned=%s", + embedding_prefix, + type(embedding_quant_method).__name__, + self.ngram_embedding.weight.dtype, + self.ngram_embedding.weight.device, + self.ngram_embedding.weight.is_pinned(), ) self.register_buffer( "positions_buffer", @@ -344,9 +386,21 @@ class Qwen4ExpNGramEmbedding(nn.Module): ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets id_blocks.append(ids[request_indices, adjusted_columns]) ngram_ids = torch.cat(id_blocks, dim=-1) + embedding = self.ngram_embedding + if embedding.supports_prefetch: + output = ngram_ids.new_empty( + (ngram_ids.shape[0], self.embedding_dim), + dtype=embedding.weight.dtype, + ) + torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( + ngram_ids, + output, + self.layer_name, + ) + return output output = ngram_ids.new_empty( (ngram_ids.shape[0], self.embedding_dim), - dtype=self.ngram_embedding.params_dtype, + dtype=embedding.params_dtype, ) torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding( ngram_ids, @@ -459,6 +513,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): vllm_config.scheduler_config.max_num_seqs, f"{prefix}.ple_embedding", prefix, + data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, + quant_config=quant_config, + params_dtype=model_config.dtype, ) self.key_proj = ReplicatedLinear( int(config.ple_embed_dim), @@ -1044,6 +1101,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): f"{hidden_states.shape[0]}" ) embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) + embeddings = self.ple_embedding.ngram_embedding.dequantize( + embeddings, + self.ple_embedding.ngram_embedding.params_dtype, + ) key, _ = self.key_proj(embeddings) value, _ = self.value_proj(embeddings) token_count = hidden_states.shape[0] @@ -1081,6 +1142,24 @@ def qwen4_exp_amd_ple_ngram_embedding( output.copy_(result) +def qwen4_exp_amd_ple_ngram_embedding_pinned( + ngram_ids: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + """Run the pinned PLE UVA lookup outside Inductor's FX graph. + + Same rationale as the device-path escape: keeping the large embedding weight + out of the graph prevents AOT compile-time autotuning from materializing a + synthetic copy of the weight. + """ + layer = get_forward_context().no_compile_layers[layer_name] + if not isinstance(layer, Qwen4ExpPLELayer): + raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner") + result = layer.ple_embedding.ngram_embedding.sync_lookup(ngram_ids).flatten(-2) + output.copy_(result) + + def qwen4_exp_ple_short_conv( inputs: torch.Tensor, output: torch.Tensor, @@ -1098,6 +1177,13 @@ direct_register_custom_op( ) +direct_register_custom_op( + op_name="qwen4_exp_amd_ple_ngram_embedding_pinned", + op_func=qwen4_exp_amd_ple_ngram_embedding_pinned, + mutates_args=["output"], +) + + direct_register_custom_op( op_name="qwen4_exp_ple_short_conv", op_func=qwen4_exp_ple_short_conv, diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py index bed48a4cba..3c8a61bd54 100644 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -75,7 +75,7 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend): return "QWEN4_EXP_QSA_TRITON" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # QSA consumes manager pages directly and does not use FA4 paged attention. return [MultipleOf(16)] diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py new file mode 100644 index 0000000000..4d1d358958 --- /dev/null +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -0,0 +1,542 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared Qwen4Exp n-gram embedding storage with device and pinned-host backends. + +Both the NVIDIA and AMD Qwen4Exp implementations use these classes so the (large) +n-gram embedding table can be kept in pinned host memory and looked up through +Unified Virtual Addressing on any CUDA-alike platform. +""" + +from abc import ABC, abstractmethod +from typing import ClassVar + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.distributed import get_dp_group, get_etp_group, get_tp_group +from vllm.forward_context import DPMetadata, get_forward_context +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptMixedPrecisionConfig, + ModelOptQuantConfigBase, +) +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + create_fp8_scale_parameter, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped, +) +from vllm.model_executor.parameter import ( + ModelWeightParameter, + PerTensorScaleParameter, +) +from vllm.model_executor.utils import set_weight_attrs +from vllm.triton_utils import tl, triton +from vllm.utils.platform_utils import is_uva_available +from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor + +from .ple import PLEVocabParallelEmbedding + +logger = init_logger(__name__) + + +class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC): + """ETP-sharded PLE table shared by device and pinned-host backends.""" + + supports_prefetch: ClassVar[bool] = False + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + params_dtype: torch.dtype, + padding_size: int, + prefix: str, + embedding_method: "Qwen4ExpPLEEmbeddingMethod", + num_ngram_heads: int = 1, + max_total_tokens: int = 0, + data_parallel_rank: int = 0, + ) -> None: + del num_ngram_heads, max_total_tokens + super().__init__( + num_embeddings, + embedding_dim, + params_dtype=params_dtype, + padding_size=padding_size, + prefix=prefix, + quant_method=embedding_method, + parallel_group=get_etp_group(), + ) + self.embedding_method = embedding_method + self.data_parallel_rank = data_parallel_rank + tp_size = get_tp_group().world_size + if self.tp_size % tp_size: + raise ValueError( + "ETP size must be divisible by TP size, but got " + f"ETP={self.tp_size} and TP={tp_size}" + ) + self.etp_data_parallel_size = self.tp_size // tp_size + + @abstractmethod + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate storage for the complete embedding weight.""" + raise NotImplementedError + + def dequantize( + self, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + """Delegate storage-format conversion to the embedding method.""" + return self.embedding_method.dequantize(self, embeddings, output_dtype) + + def _get_dp_gather_slot(self, local_num_tokens: int) -> tuple[int, int]: + """Return the per-DP slot size and this rank's slot offset.""" + if self.etp_data_parallel_size == 1: + return local_num_tokens, 0 + dp_metadata: DPMetadata | None = get_forward_context().dp_metadata + if dp_metadata is None: + raise RuntimeError("ETP spanning DP requires DP token metadata") + group_start = (self.data_parallel_rank // self.etp_data_parallel_size) * ( + self.etp_data_parallel_size + ) + group_end = group_start + self.etp_data_parallel_size + token_counts = dp_metadata.num_tokens_across_dp_cpu.tolist() + group_counts = token_counts[group_start:group_end] + slot_size = max(group_counts) + dp_rank = get_dp_group().rank_in_group + return slot_size, dp_rank * slot_size + + def _gather_dp_ids( + self, + ngram_ids: torch.Tensor, + slot_size: int, + ) -> torch.Tensor: + """Gather DP-local IDs that share one ETP-sharded PLE table.""" + if self.etp_data_parallel_size == 1: + return ngram_ids + if ngram_ids.shape[0] < slot_size: + padding = ngram_ids.new_zeros( + slot_size - ngram_ids.shape[0], ngram_ids.shape[1] + ) + ngram_ids = torch.cat((ngram_ids, padding), dim=0) + return get_dp_group().all_gather(ngram_ids, dim=0) + + def _select_embeddings( + self, + embeddings: torch.Tensor, + local_num_tokens: int, + slot_offset: int, + ) -> torch.Tensor: + """Select this DP rank's rows from the ETP-reduced embeddings.""" + if self.etp_data_parallel_size == 1: + return embeddings + return embeddings.narrow(0, slot_offset, local_num_tokens) + + @abstractmethod + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Start an asynchronous lookup when supported.""" + raise NotImplementedError + + +class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase): + """Quantization interface shared by resident and pinned PLE tables.""" + + # PLE post-load processing only validates scales in their current storage. + requires_device_loading: bool = False + + @staticmethod + def from_quant_config( + quant_config: QuantizationConfig | None, + prefix: str, + embedding_dtype: str | None = None, + ) -> "Qwen4ExpPLEEmbeddingMethod": + """Select the concrete PLE embedding format for a layer.""" + if embedding_dtype == "float8_e4m3fn": + return Qwen4ExpPLEFp8EmbeddingMethod() + if quant_config is None: + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if isinstance(quant_config, ModelOptMixedPrecisionConfig): + if quant_config._resolve_quant_algo(prefix) == "FP8": + return Qwen4ExpPLEFp8EmbeddingMethod() + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if isinstance( + quant_config, ModelOptQuantConfigBase + ) and quant_config.is_layer_excluded(prefix): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if not isinstance(quant_config, Fp8Config): + raise NotImplementedError( + "Qwen4Exp PLE embedding does not support quantization config " + f"{type(quant_config).__name__}" + ) + + ignored_layers = quant_config.ignored_layers + if is_layer_skipped( + prefix, + ignored_layers, + quant_config.packed_modules_mapping, + match_mode=quant_config.ignored_layers_match_mode, + ): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + # PLE checkpoint shards form one runtime embedding parameter. + shard_prefix = f"{prefix}.shard_" + if any(name.startswith(shard_prefix) for name in ignored_layers): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if not quant_config.is_checkpoint_fp8_serialized: + raise NotImplementedError( + "Qwen4Exp PLE embedding only supports serialized FP8 checkpoints" + ) + return Qwen4ExpPLEFp8EmbeddingMethod() + + def apply( + self, + layer: nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + raise NotImplementedError("PLE weights only support embedding lookup") + + def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + @abstractmethod + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + """Convert looked-up PLE rows to the activation dtype.""" + raise NotImplementedError + + +class Qwen4ExpPLEUnquantizedEmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): + """Unquantized PLE embedding storage and lookup semantics.""" + + def create_weights( + self, + layer: Qwen4ExpPLEEmbedding, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size + weight = nn.Parameter( + layer.allocate_embedding_weight( + sum(output_partition_sizes), + input_size_per_partition, + params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + set_weight_attrs(weight, extra_weight_attrs) + layer.register_parameter("weight", weight) + + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + del layer, output_dtype + return embeddings + + +class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): + """FP8 PLE embedding with one global checkpoint scale.""" + + def create_weights( + self, + layer: Qwen4ExpPLEEmbedding, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size, params_dtype + weight_loader = extra_weight_attrs.get("weight_loader") + weight = ModelWeightParameter( + data=layer.allocate_embedding_weight( + sum(output_partition_sizes), + input_size_per_partition, + torch.float8_e4m3fn, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + weight_scale = create_fp8_scale_parameter( + PerTensorScaleParameter, + output_partition_sizes, + input_size_per_partition, + None, + weight_loader, + scale_dtype=torch.float32, + ) + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Reject FP8 PLE checkpoints without a global scale.""" + sentinel = torch.finfo(torch.float32).min + if torch.any(layer.weight_scale == sentinel): + raise ValueError("FP8 PLE checkpoint is missing its global scale") + + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + weight_scale = getattr(layer, "weight_scale", None) + if weight_scale is None: + raise RuntimeError("FP8 PLE embedding is missing its global scale") + if weight_scale.device != embeddings.device: + raise RuntimeError("FP8 PLE embedding scale must be on the output device") + return embeddings.to(output_dtype) * weight_scale.to(output_dtype) + + +class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding): + """PLE table allocated on the active model device.""" + + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate the complete PLE weight on the active device.""" + return torch.empty(num_embeddings, embedding_dim, dtype=dtype) + + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Resident embedding prefetch is a no-op.""" + return None + + def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor: + """Gather ETP inputs, look up embeddings, and select local rows.""" + slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0]) + gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) + embeddings = super().forward(gathered_ids) + return self._select_embeddings( + embeddings, + ngram_ids.shape[0], + slot_offset, + ) + + +@triton.jit +def _lookup_ple_embedding_from_pinned_kernel( + weight_ptr, + ids_ptr, + output_ptr, + embedding_dim, + tp_vocab_start, + tp_vocab_end, + BLOCK_D: tl.constexpr, +): + """Look up TP-owned PLE rows through a CUDA view of pinned host memory.""" + row_id = tl.program_id(0) + global_idx = tl.load(ids_ptr + row_id) + in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end) + local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0) + offsets = tl.arange(0, BLOCK_D) + store_mask = offsets < embedding_dim + load_mask = store_mask & in_range + values = tl.load( + weight_ptr + local_idx * embedding_dim + offsets, + mask=load_mask, + other=0.0, + ) + tl.store( + output_ptr + row_id * embedding_dim + offsets, + values, + mask=store_mask, + ) + + +class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): + """PLE table loaded into pinned CPU memory and looked up through UVA.""" + + supports_prefetch: ClassVar[bool] = True + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + params_dtype: torch.dtype, + padding_size: int, + prefix: str, + embedding_method: Qwen4ExpPLEEmbeddingMethod, + num_ngram_heads: int = 1, + max_total_tokens: int = 0, + data_parallel_rank: int = 0, + ) -> None: + if not is_uva_available(): + raise RuntimeError("Engram CPU offload requires UVA support") + super().__init__( + num_embeddings, + embedding_dim, + params_dtype=params_dtype, + padding_size=padding_size, + prefix=prefix, + embedding_method=embedding_method, + num_ngram_heads=num_ngram_heads, + max_total_tokens=max_total_tokens, + data_parallel_rank=data_parallel_rank, + ) + self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) + self._block_d = triton.next_power_of_2(self.embedding_dim) + self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device) + self._prefetch_buffer = torch.empty( + max_total_tokens * self.etp_data_parallel_size, + num_ngram_heads, + self.embedding_dim, + dtype=self.weight.dtype, + device=self._uva_weight.device, + ) + self._output_dim = num_ngram_heads * self.embedding_dim + + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate the complete PLE weight directly in pinned CPU memory.""" + return torch.empty( + num_embeddings, + embedding_dim, + dtype=dtype, + device="cpu", + pin_memory=True, + ) + + def _lookup( + self, + input_ids: torch.Tensor, + output: torch.Tensor | None = None, + ) -> torch.Tensor: + """Look up local ETP rows while preserving the weight storage dtype.""" + expected_shape = (*input_ids.shape, self.embedding_dim) + if output is None: + output = torch.empty( + expected_shape, + dtype=self.weight.dtype, + device=input_ids.device, + ) + elif ( + tuple(output.shape) != expected_shape + or output.dtype != self.weight.dtype + or output.device != input_ids.device + ): + raise ValueError( + "PLE prefetch output must match the input shape, weight dtype, " + "and input device" + ) + + flat_ids = input_ids.reshape(-1).long() + if flat_ids.numel(): + _lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)]( + self._uva_weight, + flat_ids, + output, + self.embedding_dim, + self.shard_indices.org_vocab_start_index, + self.shard_indices.org_vocab_end_index, + BLOCK_D=self._block_d, + ) + return output + + def sync_lookup(self, ngram_ids: torch.Tensor) -> torch.Tensor: + """Synchronous UVA lookup for platforms without prefetch wiring.""" + slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0]) + gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) + embeddings = self._lookup(gathered_ids) + embeddings = self._reduce_etp_embeddings(embeddings) + return self._select_embeddings( + embeddings, + ngram_ids.shape[0], + slot_offset, + ) + + def _reduce_etp_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor: + """Combine pinned lookup results owned by different ETP ranks.""" + if self.tp_size == 1: + return embeddings + assert self.parallel_group is not None + if embeddings.dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + # Each vocabulary row has one owner, so reduce the raw FP8 bytes. + reduced = self.parallel_group.all_reduce(embeddings.view(torch.int8)) + return reduced.view(embeddings.dtype) + return self.parallel_group.all_reduce(embeddings) + + @eager_break_during_capture + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Gather ETP IDs and launch their UVA lookup on the side stream.""" + slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0]) + gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) + active_output = self._prefetch_buffer[: gathered_ids.shape[0]] + prefetch_stream = self._prefetch_stream + prefetch_stream.wait_stream(torch.cuda.current_stream()) + gathered_ids.record_stream(prefetch_stream) + with torch.cuda.stream(prefetch_stream): + self._lookup(gathered_ids, output=active_output) + + @eager_break_during_capture + def _finalize_prefetch( + self, + prefetch_output: torch.Tensor, + output: torch.Tensor, + ) -> None: + """Join the side stream, reduce ETP shards, and select local rows.""" + torch.cuda.current_stream().wait_stream(self._prefetch_stream) + slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0]) + active_output = prefetch_output[: slot_size * self.etp_data_parallel_size] + embeddings = self._reduce_etp_embeddings(active_output) + embeddings = self._select_embeddings( + embeddings, + output.shape[0], + slot_offset, + ) + output.copy_(embeddings.flatten(-2)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Finish the pinned lookup into graph-owned output storage.""" + output = self._prefetch_buffer.new_empty( + (hidden_states.shape[0], self._output_dim) + ) + self._finalize_prefetch(self._prefetch_buffer, output) + return output diff --git a/vllm/models/qwen4_exp/config.py b/vllm/models/qwen4_exp/config.py index 5d9cc0aa44..d3ff7ca69a 100644 --- a/vllm/models/qwen4_exp/config.py +++ b/vllm/models/qwen4_exp/config.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar, cast -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.models.qwen3_vl.configuration_qwen3_vl import ( Qwen3VLVisionConfig, ) @@ -219,9 +219,9 @@ class Qwen4ExpTextConfig(Qwen3NextConfig): return max(int(self.ngram_size) - 1, 0) -class Qwen4ExpConfig(PretrainedConfig): +class Qwen4ExpConfig(PreTrainedConfig): model_type = "qwen4_exp" - sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = { + sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = { "vision_config": Qwen4ExpVisionConfig, "text_config": Qwen4ExpTextConfig, } diff --git a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py index c26fd87cff..0b6fd650b1 100644 --- a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py +++ b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py @@ -2,534 +2,42 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Qwen4Exp n-gram embeddings with device and pinned-host storage.""" -from abc import ABC, abstractmethod from collections.abc import Iterable -from typing import ClassVar import torch -import torch.nn.functional as F from torch import nn -from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import get_current_vllm_config -from vllm.distributed import get_dp_group, get_etp_group, get_tp_group -from vllm.forward_context import DPMetadata, get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, - QuantizeMethodBase, -) -from vllm.model_executor.layers.quantization.fp8 import Fp8Config -from vllm.model_executor.layers.quantization.modelopt import ( - ModelOptMixedPrecisionConfig, - ModelOptQuantConfigBase, -) -from vllm.model_executor.layers.quantization.utils.fp8_utils import ( - create_fp8_scale_parameter, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - is_layer_skipped, ) from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.model_executor.parameter import ( - ModelWeightParameter, - PerTensorScaleParameter, -) -from vllm.model_executor.utils import set_weight_attrs from vllm.transformers_utils.configs.qwen4_exp import ( Qwen4ExpTextConfig, ) -from vllm.triton_utils import tl, triton -from vllm.utils.platform_utils import is_uva_available -from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor -from ..common.ple import PLEVocabParallelEmbedding +from ..common.ngram_embedding import ( + Qwen4ExpPLEDeviceEmbedding, + Qwen4ExpPLEEmbedding, + Qwen4ExpPLEEmbeddingMethod, + Qwen4ExpPLEFp8EmbeddingMethod, + Qwen4ExpPLEPinnedHostEmbedding, + Qwen4ExpPLEUnquantizedEmbeddingMethod, +) from .ops.ple import ple_ngram_ids logger = init_logger(__name__) - -class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC): - """ETP-sharded PLE table shared by device and pinned-host backends.""" - - supports_prefetch: ClassVar[bool] = False - - def __init__( - self, - num_embeddings: int, - embedding_dim: int, - *, - params_dtype: torch.dtype, - padding_size: int, - prefix: str, - embedding_method: "Qwen4ExpPLEEmbeddingMethod", - num_ngram_heads: int = 1, - max_total_tokens: int = 0, - data_parallel_rank: int = 0, - ) -> None: - del num_ngram_heads, max_total_tokens - super().__init__( - num_embeddings, - embedding_dim, - params_dtype=params_dtype, - padding_size=padding_size, - prefix=prefix, - quant_method=embedding_method, - parallel_group=get_etp_group(), - ) - self.embedding_method = embedding_method - self.data_parallel_rank = data_parallel_rank - tp_size = get_tp_group().world_size - if self.tp_size % tp_size: - raise ValueError( - "ETP size must be divisible by TP size, but got " - f"ETP={self.tp_size} and TP={tp_size}" - ) - self.etp_data_parallel_size = self.tp_size // tp_size - - @abstractmethod - def allocate_embedding_weight( - self, - num_embeddings: int, - embedding_dim: int, - dtype: torch.dtype, - ) -> torch.Tensor: - """Allocate storage for the complete embedding weight.""" - raise NotImplementedError - - def dequantize( - self, - embeddings: torch.Tensor, - output_dtype: torch.dtype, - ) -> torch.Tensor: - """Delegate storage-format conversion to the embedding method.""" - return self.embedding_method.dequantize(self, embeddings, output_dtype) - - def _get_dp_gather_slot(self, local_num_tokens: int) -> tuple[int, int]: - """Return the per-DP slot size and this rank's slot offset.""" - if self.etp_data_parallel_size == 1: - return local_num_tokens, 0 - dp_metadata: DPMetadata | None = get_forward_context().dp_metadata - if dp_metadata is None: - raise RuntimeError("ETP spanning DP requires DP token metadata") - group_start = (self.data_parallel_rank // self.etp_data_parallel_size) * ( - self.etp_data_parallel_size - ) - group_end = group_start + self.etp_data_parallel_size - token_counts = dp_metadata.num_tokens_across_dp_cpu.tolist() - group_counts = token_counts[group_start:group_end] - slot_size = max(group_counts) - dp_rank = get_dp_group().rank_in_group - return slot_size, dp_rank * slot_size - - def _gather_dp_ids( - self, - ngram_ids: torch.Tensor, - slot_size: int, - ) -> torch.Tensor: - """Gather DP-local IDs that share one ETP-sharded PLE table.""" - if self.etp_data_parallel_size == 1: - return ngram_ids - if ngram_ids.shape[0] < slot_size: - padding = ngram_ids.new_zeros( - slot_size - ngram_ids.shape[0], ngram_ids.shape[1] - ) - ngram_ids = torch.cat((ngram_ids, padding), dim=0) - return get_dp_group().all_gather(ngram_ids, dim=0) - - def _select_embeddings( - self, - embeddings: torch.Tensor, - local_num_tokens: int, - slot_offset: int, - ) -> torch.Tensor: - """Select this DP rank's rows from the ETP-reduced embeddings.""" - if self.etp_data_parallel_size == 1: - return embeddings - return embeddings.narrow(0, slot_offset, local_num_tokens) - - @abstractmethod - def start_prefetch( - self, - hidden_states: torch.Tensor, - ngram_ids: torch.Tensor, - ) -> None: - """Start an asynchronous lookup when supported.""" - raise NotImplementedError - - -class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase): - """Quantization interface shared by resident and pinned PLE tables.""" - - # PLE post-load processing only validates scales in their current storage. - requires_device_loading: bool = False - - @staticmethod - def from_quant_config( - quant_config: QuantizationConfig | None, - prefix: str, - embedding_dtype: str | None = None, - ) -> "Qwen4ExpPLEEmbeddingMethod": - """Select the concrete PLE embedding format for a layer.""" - if embedding_dtype == "float8_e4m3fn": - return Qwen4ExpPLEFp8EmbeddingMethod() - if quant_config is None: - return Qwen4ExpPLEUnquantizedEmbeddingMethod() - if isinstance(quant_config, ModelOptMixedPrecisionConfig): - if quant_config._resolve_quant_algo(prefix) == "FP8": - return Qwen4ExpPLEFp8EmbeddingMethod() - return Qwen4ExpPLEUnquantizedEmbeddingMethod() - if isinstance( - quant_config, ModelOptQuantConfigBase - ) and quant_config.is_layer_excluded(prefix): - return Qwen4ExpPLEUnquantizedEmbeddingMethod() - if not isinstance(quant_config, Fp8Config): - raise NotImplementedError( - "Qwen4Exp PLE embedding does not support quantization config " - f"{type(quant_config).__name__}" - ) - - ignored_layers = quant_config.ignored_layers - if is_layer_skipped( - prefix, - ignored_layers, - quant_config.packed_modules_mapping, - match_mode=quant_config.ignored_layers_match_mode, - ): - return Qwen4ExpPLEUnquantizedEmbeddingMethod() - # PLE checkpoint shards form one runtime embedding parameter. - shard_prefix = f"{prefix}.shard_" - if any(name.startswith(shard_prefix) for name in ignored_layers): - return Qwen4ExpPLEUnquantizedEmbeddingMethod() - if not quant_config.is_checkpoint_fp8_serialized: - raise NotImplementedError( - "Qwen4Exp PLE embedding only supports serialized FP8 checkpoints" - ) - return Qwen4ExpPLEFp8EmbeddingMethod() - - def apply( - self, - layer: nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - raise NotImplementedError("PLE weights only support embedding lookup") - - def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor: - return F.embedding(input_, layer.weight) - - @abstractmethod - def dequantize( - self, - layer: nn.Module, - embeddings: torch.Tensor, - output_dtype: torch.dtype, - ) -> torch.Tensor: - """Convert looked-up PLE rows to the activation dtype.""" - raise NotImplementedError - - -class Qwen4ExpPLEUnquantizedEmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): - """Unquantized PLE embedding storage and lookup semantics.""" - - def create_weights( - self, - layer: Qwen4ExpPLEEmbedding, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ) -> None: - del input_size, output_size - weight = nn.Parameter( - layer.allocate_embedding_weight( - sum(output_partition_sizes), - input_size_per_partition, - params_dtype, - ), - requires_grad=False, - ) - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) - set_weight_attrs(weight, extra_weight_attrs) - layer.register_parameter("weight", weight) - - def dequantize( - self, - layer: nn.Module, - embeddings: torch.Tensor, - output_dtype: torch.dtype, - ) -> torch.Tensor: - del layer, output_dtype - return embeddings - - -class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): - """FP8 PLE embedding with one global checkpoint scale.""" - - def create_weights( - self, - layer: Qwen4ExpPLEEmbedding, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ) -> None: - del input_size, output_size, params_dtype - weight_loader = extra_weight_attrs.get("weight_loader") - weight = ModelWeightParameter( - data=layer.allocate_embedding_weight( - sum(output_partition_sizes), - input_size_per_partition, - torch.float8_e4m3fn, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) - layer.register_parameter("weight", weight) - - weight_scale = create_fp8_scale_parameter( - PerTensorScaleParameter, - output_partition_sizes, - input_size_per_partition, - None, - weight_loader, - scale_dtype=torch.float32, - ) - layer.register_parameter("weight_scale", weight_scale) - - def process_weights_after_loading(self, layer: nn.Module) -> None: - """Reject FP8 PLE checkpoints without a global scale.""" - sentinel = torch.finfo(torch.float32).min - if torch.any(layer.weight_scale == sentinel): - raise ValueError("FP8 PLE checkpoint is missing its global scale") - - def dequantize( - self, - layer: nn.Module, - embeddings: torch.Tensor, - output_dtype: torch.dtype, - ) -> torch.Tensor: - weight_scale = getattr(layer, "weight_scale", None) - if weight_scale is None: - raise RuntimeError("FP8 PLE embedding is missing its global scale") - if weight_scale.device != embeddings.device: - raise RuntimeError("FP8 PLE embedding scale must be on the output device") - return embeddings.to(output_dtype) * weight_scale.to(output_dtype) - - -class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding): - """PLE table allocated on the active model device.""" - - def allocate_embedding_weight( - self, - num_embeddings: int, - embedding_dim: int, - dtype: torch.dtype, - ) -> torch.Tensor: - """Allocate the complete PLE weight on the active device.""" - return torch.empty(num_embeddings, embedding_dim, dtype=dtype) - - def start_prefetch( - self, - hidden_states: torch.Tensor, - ngram_ids: torch.Tensor, - ) -> None: - """Resident embedding prefetch is a no-op.""" - return None - - def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor: - """Gather ETP inputs, look up embeddings, and select local rows.""" - slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0]) - gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) - embeddings = super().forward(gathered_ids) - return self._select_embeddings( - embeddings, - ngram_ids.shape[0], - slot_offset, - ) - - -@triton.jit -def _lookup_ple_embedding_from_pinned_kernel( - weight_ptr, - ids_ptr, - output_ptr, - embedding_dim, - tp_vocab_start, - tp_vocab_end, - BLOCK_D: tl.constexpr, -): - """Look up TP-owned PLE rows through a CUDA view of pinned host memory.""" - row_id = tl.program_id(0) - global_idx = tl.load(ids_ptr + row_id) - in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end) - local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0) - offsets = tl.arange(0, BLOCK_D) - store_mask = offsets < embedding_dim - load_mask = store_mask & in_range - values = tl.load( - weight_ptr + local_idx * embedding_dim + offsets, - mask=load_mask, - other=0.0, - ) - tl.store( - output_ptr + row_id * embedding_dim + offsets, - values, - mask=store_mask, - ) - - -class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): - """PLE table loaded into pinned CPU memory and looked up through UVA.""" - - supports_prefetch: ClassVar[bool] = True - - def __init__( - self, - num_embeddings: int, - embedding_dim: int, - *, - params_dtype: torch.dtype, - padding_size: int, - prefix: str, - embedding_method: Qwen4ExpPLEEmbeddingMethod, - num_ngram_heads: int = 1, - max_total_tokens: int = 0, - data_parallel_rank: int = 0, - ) -> None: - if not is_uva_available(): - raise RuntimeError("Engram CPU offload requires UVA support") - super().__init__( - num_embeddings, - embedding_dim, - params_dtype=params_dtype, - padding_size=padding_size, - prefix=prefix, - embedding_method=embedding_method, - num_ngram_heads=num_ngram_heads, - max_total_tokens=max_total_tokens, - data_parallel_rank=data_parallel_rank, - ) - self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) - self._block_d = triton.next_power_of_2(self.embedding_dim) - self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device) - self._prefetch_buffer = torch.empty( - max_total_tokens * self.etp_data_parallel_size, - num_ngram_heads, - self.embedding_dim, - dtype=self.weight.dtype, - device=self._uva_weight.device, - ) - self._output_dim = num_ngram_heads * self.embedding_dim - - def allocate_embedding_weight( - self, - num_embeddings: int, - embedding_dim: int, - dtype: torch.dtype, - ) -> torch.Tensor: - """Allocate the complete PLE weight directly in pinned CPU memory.""" - return torch.empty( - num_embeddings, - embedding_dim, - dtype=dtype, - device="cpu", - pin_memory=True, - ) - - def _lookup( - self, - input_ids: torch.Tensor, - output: torch.Tensor | None = None, - ) -> torch.Tensor: - """Look up local ETP rows while preserving the weight storage dtype.""" - expected_shape = (*input_ids.shape, self.embedding_dim) - if output is None: - output = torch.empty( - expected_shape, - dtype=self.weight.dtype, - device=input_ids.device, - ) - elif ( - tuple(output.shape) != expected_shape - or output.dtype != self.weight.dtype - or output.device != input_ids.device - ): - raise ValueError( - "PLE prefetch output must match the input shape, weight dtype, " - "and input device" - ) - - flat_ids = input_ids.reshape(-1).long() - if flat_ids.numel(): - _lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)]( - self._uva_weight, - flat_ids, - output, - self.embedding_dim, - self.shard_indices.org_vocab_start_index, - self.shard_indices.org_vocab_end_index, - BLOCK_D=self._block_d, - ) - return output - - def _reduce_etp_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor: - """Combine pinned lookup results owned by different ETP ranks.""" - if self.tp_size == 1: - return embeddings - assert self.parallel_group is not None - if embeddings.dtype in (torch.float8_e4m3fn, torch.float8_e5m2): - # Each vocabulary row has one owner, so reduce the raw FP8 bytes. - reduced = self.parallel_group.all_reduce(embeddings.view(torch.int8)) - return reduced.view(embeddings.dtype) - return self.parallel_group.all_reduce(embeddings) - - @eager_break_during_capture - def start_prefetch( - self, - hidden_states: torch.Tensor, - ngram_ids: torch.Tensor, - ) -> None: - """Gather ETP IDs and launch their UVA lookup on the side stream.""" - slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0]) - gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) - active_output = self._prefetch_buffer[: gathered_ids.shape[0]] - prefetch_stream = self._prefetch_stream - prefetch_stream.wait_stream(torch.cuda.current_stream()) - gathered_ids.record_stream(prefetch_stream) - with torch.cuda.stream(prefetch_stream): - self._lookup(gathered_ids, output=active_output) - - @eager_break_during_capture - def _finalize_prefetch( - self, - prefetch_output: torch.Tensor, - output: torch.Tensor, - ) -> None: - """Join the side stream, reduce ETP shards, and select local rows.""" - torch.cuda.current_stream().wait_stream(self._prefetch_stream) - slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0]) - active_output = prefetch_output[: slot_size * self.etp_data_parallel_size] - embeddings = self._reduce_etp_embeddings(active_output) - embeddings = self._select_embeddings( - embeddings, - output.shape[0], - slot_offset, - ) - output.copy_(embeddings.flatten(-2)) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Finish the pinned lookup into graph-owned output storage.""" - output = self._prefetch_buffer.new_empty( - (hidden_states.shape[0], self._output_dim) - ) - self._finalize_prefetch(self._prefetch_buffer, output) - return output +__all__ = [ + "Qwen4ExpPLEDeviceEmbedding", + "Qwen4ExpPLEEmbedding", + "Qwen4ExpPLEEmbeddingMethod", + "Qwen4ExpPLEFp8EmbeddingMethod", + "Qwen4ExpPLEPinnedHostEmbedding", + "Qwen4ExpPLEUnquantizedEmbeddingMethod", + "Qwen4ExpNGramEmbedding", +] class Qwen4ExpNGramEmbedding(nn.Module): @@ -716,6 +224,12 @@ class Qwen4ExpNGramEmbedding(nn.Module): max_total_tokens=max_total_tokens, data_parallel_rank=data_parallel_rank, ) + if self.ngram_embedding.supports_prefetch: + # The side-stream lookup outlives eager-break args, whose + # graph-pool storage later segments may reuse. + self._prefetch_ids = torch.empty( + max_total_tokens, self.ngram_heads, dtype=torch.long + ) weight = self.ngram_embedding.weight logger.info( "Initialized PLE embedding %s: quantization_method=%s, " @@ -864,6 +378,7 @@ class Qwen4ExpNGramEmbedding(nn.Module): input_ids, query_start_loc, ngram_context, + output=self._prefetch_ids[: input_ids.numel()], ) embedding.start_prefetch(hidden_states, ngram_ids) diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py index 077dea91f0..9b28ed6dca 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py @@ -383,6 +383,7 @@ def _prefill_logits( page_table: torch.Tensor, query_start_loc: torch.Tensor, visible_blocks: torch.Tensor, + logits_workspace: torch.Tensor, max_query_len: int, logits_width: int, query_offset: int, @@ -392,10 +393,14 @@ def _prefill_logits( assert visible_blocks.shape == (q.shape[0],) assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0] assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1] + assert logits_workspace.is_contiguous() + assert logits_workspace.numel() >= num_queries * logits_width - logits = torch.empty( - (num_queries, logits_width), dtype=torch.float32, device=q.device + # slice from pre-allocated workspace + logits = logits_workspace[: num_queries * logits_width].view( + num_queries, logits_width ) + # tuned on GB300 if k_cache.dtype == torch.float8_e4m3fn: TILE_R, STAGES, num_warps = 32, 2, 8 @@ -610,8 +615,13 @@ def qsa_select_paged_prefill( logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1]) # chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB + # always allocate the worst case to avoid memory fragmentation. max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4)) + + budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4) + logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32) + topk_workspace = torch.empty( (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device ) @@ -625,6 +635,7 @@ def qsa_select_paged_prefill( page_table, query_start_loc, visible_blocks, + logits_workspace, max_query_len, logits_width, query_offset=query_start, diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index e480142a22..b6da89a863 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -103,7 +103,7 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend): return "QWEN4_EXP_QSA_TRITON" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # QSA consumes manager pages directly and does not use FA4 paged attention. return [MultipleOf(16)] diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 8d7b5aa9b6..98f8a8b2f3 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -664,9 +664,46 @@ class MultiModalDataParser: assert_never(video) + def _parse_audio_embedding_data( + self, data: dict[str, torch.Tensor] + ) -> DictEmbeddingItems: + counts = data.get("audio_num_tokens") + if ( + not isinstance(counts, torch.Tensor) + or counts.dtype not in (torch.int32, torch.int64) + or counts.ndim not in (1, 2) + or (counts.ndim == 2 and counts.shape[1] != 1) + or not torch.all(counts > 0) + ): + raise ValueError( + "audio_num_tokens must contain one positive integer per audio" + ) + + counts = counts.flatten() + if "audio_embeds" in data: + embeds = data["audio_embeds"] + if len(embeds) != len(counts) or any( + len(embedding) != count for embedding, count in zip(embeds, counts) + ): + raise ValueError("audio_num_tokens does not match audio_embeds") + + required, optional = self.embedding_field_sets("audio") + return DictEmbeddingItems( + {**data, "audio_num_tokens": counts}, + modality="audio", + required_fields=required, + optional_fields=optional, + fields_factory=lambda _: { + "audio_embeds": MultiModalFieldConfig.batched("audio"), + "audio_num_tokens": MultiModalFieldConfig.batched( + "audio", keep_on_cpu=True + ), + }, + ) + def _parse_audio_data( self, - data: ModalityData[AudioItem], + data: dict[str, torch.Tensor] | ModalityData[AudioItem], ) -> ModalityDataItems[Any, Any] | None: if data is None: return None @@ -674,6 +711,11 @@ class MultiModalDataParser: if self.is_embeddings(data): return AudioEmbeddingItems(data, self.expected_hidden_size) + if isinstance(data, dict) and "audio_num_tokens" in self.embedding_fields.get( + "audio", {} + ): + return self._parse_audio_embedding_data(data) + data_items: list[AudioItem | None] if ( (is_list_of(data, float) and len(data) > 0) diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index 359c43700d..db1cffefd6 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -26,14 +26,14 @@ from vllm.utils.jsontree import JSONTree, json_map_leaves from vllm.utils.mistral import is_mistral_tokenizer if TYPE_CHECKING: - from transformers.configuration_utils import PretrainedConfig + from transformers.configuration_utils import PreTrainedConfig from transformers.feature_extraction_utils import BatchFeature from transformers.processing_utils import ProcessorMixin from vllm.config import ModelConfig from vllm.renderers import TokenizeParams else: - PretrainedConfig = object + PreTrainedConfig = object BatchFeature = object ProcessorMixin = object @@ -123,7 +123,7 @@ class TimingContext: _T = TypeVar("_T") -_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig) +_C = TypeVar("_C", bound=PreTrainedConfig, default=PreTrainedConfig) _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) @@ -148,7 +148,7 @@ class InputProcessingContext: return self.tokenizer @overload - def get_hf_config(self, /) -> PretrainedConfig: ... + def get_hf_config(self, /) -> PreTrainedConfig: ... @overload def get_hf_config( @@ -163,7 +163,7 @@ class InputProcessingContext: /, ) -> Any: """Get the HuggingFace configuration - (`transformers.PretrainedConfig`) of the model, + (`transformers.PreTrainedConfig`) of the model, additionally checking its type. Raises: @@ -171,9 +171,9 @@ class InputProcessingContext: """ if typ is None: - from transformers.configuration_utils import PretrainedConfig + from transformers.configuration_utils import PreTrainedConfig - typ = PretrainedConfig + typ = PreTrainedConfig hf_config = self.model_config.hf_config if not isinstance(hf_config, typ): @@ -371,7 +371,7 @@ class BaseProcessingInfo: def get_tokenizer(self) -> TokenizerLike: return self.ctx.get_tokenizer() - def get_hf_config(self) -> PretrainedConfig: + def get_hf_config(self) -> PreTrainedConfig: return self.ctx.get_hf_config() def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index b8d7dc6e46..215fa0d9bd 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -82,6 +82,12 @@ class ParserEngineConfig: tool_args_json: bool = True + # The tool region body is a single JSON array of ``{"name", "arguments"}`` + # objects (one marker → N calls, no closing terminal). The engine splits + # the array into one tool call per element; the array's ``]`` ends the + # region and any trailing text becomes content. Used by Granite. + tool_call_body_array: bool = False + arg_structural_chars: frozenset[str] | None = None # Special tokens exempt from auto-drop but not state-machine terminals. diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 8e10a1b5ef..7353a6016b 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -13,6 +13,8 @@ from vllm.parser.deepseek_v41 import DeepSeekV41Parser from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser from vllm.parser.glm47_moe import Glm47MoeParser +from vllm.parser.granite import GraniteParser +from vllm.parser.granite_thinking import GraniteThinkingParser from vllm.parser.inkling import InklingParser from vllm.parser.kimi_k2 import KimiK2Parser from vllm.parser.minimax_m2 import MinimaxM2Parser @@ -46,6 +48,16 @@ from vllm.parser.seed_oss import SeedOssParser Gemma4ParserToolAdapter, ) = make_adapters(Gemma4Parser) +( + GraniteParserReasoningAdapter, + GraniteParserToolAdapter, +) = make_adapters(GraniteParser) + +( + GraniteThinkingParserReasoningAdapter, + GraniteThinkingParserToolAdapter, +) = make_adapters(GraniteThinkingParser) + ( NemotronV3ParserReasoningAdapter, NemotronV3ParserToolAdapter, diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 21d053ef28..3fd1ecdcb6 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -190,6 +190,12 @@ class StreamingParserEngine: self._args_in_string: bool = False self._args_escape_next: bool = False + def _reset_array_state(self) -> None: + # State for ``tool_call_body_array``: whether the opening ``[`` has been + # seen, and whether a ``{...}`` element is currently being streamed. + self._array_started: bool = False + self._array_in_element: bool = False + def reset(self, initial_state: ParserState | None = None) -> None: """Reset mutable state for reuse across requests. @@ -213,6 +219,7 @@ class StreamingParserEngine: self._message_header_token_count = 0 self._in_skipped_tool_span = False self._reset_args_state() + self._reset_array_state() def feed( self, @@ -306,7 +313,12 @@ class StreamingParserEngine: ParserState.TOOL_NAME, ParserState.TOOL_BETWEEN, ): - if self.tool_index >= 0: + # In array mode a closed element has already emitted its + # TOOL_CALL_END; only a truncated (still-open) element needs one. + end_open_call = ( + self._array_in_element if self.config.tool_call_body_array else True + ) + if end_open_call and self.tool_index >= 0: events.append( SemanticEvent( EventType.TOOL_CALL_END, @@ -474,6 +486,8 @@ class StreamingParserEngine: self._message_header_token_count += token_count return [] if self.state == ParserState.TOOL_ARGS: + if self.config.tool_call_body_array: + return self._feed_array_text(text) if self.config.tool_args_json: return self._feed_args_text(text) return [ @@ -564,6 +578,8 @@ class StreamingParserEngine: self._args_in_string = False self._args_escape_next = False self._args_safe_end = 0 + if self.config.tool_call_body_array: + self._reset_array_state() return events @@ -634,3 +650,88 @@ class StreamingParserEngine: tool_index=self.tool_index, ) ] + + # ── JSON-array tool body (tool_call_body_array) ───────────────────── + + def _feed_array_text(self, text: str) -> list[SemanticEvent]: + """Split a JSON-array tool body into one tool call per element. + + Granite emits ``[{"name":..,"arguments":{..}}, ...]`` after the tool + marker. Element interiors are streamed through the string/brace-aware + :meth:`_feed_args_char`, so each element behaves exactly like a + single-object tool body; the array's ``]`` ends the region and any + trailing text becomes content. + """ + events: list[SemanticEvent] = [] + for i, ch in enumerate(text): + if self.state != ParserState.TOOL_ARGS: + # The array closed mid-chunk; the rest of the text is content. + events.extend(self._emit_for_state(text[i:])) + break + events.extend(self._feed_array_char(ch)) + return events + + def _feed_array_char(self, ch: str) -> list[SemanticEvent]: + if self._array_in_element: + events = self._feed_args_char(ch) + # Element object closed: top-level '}' with all braces balanced. + if ( + ch == "}" + and not self._args_in_string + and not self._args_escape_next + and self._args_brace_depth == 0 + ): + events.extend(self._end_array_element()) + return events + + # Between elements only structural array characters are expected. + if ch in " \t\r\n": + return [] + if not self._array_started: + if ch == "[": + self._array_started = True + return [] + return self._bail_array_to_content(ch) + if ch == ",": + return [] + if ch == "{": + return self._start_array_element(ch) + if ch == "]": + self.state = ParserState.CONTENT + self._reset_array_state() + return [] + return self._bail_array_to_content(ch) + + def _start_array_element(self, ch: str) -> list[SemanticEvent]: + self.tool_index += 1 + self._reset_args_state() + self._array_in_element = True + events = [SemanticEvent(EventType.TOOL_CALL_START, tool_index=self.tool_index)] + events.extend(self._feed_args_char(ch)) + return events + + def _end_array_element(self) -> list[SemanticEvent]: + self._array_in_element = False + events: list[SemanticEvent] = [] + # Flush the held closing '}' so the slot body is the full element. + if self._args_buffer: + events.append( + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=self._args_buffer, + tool_index=self.tool_index, + ) + ) + self._args_buffer = "" + self._args_safe_end = 0 + events.append( + SemanticEvent(EventType.TOOL_CALL_END, tool_index=self.tool_index) + ) + return events + + def _bail_array_to_content(self, ch: str) -> list[SemanticEvent]: + # Marker not followed by a JSON array (malformed) — revert to content + # so the remaining tokens are emitted as plain text, no tool call. + self.state = ParserState.CONTENT + self._reset_array_state() + return self._emit_for_state(ch) diff --git a/vllm/parser/gemma4.py b/vllm/parser/gemma4.py index 185610126f..2a6246041d 100644 --- a/vllm/parser/gemma4.py +++ b/vllm/parser/gemma4.py @@ -120,7 +120,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict: result[key] = "" break - if args_str[i : i + _DELIM_LEN] == STRING_DELIM: + if args_str.startswith(STRING_DELIM, i): i += _DELIM_LEN val_start = i end_pos = args_str.find(STRING_DELIM, i) @@ -139,7 +139,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict: obj_start = i + 1 i += 1 while i < n and depth > 0: - if args_str[i : i + _DELIM_LEN] == STRING_DELIM: + if args_str.startswith(STRING_DELIM, i): # Skip over string contents to avoid counting { inside strings i += _DELIM_LEN next_delim = args_str.find(STRING_DELIM, i) @@ -162,7 +162,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict: arr_start = i + 1 i += 1 while i < n and depth > 0: - if args_str[i : i + _DELIM_LEN] == STRING_DELIM: + if args_str.startswith(STRING_DELIM, i): i += _DELIM_LEN next_delim = args_str.find(STRING_DELIM, i) i = n if next_delim == -1 else next_delim + _DELIM_LEN @@ -213,7 +213,7 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list: if i >= n: break - if arr_str[i : i + _DELIM_LEN] == STRING_DELIM: + if arr_str.startswith(STRING_DELIM, i): i += _DELIM_LEN end_pos = arr_str.find(STRING_DELIM, i) if end_pos == -1: @@ -227,7 +227,7 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list: obj_start = i + 1 i += 1 while i < n and depth > 0: - if arr_str[i : i + _DELIM_LEN] == STRING_DELIM: + if arr_str.startswith(STRING_DELIM, i): i += _DELIM_LEN nd = arr_str.find(STRING_DELIM, i) i = nd + _DELIM_LEN if nd != -1 else n @@ -247,7 +247,7 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list: sub_start = i + 1 i += 1 while i < n and depth > 0: - if arr_str[i : i + _DELIM_LEN] == STRING_DELIM: + if arr_str.startswith(STRING_DELIM, i): i += _DELIM_LEN nd = arr_str.find(STRING_DELIM, i) i = nd + _DELIM_LEN if nd != -1 else n diff --git a/vllm/parser/granite.py b/vllm/parser/granite.py new file mode 100644 index 0000000000..afa2021225 --- /dev/null +++ b/vllm/parser/granite.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Granite parser for JSON-array tool calls. + +Granite 3.0/3.1 emit tool calls as a JSON array following a marker:: + + <|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0 + [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1 + +``<|tool_call|>`` is a single special token; ```` is plain text. +The array may hold several calls, has no closing marker (the ``]`` ends it), +and any surrounding prose is content. The engine's ``tool_call_body_array`` +mode splits the array into one call per element and extracts each name via the +name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object +out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning. +""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING + +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.parser.utils import extract_json_member_object + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token) +TOOL_STRING = "" # granite 3.1 (plain text) + + +def _granite_arg_converter(raw_args: str, partial: bool) -> str: + """Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}`` + array element (see :func:`extract_json_member_object`).""" + span = extract_json_member_object(raw_args, "arguments") + if span is None: + return "" if partial else "{}" + return span + + +@functools.cache +def granite_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="granite", + initial_state=ParserState.CONTENT, + terminals={ + "TOOL_TOKEN": TOOL_TOKEN, + "TOOL_STRING": TOOL_STRING, + }, + token_id_terminals={ + "TOOL_TOKEN": TOOL_TOKEN, + }, + transitions={ + # The marker only switches into the tool region; the array splitter + # emits TOOL_CALL_START per element (empty events here). + (ParserState.CONTENT, "TOOL_TOKEN"): Transition( + ParserState.TOOL_ARGS, + (), + ), + (ParserState.CONTENT, "TOOL_STRING"): Transition( + ParserState.TOOL_ARGS, + (), + ), + }, + arg_converter=_granite_arg_converter, + tool_call_body_array=True, + stream_arg_deltas=True, + ) + + +class GraniteParser(ParserEngine): + CONFIG_NAME = "granite" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("parser_engine_config", granite_config()) + super().__init__(tokenizer, tools, **kwargs) diff --git a/vllm/parser/granite_thinking.py b/vllm/parser/granite_thinking.py new file mode 100644 index 0000000000..3ab3c8c0af --- /dev/null +++ b/vllm/parser/granite_thinking.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Granite 4.2 thinking parser. + +Granite 4.2 uses the same tool call and reasoning format as Nemotron V3 +(````/```` + ```` XML). This config reuses +:class:`NemotronV3Parser` with two Granite-specific behaviors: + +1. Strip leading newlines from content — the Granite chat template writes + ``\\n\\n``, so the first content character is always ``\\n``. +2. Inherit the Nemotron V3 reasoning-to-content swap for + ``enable_thinking=False`` / ``force_nonempty_content=True``. +""" + +from __future__ import annotations + +import dataclasses +import functools +from typing import TYPE_CHECKING + +from vllm.parser.nemotron_v3 import NemotronV3Parser, nemotron_v3_config +from vllm.parser.qwen3 import Qwen3Parser + +if TYPE_CHECKING: + from vllm.entrypoints.generate.base.protocol import DeltaMessage + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.parser.engine.parser_engine import SemanticEvent + from vllm.parser.engine.parser_engine_config import ParserEngineConfig + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + + +@functools.cache +def granite_thinking_config(thinking: bool = True) -> ParserEngineConfig: + return dataclasses.replace( + nemotron_v3_config(thinking=thinking), + name="granite_thinking_parser", + ) + + +class GraniteThinkingParser(NemotronV3Parser): + """Granite 4.2 parser: same format as Nemotron V3, with + leading-newline stripping on content after ````. + + The ``reasoning_ended`` property is overridden to delay the + reasoning-to-content transition reported to + :class:`~vllm.parser.abstract_parser.DelegatingParser` until the + template's leading newline has been consumed. Without this delay, + ``DelegatingParser.parse_delta`` passes newline tokens through a + raw pass-through path that bypasses the engine's + ``_events_to_delta`` (and therefore our stripping logic). + """ + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + thinking = chat_kwargs.get("enable_thinking", True) + kwargs.pop("parser_engine_config", None) + Qwen3Parser.__init__( + self, + tokenizer, + tools, + parser_engine_config=granite_thinking_config(thinking=thinking), + **kwargs, + ) + self._streamed_reasoning: list[str] = [] + self._content_started = False + + @property + def reasoning_ended(self) -> bool: + # Internal engine logic uses _reasoning_ended directly; this + # property is only read by DelegatingParser (via the adapter's + # has_engine_confirmed_reasoning_end) to decide when to stop + # routing deltas through extract_reasoning_streaming. + if not self._reasoning_ended: + return False + return self._content_started + + def _reset(self, initial_state=None) -> None: + super()._reset(initial_state=initial_state) + self._content_started = False + + def _events_to_delta( + self, + events: list[SemanticEvent], + finished: bool = False, + ) -> DeltaMessage | None: + delta = super()._events_to_delta(events, finished=finished) + if ( + delta is not None + and delta.content is not None + and not self._content_started + ): + stripped = delta.content.lstrip("\n") + if not stripped: + delta.content = None + if delta.reasoning is None and not delta.tool_calls: + return None + else: + self._content_started = True + delta.content = stripped + return delta + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + reasoning, content = Qwen3Parser.extract_reasoning(self, model_output, request) + + # Track whether content was truly absent (None) vs present + # but whitespace-only — the swap should only fire for absent. + content_was_absent = content is None + + if content is not None: + content = content.lstrip("\n") or None + + # Only swap when content was truly absent from model output + # (truncated reasoning or max_tokens cut at ), not + # when it was present but stripped to empty (newlines-only). + if self._should_force_content(request) and content_was_absent: + reasoning, content = content, reasoning + + return reasoning, content diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 8c944733e3..5748bd6653 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -38,6 +38,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ) from vllm.entrypoints.openai.parser.harmony_utils import ( extract_function_from_recipient, + get_encoding, get_streamable_parser_for_assistant, is_function_recipient, ) @@ -114,6 +115,9 @@ class HarmonyParser(DelegatingParser): self._next_tool_call_index = 0 self._num_processed_messages = 0 + self._num_counted_tokens = 0 + self._num_reasoning_tokens = 0 + # For error recovery self._current_message_tokens: list[int] = [] @@ -348,9 +352,7 @@ class HarmonyParser(DelegatingParser): else: self._current_message_tokens.append(token_id) - if channel == "analysis" or ( - channel == "commentary" and recipient is not None - ): + if self._is_reasoning_token(token_id, channel, recipient): reasoning_token_count += 1 segments.append( @@ -364,11 +366,35 @@ class HarmonyParser(DelegatingParser): # TODO: Optionally merge and suppress empty Segments + self._num_counted_tokens += len(token_ids) + self._num_reasoning_tokens += reasoning_token_count return ChunkResult( segments=segments, reasoning_token_count=reasoning_token_count, ) + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + if len(token_ids) == self._num_counted_tokens: + return self._num_reasoning_tokens + + parser = get_streamable_parser_for_assistant() + count = 0 + for token_id in token_ids: + parser.process(token_id) + recipient = self._normalize_recipient(parser.current_recipient) + if self._is_reasoning_token(token_id, parser.current_channel, recipient): + count += 1 + return count + + @staticmethod + def _is_reasoning_token( + token_id: int, channel: str | None, recipient: str | None + ) -> bool: + is_reasoning_channel = channel == "analysis" or ( + channel == "commentary" and recipient is not None + ) + return is_reasoning_channel and not get_encoding().is_special_token(token_id) + def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py index 2081fb2784..46be6f02b6 100644 --- a/vllm/parser/inkling.py +++ b/vllm/parser/inkling.py @@ -37,6 +37,7 @@ from vllm.parser.engine.parser_engine_config import ( ParserState, Transition, ) +from vllm.parser.utils import extract_json_member_object if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike @@ -62,84 +63,6 @@ INKLING_SPECIAL_TOKENS = ( END_MESSAGE, ) -_WS = " \t\r\n" - - -def _scan_json_value(raw: str, start: int) -> int | None: - """Return the end index (exclusive) of the JSON object starting at - ``raw[start]``, or ``None`` when the object is still unterminated.""" - depth = 0 - in_string = False - escape = False - for i in range(start, len(raw)): - ch = raw[i] - if escape: - escape = False - continue - if in_string: - if ch == "\\": - escape = True - elif ch == '"': - in_string = False - continue - if ch == '"': - in_string = True - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - return i + 1 - return None - - -def _args_value_span(raw: str) -> str | None: - """Extract the raw text span of the top-level ``"args"`` value from a - (possibly incomplete) ``{"name":...,"args":{...}}`` wrapper. - - Returns the verbatim substring (prefix-stable across growing input, - which the engine's argument-delta diffing relies on), possibly an - unterminated object prefix; ``None`` when the value has not started. - Raises ``ValueError`` when the value is not a JSON object. - """ - depth = 0 - in_string = False - escape = False - string_start = -1 - last_string: str | None = None - for i, ch in enumerate(raw): - if escape: - escape = False - continue - if in_string: - if ch == "\\": - escape = True - elif ch == '"': - in_string = False - if depth == 1: - last_string = raw[string_start + 1 : i] - continue - if ch == '"': - in_string = True - string_start = i - elif ch == ":" and depth == 1 and last_string == "args": - value_start = i + 1 - while value_start < len(raw) and raw[value_start] in _WS: - value_start += 1 - if value_start >= len(raw): - return None - if raw[value_start] != "{": - raise ValueError("Inkling tool call args must be a JSON object") - value_end = _scan_json_value(raw, value_start) - if value_end is None: - return raw[value_start:] - return raw[value_start:value_end] - elif ch in "{[": - depth += 1 - elif ch in "}]": - depth -= 1 - return None - def _inkling_arg_converter(raw_args: str, partial: bool) -> str: """Carve the ``args`` object out of the tool-call JSON wrapper. @@ -150,19 +73,10 @@ def _inkling_arg_converter(raw_args: str, partial: bool) -> str: converter, ``_compute_arg_delta`` streams the wrapper verbatim into the OpenAI ``arguments`` field (``converter is None -> raw delta``, unconditionally; ``stream_arg_deltas=False`` does not stop it). - - Why a hand-rolled scanner instead of (partial) ``json.loads`` + - ``json.dumps``: the engine diffs successive converter outputs and - requires each to extend the previous one (``startswith``); a - violation silently drops argument deltas. Re-serialization changes - whitespace and closes unterminated structures differently across - ticks, so the only prefix-stable output is a verbatim substring of - the input. The scanner is also string/escape-aware so an ``"args"`` - literal inside the name or a string value cannot mislead it, and it - still recovers a partial span when EOS truncates the wrapper (where - ``json.loads`` would fail). + :func:`extract_json_member_object` returns a prefix-stable verbatim + substring, which the engine's argument-delta diffing requires. """ - span = _args_value_span(raw_args) + span = extract_json_member_object(raw_args, "args") if span is None: # No args value yet (streaming) or none at all (treat as empty). return "" if partial else "{}" @@ -253,11 +167,12 @@ def inkling_config() -> ParserEngineConfig: (EventType.REASONING_END, EventType.TOOL_CALL_START), ), } - # Block-end terminals behave identically regardless of label. A - # closed tool block returns to CONTENT (Inkling has no section wrapper; + # A closed tool block returns to CONTENT (Inkling has no section wrapper; # blocks of any kind may follow), which also keeps the block-kind # and role tokens out of the engine's tool-terminal set so the # skip_tool_parsing reasoning pass still classifies reasoning. + # In a reasoning block, THINK_END defers REASONING_END to the next block + # opener so the function-name header is not emitted as content. for end in ("THINK_END", "END_SAMPLING"): transitions[(ParserState.CONTENT, end)] = Transition( ParserState.CONTENT, @@ -265,7 +180,7 @@ def inkling_config() -> ParserEngineConfig: ) transitions[(ParserState.REASONING, end)] = Transition( ParserState.CONTENT, - (EventType.REASONING_END,), + () if end == "THINK_END" else (EventType.REASONING_END,), ) transitions[(ParserState.TOOL_ARGS, end)] = Transition( ParserState.CONTENT, diff --git a/vllm/parser/utils.py b/vllm/parser/utils.py index 51382cd290..9194b7ef18 100644 --- a/vllm/parser/utils.py +++ b/vllm/parser/utils.py @@ -63,3 +63,88 @@ def count_history_tool_calls( return 0 return count_response_history_tool_calls(request_input) + + +_JSON_WS = " \t\r\n" + + +def scan_json_object_end(raw: str, start: int) -> int | None: + """Return the end index (exclusive) of the JSON object starting at + ``raw[start]`` (which must be ``{``), or ``None`` when it is still + unterminated. String- and escape-aware so braces inside strings do not + mislead the depth count.""" + depth = 0 + in_string = False + escape = False + for i in range(start, len(raw)): + ch = raw[i] + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return i + 1 + return None + + +def extract_json_member_object(raw: str, key: str) -> str | None: + """Extract the raw text span of the top-level object-valued member ``key`` + from a (possibly incomplete) ``{...}`` JSON wrapper. + + Returns the verbatim substring (prefix-stable as the input grows, which + the engine's argument-delta diffing relies on), possibly an unterminated + object prefix; ``None`` when the value has not started. Raises + ``ValueError`` when the member is present but its value is not a JSON + object. A verbatim substring (not re-serialized JSON) is required because + the engine diffs successive converter outputs and expects each to extend + the previous one; the scanner is string/escape-aware so a matching literal + inside another value cannot mislead it, and it recovers a partial span when + the wrapper is truncated (where ``json.loads`` would fail).""" + depth = 0 + in_string = False + escape = False + string_start = -1 + last_string: str | None = None + for i, ch in enumerate(raw): + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + if depth == 1: + last_string = raw[string_start + 1 : i] + continue + if ch == '"': + in_string = True + string_start = i + elif ch == ":" and depth == 1 and last_string == key: + value_start = i + 1 + while value_start < len(raw) and raw[value_start] in _JSON_WS: + value_start += 1 + if value_start >= len(raw): + return None + if raw[value_start] != "{": + raise ValueError(f"member {key!r} must be a JSON object") + value_end = scan_json_object_end(raw, value_start) + if value_end is None: + return raw[value_start:] + return raw[value_start:value_end] + elif ch in "{[": + depth += 1 + elif ch in "}]": + depth -= 1 + return None diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 390c0dbfed..c9d57fd5d3 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -199,6 +199,15 @@ class CpuPlatform(Platform): if model_config is not None: model_config.disable_cascade_attn = True + # Import lazily: vllm.triton_utils imports vllm.platforms.current_platform, + # which is still being resolved while this platform class is loading. + from vllm.triton_utils import HAS_TRITON + + if cls.get_cpu_architecture() == CpuArchEnum.X86 and not HAS_TRITON: + logger.warning_once( + "Triton is not installed. triton-cpu is expected on x86 CPUs." + ) + cache_config = vllm_config.cache_config is_deepseek_v4 = ( @@ -395,11 +404,23 @@ class CpuPlatform(Platform): # Avoid inductor generates num_thread() and breaks the thread binding os.environ["TORCHINDUCTOR_CPP_DYNAMIC_THREADS"] = "1" - # For efficient conv state memory access. The C++ causal_conv1d - # kernels (VDPBF16PS, no AMX tiles) consume the SD layout on any - # AVX-512BF16 CPU, so apply it beyond AMX (e.g. AMD Zen5/Turin). - if torch.cpu._is_avx512_bf16_supported(): - os.environ["VLLM_SSM_CONV_STATE_LAYOUT"] = "SD" + # NIXL's Mamba descriptors require DS conv state storage. Select it + # before cache shapes are created, while preserving an explicit layout. + conv_state_layout_env = "VLLM_SSM_CONV_STATE_LAYOUT" + if conv_state_layout_env not in os.environ: + kv_transfer_config = vllm_config.kv_transfer_config + uses_nixl = kv_transfer_config is not None and any( + kv_transfer_config.has_connector(name) + for name in ( + "NixlConnector", + "NixlPullConnector", + "NixlPushConnector", + ) + ) + if uses_nixl: + os.environ[conv_state_layout_env] = "DS" + elif torch.cpu._is_avx512_bf16_supported(): + os.environ[conv_state_layout_env] = "SD" ld_preload_str = os.getenv("LD_PRELOAD", "") cpu_architecture = Platform.get_cpu_architecture() @@ -498,11 +519,11 @@ class CpuPlatform(Platform): return # reconcile attention and mamba page sizes - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) @classmethod def discover_numa_topology(cls) -> list[list[int]]: @@ -626,6 +647,60 @@ class CpuPlatform(Platform): except ImportError as e: logger.warning_once("Failed to import from vllm._C: %s", repr(e)) + @classmethod + def register_triton_kernel_overrides(cls) -> None: + """Fallback C++ implementations for core Triton kernels. + + Used when the Triton-CPU backend is unavailable. When Triton-CPU is + present, the Triton kernels can run natively and the overrides + would only shadow them, so registration is skipped. + """ + from vllm.triton_utils import HAS_TRITON + + if HAS_TRITON: + return + + from vllm.triton_utils.dispatcher import register_kernels + from vllm.utils import cpu_triton_utils as cpu_tl + + register_kernels( + { + "vllm.v1.worker.block_table.ComputeSlotMappingKernel.kernel": ( + cpu_tl._compute_slot_mapping_kernel_impl + ), + "vllm.v1.spec_decode.utils.eagle_step_slot_mapping_metadata_kernel": ( + cpu_tl._eagle_step_slot_mapping_metadata_kernel_impl + ), + "vllm.v1.spec_decode.utils.eagle_prepare_inputs_padded_kernel": ( + cpu_tl._eagle_prepare_inputs_padded_kernel_impl + ), + "vllm.v1.spec_decode.utils.eagle_prepare_next_token_padded_kernel": ( + cpu_tl._eagle_prepare_next_token_padded_kernel_impl + ), + "vllm.v1.spec_decode.utils.copy_and_expand_eagle_inputs_kernel": ( + cpu_tl._copy_and_expand_eagle_inputs_kernel_impl + ), + "vllm.v1.spec_decode.utils.copy_and_expand_dflash_inputs_kernel": ( + cpu_tl._copy_and_expand_dflash_inputs_kernel_impl + ), + "vllm.v1.sample.rejection_sampler.rejection_greedy_sample_kernel": ( + cpu_tl._rejection_greedy_sample_kernel_impl + ), + "vllm.v1.sample.rejection_sampler.rejection_random_sample_kernel": ( + cpu_tl._rejection_random_sample_kernel_impl + ), + "vllm.v1.sample.rejection_sampler.expand_kernel": ( + cpu_tl._expand_kernel_impl + ), + "vllm.v1.sample.rejection_sampler.sample_recovered_tokens_kernel": ( + cpu_tl._sample_recovered_tokens_kernel_impl + ), + "vllm.v1.worker.mamba_utils.batch_memcpy_kernel": ( + cpu_tl._batch_memcpy_impl + ), + } + ) + @classmethod def pack_kv_cache( cls, diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index a46b607c8e..68c3aa7851 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -298,10 +298,6 @@ class CudaPlatformBase(Platform): def is_fully_connected(cls, device_ids: list[int]) -> bool: raise NotImplementedError - @classmethod - def log_warnings(cls): - pass - @classmethod def is_pin_memory_available(cls) -> bool: if in_wsl(): @@ -329,12 +325,6 @@ class CudaPlatformBase(Platform): parallel_config = vllm_config.parallel_config model_config = vllm_config.model_config - if ( - parallel_config.prefill_context_parallel_size > 1 - and parallel_config.data_parallel_size > 1 - ): - raise ValueError("PCP does not support data parallelism on CUDA yet.") - if parallel_config.worker_cls == "auto": parallel_config.worker_cls = "vllm.v1.worker.gpu_worker.Worker" @@ -1015,11 +1005,12 @@ class NvmlCudaPlatform(CudaPlatformBase): len(set(device_names)) > 1 and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" ): - logger.warning( + logger.warning_once( "Detected different devices in the system: %s. Please" " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to " "avoid unexpected behavior.", ", ".join(device_names), + scope="process", ) @@ -1071,5 +1062,3 @@ finally: pynvml.nvmlShutdown() CudaPlatform = NvmlCudaPlatform if nvml_available else NonNvmlCudaPlatform - -CudaPlatform.log_warnings() diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 13e5b55402..752024e2b2 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -3,6 +3,8 @@ import contextlib import enum import functools +import itertools +import math import os import platform import sys @@ -145,6 +147,10 @@ class Platform: # empty string means the device does not support ray ray_device_key: str = "" + @classmethod + def log_warnings(cls) -> None: + """Log platform-specific diagnostics after logging is configured.""" + # platform-agnostic way to specify the device control environment variable, # .e.g. CUDA_VISIBLE_DEVICES for CUDA. # hint: search for "get_visible_accelerator_ids_env_var" in @@ -364,6 +370,19 @@ class Platform: with contextlib.suppress(ImportError): import vllm._moe_C_stable_libtorch # noqa: F401 + @classmethod + def register_triton_kernel_overrides(cls) -> None: + """Override core Triton kernels with platform implementations. + + Platforms whose Triton kernels need replacement implementations + (e.g. CPU fallbacks when the Triton-CPU backend is unavailable) + override this method and call + :func:`vllm.triton_utils.dispatcher.register_kernels` + with their overrides. It runs during model runner init, before any + kernel launch. The default implementation does nothing. + """ + return + @classmethod def get_attn_backend_cls( cls, @@ -573,10 +592,10 @@ class Platform: pass @classmethod - def _find_non_ssm_backend( + def _find_non_ssm_backends( cls, vllm_config: "VllmConfig" - ) -> "type[AttentionBackend] | None": - """Find the first non-SSM attention backend from model layers.""" + ) -> "list[type[AttentionBackend]]": + """Distinct non-SSM attention backends, in layer order.""" from vllm.config.vllm import get_layers_from_vllm_config from vllm.model_executor.layers.attention_layer_base import ( AttentionLayerBase, @@ -586,11 +605,62 @@ class Platform: vllm_config, AttentionLayerBase, # type: ignore[type-abstract] ) + backends: list[type[AttentionBackend]] = [] for layer in attn_layers.values(): b = layer.get_attn_backend() - if not b.is_ssm(): - return b - return None + if not b.is_ssm() and b not in backends: + backends.append(b) + return backends + + @classmethod + def _preferred_block_size_for_backends( + cls, + backend_classes: "list[type[AttentionBackend]]", + default_block_size: int, + vllm_config: "VllmConfig", + ) -> int: + """Smallest block size every backend accepts. + + ``supports_block_size`` may be overridden to accept exact sizes only + (CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of + one supported size per backend, each checked against every backend. + + Raises: + ValueError: If the backends share no supported block size. + + """ + from vllm.config.vllm import set_current_vllm_config + from vllm.v1.attention.backend import MultipleOf + + # Backends may read the current config to decide a preference. + with set_current_vllm_config(vllm_config): + if len(backend_classes) == 1: + return backend_classes[0].get_preferred_block_size(default_block_size) + if all(b.supports_block_size(default_block_size) for b in backend_classes): + return default_block_size + # A backend declaring no sizes accepts any, so it contributes 1. + per_backend_sizes = [ + [ + s.base if isinstance(s, MultipleOf) else s + for s in b.get_supported_kernel_block_sizes() + ] + or [1] + for b in backend_classes + ] + candidates = sorted( + {math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)} + ) + for candidate in candidates: + if all(b.supports_block_size(candidate) for b in backend_classes): + return candidate + raise ValueError( + "The attention backends share no supported KV cache block size (" + + "; ".join( + f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" + for b in backend_classes + ) + + ")." + ) @classmethod def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: @@ -598,7 +668,6 @@ class Platform: For hybrid models, also aligns block_size with mamba page sizes. """ from vllm.config.cache import CacheConfig - from vllm.config.vllm import set_current_vllm_config cache_config = vllm_config.cache_config model_config = vllm_config.model_config @@ -607,34 +676,36 @@ class Platform: if not model_config: return - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - # Phase 1: Pick block size from backend (skip if user set --block-size) + # Phase 1: Pick a block size every attention backend supports (skip if + # user set --block-size). Models can mix backends with disjoint + # preferences, and a size from the first alone later fails + # select_common_block_size(). if not cache_config.user_specified_block_size: - with set_current_vllm_config(vllm_config): - preferred = backend_cls.get_preferred_block_size( - CacheConfig.DEFAULT_BLOCK_SIZE - ) + preferred = cls._preferred_block_size_for_backends( + backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config + ) if preferred != CacheConfig.DEFAULT_BLOCK_SIZE: logger.info( - "Setting kv cache block size to %d for %s backend.", + "Setting kv cache block size to %d for %s backend(s).", preferred, - backend_cls.get_name(), + "/".join(b.get_name() for b in backend_classes), ) cache_config.block_size = preferred # Phase 2: Align block/mamba sizes for hybrid models # (may override user settings). if model_config.is_hybrid: - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) # Phase 3: Align block/page sizes when multiple KV dtypes share the # block pool (e.g. nvfp4 primary + unquantized skip layers). # May override the user's --block-size. if cache_config.kv_cache_dtype_skip_layers: - cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls) + cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0]) @classmethod def _align_heterogeneous_kv_block_size( diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 9089d025e0..726673b29b 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib.metadata import os import platform from datetime import timedelta @@ -22,10 +23,14 @@ if TYPE_CHECKING: from vllm.config import VllmConfig from vllm.config.kernel import IrOpPriorityConfig from vllm.utils.argparse_utils import FlexibleArgumentParser + from vllm.v1.attention.backend import AttentionBackend from vllm.v1.attention.selector import AttentionSelectorConfig logger = init_logger(__name__) +_KV_CACHE_DTYPE_REASON = "kv_cache_dtype not supported" +_TURBOQUANT_LAYOUT_REASON = "no KV cache layout in common with TURBOQUANT" + try: from amdsmi import ( AmdSmiException, @@ -361,6 +366,25 @@ def get_cdna_version() -> int: return 0 +@cache +def get_rocm_version() -> tuple[int, ...] | None: + """Return the installed ROCm release as (major, minor, patch), or None.""" + # ROCm 10+ ships as the `rocm` pip SDK; older releases install to /opt/rocm. + try: + version = importlib.metadata.version("rocm") + except importlib.metadata.PackageNotFoundError: + rocm_path = os.environ.get("ROCM_PATH", "/opt/rocm") + try: + with open(os.path.join(rocm_path, ".info", "version")) as f: + version = f.read() + except OSError: + return None + match = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", version) + if match is None: + return None + return tuple(int(part) for part in match.groups() if part is not None) + + # Enable HIP online tuning early, before hipBLASLt initializes. # Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM. if ( @@ -485,6 +509,53 @@ def _get_backend_priorities( return backends +def _uses_turboquant(vllm_config: "VllmConfig | None") -> bool: + """Whether the run's KV cache dtype is one of the turboquant_* presets.""" + cache_config = getattr(vllm_config, "cache_config", None) + return cache_config is not None and str(cache_config.cache_dtype).startswith( + "turboquant_" + ) + + +def _shares_layout_with_turboquant(backend_class: type["AttentionBackend"]) -> bool: + """Whether this backend reads a KV cache layout TURBOQUANT reads too. + + A turboquant_* run keeps its boundary layers at the native dtype (see + TurboQuantConfig.get_boundary_skip_layers), so those layers pick a backend of + their own while every other layer picks TURBOQUANT. One layout has to serve + the whole worker, and get_supported_kv_cache_layouts() turns an empty + intersection into a hard error at engine startup. + """ + layouts = backend_class.supported_kv_cache_layouts() + turboquant_layouts = ( + AttentionBackendEnum.TURBOQUANT.get_class().supported_kv_cache_layouts() + ) + if layouts is None or turboquant_layouts is None: + return True + return not set(layouts).isdisjoint(turboquant_layouts) + + +def _get_invalid_reasons( + backend_class: type["AttentionBackend"], + device_capability: DeviceCapability, + attn_selector_config: "AttentionSelectorConfig", + *, + is_turboquant_run: bool, +) -> list[str]: + """Why this backend cannot serve the layer, empty when it can.""" + invalid_reasons = backend_class.validate_configuration( + device_capability=device_capability, + **attn_selector_config._asdict(), + ) + if ( + not invalid_reasons + and is_turboquant_run + and not _shares_layout_with_turboquant(backend_class) + ): + invalid_reasons = [_TURBOQUANT_LAYOUT_REASON] + return invalid_reasons + + class RocmPlatform(Platform): _enum = PlatformEnum.ROCM device_name: str = "rocm" @@ -591,12 +662,14 @@ class RocmPlatform(Platform): # TODO: Make this explicit in the selector in a future PR. if is_encoder_decoder and AttentionBackendEnum.ROCM_ATTN in backend_priorities: backend_priorities.remove(AttentionBackendEnum.ROCM_ATTN) + is_turboquant_run = _uses_turboquant(vllm_config) for priority, backend in enumerate(backend_priorities): try: - backend_class = backend.get_class() - invalid_reasons_i = backend_class.validate_configuration( - device_capability=device_capability, - **attn_selector_config._asdict(), + invalid_reasons_i = _get_invalid_reasons( + backend.get_class(), + device_capability, + attn_selector_config, + is_turboquant_run=is_turboquant_run, ) except ImportError: invalid_reasons_i = ["ImportError"] @@ -619,11 +692,16 @@ class RocmPlatform(Platform): # First try checking just the selected backend, if there is one. if selected_backend is not None: + # Keep lazy: vllm.config imports current_platform during initialization. + from vllm.config import get_current_vllm_config_or_none + + is_turboquant_run = _uses_turboquant(get_current_vllm_config_or_none()) try: - backend_class = selected_backend.get_class() - sel_invalid_reasons = backend_class.validate_configuration( - device_capability=device_capability, - **attn_selector_config._asdict(), + sel_invalid_reasons = _get_invalid_reasons( + selected_backend.get_class(), + device_capability, + attn_selector_config, + is_turboquant_run=is_turboquant_run, ) except ImportError: sel_invalid_reasons = ["ImportError"] @@ -633,21 +711,29 @@ class RocmPlatform(Platform): selected_backend.name, ) return selected_backend.get_path() - # Only tolerate the mismatch for turboquant_* KV-cache layers: - # boundary layers keep the native dtype (served by the selected - # backend) while turboquant_* layers need TURBOQUANT, so no single - # --attention-backend can serve every layer. For any other dtype - # the explicit selection is genuinely invalid -> fail loud. + # Only tolerate the mismatch when turboquant is in play: boundary + # layers keep the native dtype while every other layer needs + # TURBOQUANT, so no single --attention-backend can serve every layer. + # For any other dtype the selection is genuinely invalid -> fail loud. kv_dtype = attn_selector_config.kv_cache_dtype - if not (kv_dtype is not None and str(kv_dtype).startswith("turboquant")): + layer_is_turboquant = kv_dtype is not None and str(kv_dtype).startswith( + "turboquant" + ) + is_turboquant_fallback = ( + is_turboquant_run or layer_is_turboquant + ) and sel_invalid_reasons in ( + [_KV_CACHE_DTYPE_REASON], + [_TURBOQUANT_LAYOUT_REASON], + ) + if not is_turboquant_fallback: raise ValueError( f"Selected backend {selected_backend} is not valid for " f"this configuration. Reason: {sel_invalid_reasons}" ) # NOTE: pass a str (not the list) -- info_once hashes its args. logger.info_once( - "Selected backend %s is incompatible with this turboquant " - "layer (%s); using the auto-selected per-layer backend. " + "Selected backend %s is incompatible with this layer (%s) of " + "the turboquant run; using the auto-selected per-layer backend. " "Reason: %s", selected_backend.name, attn_selector_config.attn_type, @@ -914,12 +1000,6 @@ class RocmPlatform(Platform): compilation_config = vllm_config.compilation_config parallel_config = vllm_config.parallel_config - if ( - parallel_config.prefill_context_parallel_size > 1 - and parallel_config.data_parallel_size > 1 - ): - raise ValueError("PCP does not support data parallelism on ROCm yet.") - if ( compilation_config.cudagraph_mode.has_full_cudagraphs() and parallel_config.prefill_context_parallel_size > 1 diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index ac8489599c..d9277fb2c5 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -336,16 +336,15 @@ class XPUPlatform(Platform): "XPU Graph is not supported in the current PyTorch version, " "disabling cudagraph_mode." ) - elif not envs.VLLM_XPU_ENABLE_XPU_GRAPH: + + if ( + vllm_config.model_config is not None + and vllm_config.model_config.enable_sleep_mode + and compilation_config.cudagraph_mode != CUDAGraphMode.NONE + ): compilation_config.cudagraph_mode = CUDAGraphMode.NONE logger.warning_once( - "XPU Graph is disabled by environment variable, " - "please set VLLM_XPU_ENABLE_XPU_GRAPH=1 to enable it." - ) - else: - logger.warning_once( - "XPU Graph support is experimental and currently only supports " - "single-GPU execution." + "XPU Graph is not compatible with sleep mode, disabling cudagraph_mode." ) # Disable fusion passes not yet supported on XPU. diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 615820bb29..cad835ff1f 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -76,6 +76,10 @@ _REASONING_PARSERS_TO_REGISTER = { "granite_reasoning_parser", "GraniteReasoningParser", ), + "granite_thinking_parser": ( + "granite_thinking_engine_reasoning_parser", + "GraniteThinkingParserReasoningAdapter", + ), "holo2": ( "deepseek_v3_reasoning_parser", "DeepSeekV3ReasoningWithThinkingParser", diff --git a/vllm/reasoning/deepseek_v3_reasoning_parser.py b/vllm/reasoning/deepseek_v3_reasoning_parser.py index 072048af91..a7613112f4 100644 --- a/vllm/reasoning/deepseek_v3_reasoning_parser.py +++ b/vllm/reasoning/deepseek_v3_reasoning_parser.py @@ -55,6 +55,9 @@ class DeepSeekV3ReasoningParser(ReasoningParser): def extract_content_ids(self, input_ids: list[int]) -> list[int]: return self._parser.extract_content_ids(input_ids) + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + return self._parser.count_reasoning_tokens(token_ids) + def extract_reasoning( self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" ) -> tuple[str | None, str | None]: diff --git a/vllm/reasoning/granite_thinking_engine_reasoning_parser.py b/vllm/reasoning/granite_thinking_engine_reasoning_parser.py new file mode 100644 index 0000000000..37c7d4b350 --- /dev/null +++ b/vllm/reasoning/granite_thinking_engine_reasoning_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import ( + GraniteThinkingParserReasoningAdapter, +) + +__all__ = ["GraniteThinkingParserReasoningAdapter"] diff --git a/vllm/reasoning/kimi_k3_reasoning_parser.py b/vllm/reasoning/kimi_k3_reasoning_parser.py index e5618d47be..1a1c62ba6d 100644 --- a/vllm/reasoning/kimi_k3_reasoning_parser.py +++ b/vllm/reasoning/kimi_k3_reasoning_parser.py @@ -134,6 +134,9 @@ class KimiK3ReasoningParser(ReasoningParser): self._think_close_ids = tokenizer.encode( self._think_close, add_special_tokens=False ) + self._response_open_ids = tokenizer.encode( + self._response_open, add_special_tokens=False + ) self._last_streaming_delta_token_ids: tuple[int, ...] | None = None self._last_streaming_content_token_ids: list[int] | None = None @@ -211,6 +214,43 @@ class KimiK3ReasoningParser(ReasoningParser): return cached_content_ids return self._extract_content_ids(input_ids) + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + if not self._thinking_enabled: + return 0 + think_open = self._think_open_ids + think_close = self._think_close_ids + response_open = self._response_open_ids + first_ids = {m[0] for m in (think_open, think_close, response_open) if m} + n = len(token_ids) + count = 0 + in_reasoning = True + seen_open = False + i = 0 + while i < n: + head = token_ids[i] + if head in first_ids: + if i + len(think_open) <= n and _match_at(token_ids, i, think_open): + in_reasoning = True + seen_open = True + i += len(think_open) + continue + if i + len(think_close) <= n and _match_at(token_ids, i, think_close): + in_reasoning = False + i += len(think_close) + continue + if ( + not seen_open + and i + len(response_open) <= n + and _match_at(token_ids, i, response_open) + ): + in_reasoning = False + i += len(response_open) + continue + if in_reasoning: + count += 1 + i += 1 + return count + def _strip_content_wrapper(self, text: str) -> str: """Strip ``<|open|>response<|sep|>…<|close|>response<|sep|>`` wrapper and ``<|close|>message<|sep|>`` from *text*. diff --git a/vllm/reasoning/step3_reasoning_parser.py b/vllm/reasoning/step3_reasoning_parser.py index 16ab1cf9f4..e0ada498c2 100644 --- a/vllm/reasoning/step3_reasoning_parser.py +++ b/vllm/reasoning/step3_reasoning_parser.py @@ -121,3 +121,8 @@ class Step3ReasoningParser(ReasoningParser): return [] else: return input_ids[input_ids.index(self.think_end_token_id) + 1 :] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + if self.think_end_token_id not in token_ids: + return len(token_ids) + return list(token_ids).index(self.think_end_token_id) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 53e3f50b4e..87d11b7b92 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]): return tokenizer + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + """Render OpenAI completion suffix input when the renderer supports FIM.""" + return None + def _decode(self, *args, **kwargs): return self.get_tokenizer().decode(*args, **kwargs) diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index ede2e9530c..470e21f1eb 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -16,6 +16,10 @@ from .inputs import DictPrompt from .inputs.preprocess import parse_dec_only_prompt from .params import ChatParams +_FIM_BEGIN = "<|fim▁begin|>" +_FIM_HOLE = "<|fim▁hole|>" +_FIM_END = "<|fim▁end|>" + class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def __init__( @@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def _apply_chat_template(self, *args, **kwargs): return self.get_tokenizer().apply_chat_template(*args, **kwargs) + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}" + def render_messages( self, messages: list[ChatCompletionMessageParam], diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 36605c62b9..4ef14aee58 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -966,8 +966,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): ) def _can_produce_offsets(self) -> bool: - # HF tokenizers may be slow (use_fast=False); only fast tokenizers - # expose offset_mapping. + # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast def render_messages( diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 9dfa0c412f..6f7cd10758 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -101,8 +101,10 @@ class OnlineDerenderer: self._derender_completion_async = make_async( self._derender_completion, executor=renderer._executor ) - # Replay is O(n) per chunk (unlike the O(delta) detok-only paths), - # so it must not run on the event loop either. + self._detokenize_delta_async = make_async( + self._detokenize_delta, executor=renderer._executor + ) + # Replay is O(n) per chunk, so it must not run on the event loop. self._derender_chat_stream_parsed_async = make_async( self._derender_chat_stream_parsed, executor=renderer._executor ) @@ -362,7 +364,7 @@ class OnlineDerenderer: for choice in generate_chunk.choices: delta_tids = choice.token_ids or [] - new_text, updated_state = self._detokenize_delta( + new_text, updated_state = await self._detokenize_delta_async( tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special ) @@ -716,7 +718,7 @@ class OnlineDerenderer: for choice in generate_chunk.choices: delta_tids = choice.token_ids or [] - new_text, updated_state = self._detokenize_delta( + new_text, updated_state = await self._detokenize_delta_async( tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special ) stream_choices.append( diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index f4d21dedad..139b783aaa 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -4,7 +4,7 @@ import time from collections.abc import Sequence from dataclasses import dataclass from http import HTTPStatus -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias, cast from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem from openai.types.responses.tool import Mcp, Tool @@ -17,6 +17,7 @@ from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, ) +from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, @@ -123,6 +124,7 @@ class OnlineRenderer: chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, trust_request_chat_template: bool = False, + trust_request_mm_kwargs: bool = False, enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, @@ -155,6 +157,7 @@ class OnlineRenderer: default_chat_template_kwargs or {} ) self.trust_request_chat_template = trust_request_chat_template + self.trust_request_mm_kwargs = trust_request_mm_kwargs self.log_error_stack = log_error_stack self.supports_browsing = False @@ -548,9 +551,58 @@ class OnlineRenderer: Called directly by render_completion_request and delegated to by OpenAIServingCompletion.render_completion_request after its engine-aware checks. """ - # Return error for unsupported features. + prompt_input = request.prompt if request.suffix is not None: - return self.create_error_response("suffix is not currently supported") + if request.echo: + return self.create_error_response( + "Echo is unsupported with suffix.", + param="suffix", + ) + + if request.prompt_embeds is not None: + return self.create_error_response( + "suffix is not supported with prompt_embeds", + param="suffix", + ) + + if request.truncate_prompt_tokens is not None: + return self.create_error_response( + "suffix is not supported with truncate_prompt_tokens", + param="suffix", + ) + + if isinstance(request.prompt, str): + rendered_prompt = self.renderer.render_completion_suffix( + request.prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + prompt_input = rendered_prompt + elif isinstance(request.prompt, list) and all( + isinstance(prompt, str) for prompt in request.prompt + ): + rendered_prompts = [] + for prompt in cast(list[str], request.prompt): + rendered_prompt = self.renderer.render_completion_suffix( + prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + rendered_prompts.append(rendered_prompt) + prompt_input = rendered_prompts + else: + return self.create_error_response( + "suffix requires text prompt input for FIM completion rendering", + param="suffix", + ) if request.echo and request.prompt_embeds is not None: return self.create_error_response("Echo is unsupported with prompt embeds.") @@ -562,7 +614,7 @@ class OnlineRenderer: engine_inputs = await self.preprocess_completion( request, - prompt_input=request.prompt, + prompt_input=prompt_input, prompt_embeds=request.prompt_embeds, skip_mm_cache=skip_mm_cache, ) @@ -625,6 +677,11 @@ class OnlineRenderer: """Copied from GenerateBaseServing._preprocess_cmpl.""" renderer = self.renderer model_config = self.model_config + validate_request_mm_kwargs( + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + media_io_kwargs=getattr(request, "media_io_kwargs", None), + trust_request_mm_kwargs=self.trust_request_mm_kwargs, + ) parsed_prompts = [ ( @@ -661,6 +718,11 @@ class OnlineRenderer: ) -> tuple[list[ConversationMessage], list[EngineInput]]: """Copied from GenerateBaseServing._preprocess_chat.""" renderer = self.renderer + validate_request_mm_kwargs( + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + media_io_kwargs=getattr(request, "media_io_kwargs", None), + trust_request_mm_kwargs=self.trust_request_mm_kwargs, + ) mm_config = self.model_config.multimodal_config default_template_kwargs = merge_kwargs( diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 9f76a2ed49..9e62dfbb2d 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -234,7 +234,7 @@ class TokenizeParams: raise VLLMValidationError( f"{self.max_output_tokens_param}={max_output_tokens} " f"cannot be greater than " - f"{self.max_total_tokens_param}={max_total_tokens=}. " + f"{self.max_total_tokens_param}={max_total_tokens}. " f"Please request fewer output tokens.", parameter=self.max_output_tokens_param, value=max_output_tokens, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 77dd91b6b6..88a8978d69 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -1178,6 +1178,7 @@ class SamplingParams( validate_structured_output_request_outlines, ) from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + from vllm.v1.structured_output.utils import grammar_is_likely_lark if backend.startswith("xgrammar"): # xgrammar with no fallback @@ -1210,6 +1211,22 @@ class SamplingParams( "backends or tokenizer_mode='hf' instead." ) validate_structured_output_request_lm_format_enforcer(self) + elif ( + backend == "auto" + and is_mistral_tokenizer(tokenizer) + and tokenizer.is_tekken + and self.structured_outputs.grammar + and grammar_is_likely_lark(self.structured_outputs.grammar) + ): + # Lark grammars for Tekken Mistral tokenizers, including the ones + # the Mistral tool parser generates, go to guidance. llguidance + # takes the tokenizer from mistral-common and never lets a regex + # match a special token. The xgrammar backend declares no special + # tokens for Mistral tokenizers, so a regex like `.` can match + # `[TOOL_CALLS]` or `[INST]` as plain text. + validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer)) + self.structured_outputs._backend = "guidance" + self.structured_outputs._backend_was_auto = True else: # NOTE: backend must be "auto" here, because we have # checked supported_backends above. diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py index d73827f499..311b328c40 100644 --- a/vllm/scalar_type.py +++ b/vllm/scalar_type.py @@ -316,8 +316,13 @@ class ScalarType: class scalar_types: + uint2 = ScalarType.uint(2, None) + uint3 = ScalarType.uint(3, None) int4 = ScalarType.int_(4, None) uint4 = ScalarType.uint(4, None) + uint5 = ScalarType.uint(5, None) + uint6 = ScalarType.uint(6, None) + uint7 = ScalarType.uint(7, None) int8 = ScalarType.int_(8, None) uint8 = ScalarType.uint(8, None) float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN) diff --git a/vllm/tokenizers/deepseek_v41_encoding.py b/vllm/tokenizers/deepseek_v41_encoding.py index 841de385cb..6851d42f91 100644 --- a/vllm/tokenizers/deepseek_v41_encoding.py +++ b/vllm/tokenizers/deepseek_v41_encoding.py @@ -181,8 +181,8 @@ REASONING_EFFORT_TEMPLATE = ( "(range 1-100, the higher the value, the more thorough the reasoning)\n\n" ) REASONING_EFFORT_MAPPINGS: Dict[str, int] = { - "low": 25, - "high": 50, + "low": 50, + "high": 75, "xhigh": 75, "max": 100, } diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 46446ecd6d..09a715c117 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -145,13 +145,6 @@ def resolve_tokenizer_args( else: assert_never(runner_type) - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - - tokenizer_mode = "hf" - kwargs["use_fast"] = False - # Try to use official Mistral tokenizer if possible if ( tokenizer_mode == "auto" diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index c43a34772f..2515009d0e 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -75,8 +75,8 @@ _TOOL_PARSERS_TO_REGISTER = { "Granite20bFCToolParser", ), "granite": ( - "granite_tool_parser", - "GraniteToolParser", + "granite_engine_tool_parser", + "GraniteEngineToolParser", ), "granite4": ( "granite4_tool_parser", diff --git a/vllm/tool_parsers/granite_engine_tool_parser.py b/vllm/tool_parsers/granite_engine_tool_parser.py new file mode 100644 index 0000000000..1fb701d5a4 --- /dev/null +++ b/vllm/tool_parsers/granite_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import GraniteParserToolAdapter + + +class GraniteEngineToolParser(GraniteParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = None diff --git a/vllm/tool_parsers/granite_tool_parser.py b/vllm/tool_parsers/granite_tool_parser.py deleted file mode 100644 index 1e51c3227e..0000000000 --- a/vllm/tool_parsers/granite_tool_parser.py +++ /dev/null @@ -1,257 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import partial_json_parser -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.chat_utils import make_tool_call_id -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - consume_space, - find_common_prefix, - is_complete_json, - partial_json_loads, -) - -logger = init_logger(__name__) - - -class GraniteToolParser(ToolParser): - """Tool call parser for the granite 3.0 models. Intended - for use with the examples/tool_chat_template_granite.jinja - template. - - Used when --enable-auto-tool-choice --tool-call-parser granite - are all set - """ - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - # for granite 3.0, the token `<|tool_call|>` - self.bot_token = "<|tool_call|>" - # for granite 3.1, the string `` - self.bot_string = "" - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - stripped = ( - model_output.strip() - .removeprefix(self.bot_token) - .removeprefix(self.bot_string) - .lstrip() - ) - if not stripped or stripped[0] != "[": - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - try: - raw_function_calls = json.loads(stripped) - if not isinstance(raw_function_calls, list): - raise Exception( - f"Expected dict or list, got {type(raw_function_calls)}" - ) - - logger.debug("Extracted %d tool calls", len(raw_function_calls)) - tool_calls = [ - ToolCall( - type="function", - function=FunctionCall( - name=function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - function_call["arguments"], ensure_ascii=False - ), - ), - ) - for function_call in raw_function_calls - ] - - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=None, - ) - - except Exception as e: - logger.error("Error in extracting tool call from response %s", e) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - start_idx = consume_space(0, current_text) - if current_text[start_idx:].startswith(self.bot_token): - start_idx = consume_space(start_idx + len(self.bot_token), current_text) - if current_text[start_idx:].startswith(self.bot_string): - start_idx = consume_space(start_idx + len(self.bot_string), current_text) - if ( - not current_text - or start_idx >= len(current_text) - or current_text[start_idx] != "[" - ): - return DeltaMessage(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - tool_call_arr = None - is_complete = None - try: - tool_calls, end_idx = partial_json_loads( - current_text[start_idx:], flags - ) - if type(tool_calls) is list: - tool_call_arr = tool_calls - else: - return DeltaMessage(content=delta_text) - - is_complete = [True] * len(tool_calls) - if not is_complete_json(current_text[start_idx : start_idx + end_idx]): - is_complete[-1] = False - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if not tool_call_arr: - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = tool_call_arr[self.current_tool_id] - - delta = None - # Only advance once the current tool name is streamed; granite - # emits arguments before name, so advancing early would drop it. - if len(tool_call_arr) > self.current_tool_id + 1 and ( - self.current_tool_id < 0 or self.current_tool_name_sent - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get("arguments") - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - # re-set stuff pertaining to progress in the current tool - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get("arguments") - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception as e: - logger.error("Error trying to handle streaming tool call: %s", e) - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index e78a718e28..432fb1379f 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -17,7 +17,7 @@ import transformers.configuration_utils as hf_configuration_utils from huggingface_hub import constants from packaging.version import Version from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE -from transformers import GenerationConfig, PretrainedConfig +from transformers import GenerationConfig, PreTrainedConfig from transformers.configuration_utils import ALLOWED_LAYER_TYPES from transformers.models.auto.image_processing_auto import get_image_processor_config from transformers.models.auto.modeling_auto import ( @@ -86,7 +86,7 @@ class LazyConfigDict(dict): return getattr(configs, value) -_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( +_CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( afmoe="AfmoeConfig", axk1="AXK1Config", bagel="BagelConfig", @@ -189,13 +189,13 @@ _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = { def _register_config_class( - model_type: str, config_class: type[PretrainedConfig] + model_type: str, config_class: type[PreTrainedConfig] ) -> None: config_class.model_type = model_type AutoConfig.register(model_type, config_class, exist_ok=True) -def _maybe_register_hf_config(config: PretrainedConfig | None) -> None: +def _maybe_register_hf_config(config: PreTrainedConfig | None) -> None: if config is None: return @@ -228,7 +228,7 @@ def _mistral_patch_hf_hub_constants() -> Iterator[None]: def _install_hf_config_validator( name: str, validator: Callable, supersedes: Callable ) -> None: - """Replace a ``PretrainedConfig`` validator on every ``@strict`` snapshot. + """Replace a ``PreTrainedConfig`` validator on every ``@strict`` snapshot. ``@strict`` snapshots each ``validate_*`` method into ``__class_validators__`` at class creation and automatic post-``__init__`` validation dispatches off @@ -237,10 +237,10 @@ def _install_hf_config_validator( class owns a snapshot, so rewrite them all, matching ``supersedes`` by identity to leave a genuine per-model override in place. """ - setattr(PretrainedConfig, name, validator) + setattr(PreTrainedConfig, name, validator) seen: set[int] = set() - stack = [PretrainedConfig] + stack = [PreTrainedConfig] while stack: cls = stack.pop() if id(cls) in seen: @@ -261,10 +261,10 @@ def _patch_hf_transformers_validate_rope(): validate_rope() with the ignore_keys parameter work with newer versions of hf transformers (from v5 onwards) """ - if hasattr(PretrainedConfig.validate_rope, "__vllm_patched__"): + if hasattr(PreTrainedConfig.validate_rope, "__vllm_patched__"): return - _original_validate_rope = PretrainedConfig.validate_rope + _original_validate_rope = PreTrainedConfig.validate_rope @wraps(_original_validate_rope) def patched_validate_rope(self, *args, **kwargs): @@ -293,10 +293,10 @@ def _patch_hf_transformers_nested_rope_validation() -> None: ``AttributeError``. The per-layer dicts already carry their own defaults by the time validation runs, so the shared entries can be dropped. """ - if hasattr(PretrainedConfig.validate_rope, "__vllm_nested_rope_patched__"): + if hasattr(PreTrainedConfig.validate_rope, "__vllm_nested_rope_patched__"): return - _original_validate_rope = PretrainedConfig.validate_rope + _original_validate_rope = PreTrainedConfig.validate_rope @wraps(_original_validate_rope) def patched_validate_rope(self, *args, **kwargs): @@ -347,11 +347,11 @@ class HFConfigParser(ConfigParserBase): revision: str | None = None, code_revision: str | None = None, **kwargs, - ) -> tuple[dict, PretrainedConfig]: + ) -> tuple[dict, PreTrainedConfig]: kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE trust_remote_code |= kwargs.get("trust_remote_code", False) kwargs = without_trust_remote_code(kwargs) - config_dict, _ = PretrainedConfig.get_config_dict( + config_dict, _ = PreTrainedConfig.get_config_dict( model, revision=revision, code_revision=code_revision, @@ -374,7 +374,7 @@ class HFConfigParser(ConfigParserBase): # through and remain unchanged by this elif block dummy_model_type = f"dummy_{model_type}" dummy_kwargs = dict(architectures=[""], model_type=dummy_model_type) - dummy_config = PretrainedConfig(**dummy_kwargs) + dummy_config = PreTrainedConfig(**dummy_kwargs) dummy_model_type = hf_overrides(dummy_config).model_type model_type = dummy_model_type.removeprefix("dummy_") @@ -387,7 +387,20 @@ class HFConfigParser(ConfigParserBase): if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type): _patch_hf_transformers_allowed_layer_types(extra_layer_types) - if model_type in _SPECULATIVE_DECODING_CONFIGS: + rope_parameters = config_dict.get("rope_parameters") or {} + if model_type == "gemma4_text" and "full_attention" in rope_parameters: + from transformers import Gemma4TextConfig + + # Published DSpark configs mix redundant scalar entries with per-layer + # RoPE dicts. Remove them before Transformers' constructor validates. + config_dict["rope_parameters"] = { + k: v + for k, v in rope_parameters.items() + if k not in ("rope_type", "rope_theta") + } + kwargs.setdefault("name_or_path", str(model)) + config = Gemma4TextConfig.from_dict(config_dict, **kwargs) + elif model_type in _SPECULATIVE_DECODING_CONFIGS: config_class = _CONFIG_REGISTRY[model_type] config = config_class.from_pretrained( model, @@ -450,7 +463,7 @@ class MistralConfigParser(ConfigParserBase): revision: str | None = None, code_revision: str | None = None, **kwargs, - ) -> tuple[dict, PretrainedConfig]: + ) -> tuple[dict, PreTrainedConfig]: # This function loads a params.json config which # should be used when loading models in mistral format config_dict = _download_mistral_config_file(model, revision) @@ -466,7 +479,7 @@ class MistralConfigParser(ConfigParserBase): # Get missing fields from HF config if available try: - hf_config_dict, _ = PretrainedConfig.get_config_dict( + hf_config_dict, _ = PreTrainedConfig.get_config_dict( model, revision=revision, code_revision=code_revision, @@ -540,7 +553,7 @@ def register_config_parser(config_format: str): ... revision: str | None = None, ... code_revision: str | None = None, ... **kwargs, - ... ) -> tuple[dict, PretrainedConfig]: + ... ) -> tuple[dict, PreTrainedConfig]: ... raise NotImplementedError >>> >>> type(get_config_parser("custom_config_parser")) @@ -571,7 +584,7 @@ def register_config_parser(config_format: str): return _wrapper -def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None: +def set_default_rope_theta(config: PreTrainedConfig, default_theta: float) -> None: """Some models may have no rope_theta in their config but still use RoPE. This function sets a default rope_theta if it's missing.""" if getattr(config, "rope_parameters", None) is None: @@ -639,7 +652,7 @@ def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None: _patch_legacy_rope_type(rope_parameters) -def patch_rope_parameters(config: PretrainedConfig) -> None: +def patch_rope_parameters(config: PreTrainedConfig) -> None: """Provide backwards compatibility for RoPE.""" from vllm.config.utils import getattr_iter @@ -661,7 +674,7 @@ def patch_rope_parameters(config: PretrainedConfig) -> None: config.validate_rope() -def _iter_rope_parameters(config: PretrainedConfig) -> Iterator[dict[str, Any]]: +def _iter_rope_parameters(config: PreTrainedConfig) -> Iterator[dict[str, Any]]: """Yield a config's rope parameters, one dict per layer type if nested.""" rope_parameters = getattr(config, "rope_parameters", None) if not isinstance(rope_parameters, dict): @@ -673,7 +686,7 @@ def _iter_rope_parameters(config: PretrainedConfig) -> Iterator[dict[str, Any]]: yield rope_parameters -def _mrope_section(config: PretrainedConfig) -> Sequence[int] | None: +def _mrope_section(config: PreTrainedConfig) -> Sequence[int] | None: """Return the M-RoPE section this config declares, if any. `xdrope_section` is the legacy name HunYuan-VL checkpoints use for the same @@ -700,11 +713,11 @@ def _mrope_section(config: PretrainedConfig) -> Sequence[int] | None: return section if isinstance(section, (list, tuple)) else None -def _uses_mrope(config: PretrainedConfig) -> bool: +def _uses_mrope(config: PreTrainedConfig) -> bool: return _mrope_section(config) is not None -def uses_mrope(config: PretrainedConfig) -> bool: +def uses_mrope(config: PreTrainedConfig) -> bool: """Detect if the model with this config uses M-ROPE.""" return ( _uses_mrope(config) @@ -713,7 +726,7 @@ def uses_mrope(config: PretrainedConfig) -> bool: ) -def thinker_uses_mrope(config: PretrainedConfig) -> bool: +def thinker_uses_mrope(config: PreTrainedConfig) -> bool: """Detect if the model contains a thinker config and it uses M-ROPE.""" thinker_config = getattr(config, "thinker_config", None) if thinker_config is None: @@ -726,7 +739,7 @@ def thinker_uses_mrope(config: PretrainedConfig) -> bool: return uses_mrope(thinker_text_config) -def mrope_num_dims(config: PretrainedConfig) -> int: +def mrope_num_dims(config: PreTrainedConfig) -> int: """Number of M-RoPE position channels the model consumes. Each section entry sizes one channel, so the section length is the channel @@ -745,10 +758,10 @@ def mrope_num_dims(config: PretrainedConfig) -> int: return 3 -def is_encoder_decoder(config: PretrainedConfig) -> bool: +def is_encoder_decoder(config: PreTrainedConfig) -> bool: """Detect if the model with this config is used as an encoder/decoder.""" - def _is_encoder_decoder(config: PretrainedConfig) -> bool: + def _is_encoder_decoder(config: PreTrainedConfig) -> bool: return getattr(config, "is_encoder_decoder", False) return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config()) @@ -761,7 +774,7 @@ def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): return kwargs -def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig: +def _maybe_remap_hf_config_attrs(config: PreTrainedConfig) -> PreTrainedConfig: """Remap config attributes to match the expected names.""" for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items(): if hasattr(config, old_attr): @@ -798,7 +811,7 @@ def maybe_override_with_speculators( """ kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE - config_dict, _ = PretrainedConfig.get_config_dict( + config_dict, _ = PreTrainedConfig.get_config_dict( model, revision=revision, token=hf_token, @@ -840,9 +853,9 @@ def get_config( code_revision: str | None = None, config_format: str | ConfigFormat = "auto", hf_overrides_kw: dict[str, Any] | None = None, - hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None, + hf_overrides_fn: Callable[[PreTrainedConfig], PreTrainedConfig] | None = None, **kwargs, -) -> PretrainedConfig: +) -> PreTrainedConfig: if config_format == "auto": try: # First check for Mistral to avoid defaulting to @@ -953,7 +966,7 @@ def get_config( # Exhaustively patch RoPE parameters everywhere they might be patch_rope_parameters(config) patch_rope_parameters(config.get_text_config()) - SubConfigs: TypeAlias = dict[str, PretrainedConfig] + SubConfigs: TypeAlias = dict[str, PreTrainedConfig] sub_configs: SubConfigs | None = getattr(config, "sub_configs", None) if sub_configs: for sub_config in sub_configs: @@ -1212,7 +1225,7 @@ def get_hf_image_processor_config( ) -def get_hf_text_config(config: PretrainedConfig): +def get_hf_text_config(config: PreTrainedConfig): """Get the "sub" config relevant to llm for multi modal models. No op for pure text models. """ diff --git a/vllm/transformers_utils/config_parser_base.py b/vllm/transformers_utils/config_parser_base.py index 79d47ff560..46cbab5be9 100644 --- a/vllm/transformers_utils/config_parser_base.py +++ b/vllm/transformers_utils/config_parser_base.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from pathlib import Path -from transformers import PretrainedConfig +from transformers import PreTrainedConfig class ConfigParserBase(ABC): @@ -16,5 +16,5 @@ class ConfigParserBase(ABC): revision: str | None = None, code_revision: str | None = None, **kwargs, - ) -> tuple[dict, PretrainedConfig]: + ) -> tuple[dict, PreTrainedConfig]: raise NotImplementedError diff --git a/vllm/transformers_utils/configs/AXK1.py b/vllm/transformers_utils/configs/AXK1.py index 52af5bec48..50bc891573 100644 --- a/vllm/transformers_utils/configs/AXK1.py +++ b/vllm/transformers_utils/configs/AXK1.py @@ -2,16 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class AXK1Config(PretrainedConfig): +class AXK1Config(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`AXK1Model`]. It is used to instantiate an A.X model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the A.X K1. - Configuration objects inherit from [`PretrainedConfig`] and can be used to control - the model outputs. Read the documentation from [`PretrainedConfig`] for more + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control + the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: diff --git a/vllm/transformers_utils/configs/afmoe.py b/vllm/transformers_utils/configs/afmoe.py index 47fee9882f..b3d10dc5f2 100644 --- a/vllm/transformers_utils/configs/afmoe.py +++ b/vllm/transformers_utils/configs/afmoe.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class AfmoeConfig(PretrainedConfig): +class AfmoeConfig(PreTrainedConfig): model_type = "afmoe" def __init__( diff --git a/vllm/transformers_utils/configs/bagel.py b/vllm/transformers_utils/configs/bagel.py index 53347ef452..a972000db4 100644 --- a/vllm/transformers_utils/configs/bagel.py +++ b/vllm/transformers_utils/configs/bagel.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import PretrainedConfig, SiglipVisionConfig +from transformers import PreTrainedConfig, SiglipVisionConfig from transformers.models.qwen2 import Qwen2Config -class BagelConfig(PretrainedConfig): +class BagelConfig(PreTrainedConfig): """Configuration class for BAGEL model.""" model_type = "bagel" diff --git a/vllm/transformers_utils/configs/bailing_moe_v3_vl.py b/vllm/transformers_utils/configs/bailing_moe_v3_vl.py index 484e27991e..e009a4db14 100644 --- a/vllm/transformers_utils/configs/bailing_moe_v3_vl.py +++ b/vllm/transformers_utils/configs/bailing_moe_v3_vl.py @@ -9,11 +9,11 @@ from __future__ import annotations from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig def _normalize_mrope_parameters( - config: PretrainedConfig, + config: PreTrainedConfig, mrope_section: list[int], ) -> None: """Add the top-level Bailing M-RoPE split to a text config.""" @@ -50,7 +50,7 @@ def _build_layer_types( ] -class BailingMoeV3TextConfig(PretrainedConfig): +class BailingMoeV3TextConfig(PreTrainedConfig): model_type = "bailing_hybrid" base_config_key = "text_config" keys_to_ignore_at_inference = ["past_key_values"] @@ -230,7 +230,7 @@ class BailingMoeV3TextConfig(PretrainedConfig): self.validate_layer_type() -class BailingMoeV3VisionConfig(PretrainedConfig): +class BailingMoeV3VisionConfig(PreTrainedConfig): model_type = "qwen3_moe_vit" base_config_key = "vision_config" @@ -273,7 +273,7 @@ class BailingMoeV3VisionConfig(PretrainedConfig): self.initializer_range = initializer_range -class BailingMoeV3VLConfig(PretrainedConfig): +class BailingMoeV3VLConfig(PreTrainedConfig): model_type = "bailing_moe_v3_vl" keys_to_ignore_at_inference = ["past_key_values"] sub_configs = { diff --git a/vllm/transformers_utils/configs/chatglm.py b/vllm/transformers_utils/configs/chatglm.py index 1d795b55c8..3908df740a 100644 --- a/vllm/transformers_utils/configs/chatglm.py +++ b/vllm/transformers_utils/configs/chatglm.py @@ -3,10 +3,10 @@ # Adapted from # https://github.com/zai-org/ChatGLM2-6B -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class ChatGLMConfig(PretrainedConfig): +class ChatGLMConfig(PreTrainedConfig): model_type = "chatglm" attribute_map = { "num_hidden_layers": "num_layers", diff --git a/vllm/transformers_utils/configs/colmodernvbert.py b/vllm/transformers_utils/configs/colmodernvbert.py index 17558aafce..a830541757 100644 --- a/vllm/transformers_utils/configs/colmodernvbert.py +++ b/vllm/transformers_utils/configs/colmodernvbert.py @@ -8,10 +8,10 @@ with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings. Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged """ -from transformers import ModernBertConfig, PretrainedConfig, SiglipVisionConfig +from transformers import ModernBertConfig, PreTrainedConfig, SiglipVisionConfig -class ColModernVBertConfig(PretrainedConfig): +class ColModernVBertConfig(PreTrainedConfig): model_type = "colmodernvbert" def __init__( diff --git a/vllm/transformers_utils/configs/cosmos3_edge.py b/vllm/transformers_utils/configs/cosmos3_edge.py index 743e7d5deb..447b64cc99 100644 --- a/vllm/transformers_utils/configs/cosmos3_edge.py +++ b/vllm/transformers_utils/configs/cosmos3_edge.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig from .nemotron_h import NemotronHConfig @@ -50,7 +50,7 @@ class Cosmos3EdgeTextConfig(NemotronHConfig): return config_dict -class Cosmos3EdgeProjectorConfig(PretrainedConfig): +class Cosmos3EdgeProjectorConfig(PreTrainedConfig): model_type = "cosmos3_edge_projector" def __init__( @@ -74,7 +74,7 @@ class Cosmos3EdgeVisionConfig(Siglip2VisionConfig): model_type = "cosmos3_edge_vision" -class Cosmos3EdgeConfig(PretrainedConfig): +class Cosmos3EdgeConfig(PreTrainedConfig): model_type = "cosmos3_edge" keys_to_ignore_at_inference = ["past_key_values"] has_no_defaults_at_init = True diff --git a/vllm/transformers_utils/configs/deepseek_v4.py b/vllm/transformers_utils/configs/deepseek_v4.py index 15346e35e5..278f477864 100755 --- a/vllm/transformers_utils/configs/deepseek_v4.py +++ b/vllm/transformers_utils/configs/deepseek_v4.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class DeepseekV4Config(PretrainedConfig): +class DeepseekV4Config(PreTrainedConfig): model_type = "deepseek_v4" def __init__( diff --git a/vllm/transformers_utils/configs/deepseek_v41.py b/vllm/transformers_utils/configs/deepseek_v41.py index d8b4335438..f1ed6f9aa8 100644 --- a/vllm/transformers_utils/configs/deepseek_v41.py +++ b/vllm/transformers_utils/configs/deepseek_v41.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class DeepseekV41Config(PretrainedConfig): +class DeepseekV41Config(PreTrainedConfig): """DeepSeek V4.1 config. The HF config nests the text model under ``text_config`` and the vision @@ -34,7 +34,7 @@ class DeepseekV41Config(PretrainedConfig): for key, value in text_config.items(): if key == "model_type": continue - # Don't clobber PretrainedConfig properties (e.g. is_encoder_decoder). + # Don't clobber PreTrainedConfig properties (e.g. is_encoder_decoder). if isinstance(getattr(type(self), key, None), property): continue setattr(self, key, value) diff --git a/vllm/transformers_utils/configs/deepseek_vl2.py b/vllm/transformers_utils/configs/deepseek_vl2.py index 3b980a824d..3543db833a 100644 --- a/vllm/transformers_utils/configs/deepseek_vl2.py +++ b/vllm/transformers_utils/configs/deepseek_vl2.py @@ -4,10 +4,10 @@ # adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268 from huggingface_hub.dataclasses import strict -from transformers import DeepseekV2Config, PretrainedConfig +from transformers import DeepseekV2Config, PreTrainedConfig -class VisionEncoderConfig(PretrainedConfig): +class VisionEncoderConfig(PreTrainedConfig): model_type: str = "vision" model_name: str = "vit_so400m_patch14_siglip_384.webli" @@ -58,7 +58,7 @@ class VisionEncoderConfig(PretrainedConfig): super().__init__(**kwargs) -class MlpProjectorConfig(PretrainedConfig): +class MlpProjectorConfig(PreTrainedConfig): model_type = "mlp_projector" projector_type: str = "downsample_mlp_gelu" input_dim: int = 1152 @@ -98,7 +98,7 @@ class DeepseekVLV2TextConfig(DeepseekV2Config): kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank -class DeepseekVLV2Config(PretrainedConfig): +class DeepseekVLV2Config(PreTrainedConfig): model_type = "deepseek_vl_v2" tile_tag: str = "2D" diff --git a/vllm/transformers_utils/configs/diffusion_gemma.py b/vllm/transformers_utils/configs/diffusion_gemma.py index 246a25b32c..884d176cd0 100644 --- a/vllm/transformers_utils/configs/diffusion_gemma.py +++ b/vllm/transformers_utils/configs/diffusion_gemma.py @@ -2,12 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.models.gemma4.configuration_gemma4 import Gemma4VisionConfig -def _init_text_config(self: PretrainedConfig, **kwargs: Any) -> None: - PretrainedConfig.__init__(self, **kwargs) +def _init_text_config(self: PreTrainedConfig, **kwargs: Any) -> None: + PreTrainedConfig.__init__(self, **kwargs) # DiffusionGemma always uses MoE and K=V sharing for full_attention # layers. The HF reference removed these config fields entirely. if getattr(self, "num_experts", None): @@ -15,14 +15,14 @@ def _init_text_config(self: PretrainedConfig, **kwargs: Any) -> None: self.attention_k_eq_v = True -class DiffusionGemmaTextConfig(PretrainedConfig): +class DiffusionGemmaTextConfig(PreTrainedConfig): model_type = "diffusion_gemma_text" def __init__(self, **kwargs: Any): _init_text_config(self, **kwargs) -class DiffusionGemmaConfig(PretrainedConfig): +class DiffusionGemmaConfig(PreTrainedConfig): model_type = "diffusion_gemma" def __init__( @@ -41,4 +41,4 @@ class DiffusionGemmaConfig(PretrainedConfig): else: self.vision_config = vision_config self.audio_config = None - PretrainedConfig.__init__(self, **kwargs) + PreTrainedConfig.__init__(self, **kwargs) diff --git a/vllm/transformers_utils/configs/dotsocr.py b/vllm/transformers_utils/configs/dotsocr.py index 1e42cb2fd8..918179dcb8 100644 --- a/vllm/transformers_utils/configs/dotsocr.py +++ b/vllm/transformers_utils/configs/dotsocr.py @@ -2,11 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.models.qwen2 import Qwen2Config -class DotsVisionConfig(PretrainedConfig): +class DotsVisionConfig(PreTrainedConfig): model_type: str = "dots_vit" def __init__( diff --git a/vllm/transformers_utils/configs/eagle.py b/vllm/transformers_utils/configs/eagle.py index 682828c0a5..4a5efead72 100644 --- a/vllm/transformers_utils/configs/eagle.py +++ b/vllm/transformers_utils/configs/eagle.py @@ -3,22 +3,22 @@ import os -from transformers import AutoConfig, DeepseekV2Config, PretrainedConfig +from transformers import AutoConfig, DeepseekV2Config, PreTrainedConfig from vllm.transformers_utils.utils import without_trust_remote_code -class EAGLEConfig(PretrainedConfig): +class EAGLEConfig(PreTrainedConfig): model_type = "eagle" def __init__( self, - model: PretrainedConfig | dict | None = None, + model: PreTrainedConfig | dict | None = None, truncated_vocab_size: int | None = None, method: str | None = "eagle", **kwargs, ): - model_config: PretrainedConfig | DeepseekV2Config | None + model_config: PreTrainedConfig | DeepseekV2Config | None if isinstance(model, dict): model_config = AutoConfig.for_model(**model) else: @@ -68,7 +68,9 @@ class EAGLEConfig(PretrainedConfig): ) kwargs["architectures"] = [ arch - if arch.startswith("DFlash") or arch.endswith("DFlash") + if arch.startswith("DFlash") + or arch.endswith("DFlash") + or arch == "LiLiCorrDraftModel" else f"DFlash{arch}" for arch in self.model.architectures ] diff --git a/vllm/transformers_utils/configs/extract_hidden_states.py b/vllm/transformers_utils/configs/extract_hidden_states.py index 2beec0e308..0b984c7722 100644 --- a/vllm/transformers_utils/configs/extract_hidden_states.py +++ b/vllm/transformers_utils/configs/extract_hidden_states.py @@ -5,17 +5,17 @@ the extract_hidden_states spec decoding method.""" import os -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.transformers_utils.utils import without_trust_remote_code -class ExtractHiddenStatesConfig(PretrainedConfig): +class ExtractHiddenStatesConfig(PreTrainedConfig): model_type = "extract_hidden_states" def __init__( self, - model: PretrainedConfig | dict | None = None, + model: PreTrainedConfig | dict | None = None, method: str | None = "extract_hidden_states", **kwargs, ): @@ -24,7 +24,7 @@ class ExtractHiddenStatesConfig(PretrainedConfig): if isinstance(model, dict): model_dict = model source_text_config = None - elif isinstance(model, PretrainedConfig): + elif isinstance(model, PreTrainedConfig): model_dict = model.to_dict() text_config = model.get_text_config() source_text_config = text_config if text_config is not model else None @@ -40,7 +40,7 @@ class ExtractHiddenStatesConfig(PretrainedConfig): combined["architectures"] = ["ExtractHiddenStatesModel"] # to_dict() and kwargs both flatten text_config to a plain dict; - # downstream get_hf_text_config() needs it as a PretrainedConfig + # downstream get_hf_text_config() needs it as a PreTrainedConfig # for attribute access. Re-insert the original object. if source_text_config is not None: combined["text_config"] = source_text_config diff --git a/vllm/transformers_utils/configs/falcon.py b/vllm/transformers_utils/configs/falcon.py index f8665a1395..7dfe21d0b8 100644 --- a/vllm/transformers_utils/configs/falcon.py +++ b/vllm/transformers_utils/configs/falcon.py @@ -20,10 +20,10 @@ # limitations under the License. """Falcon configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class RWConfig(PretrainedConfig): +class RWConfig(PreTrainedConfig): model_type = "falcon" keys_to_ignore_at_inference = ["past_key_values"] attribute_map = { diff --git a/vllm/transformers_utils/configs/funaudiochat.py b/vllm/transformers_utils/configs/funaudiochat.py index 36a446860c..dd2573aac4 100644 --- a/vllm/transformers_utils/configs/funaudiochat.py +++ b/vllm/transformers_utils/configs/funaudiochat.py @@ -3,7 +3,7 @@ from __future__ import annotations -from transformers import CONFIG_MAPPING, PretrainedConfig +from transformers import CONFIG_MAPPING, PreTrainedConfig # NOTE: Temporary shim for FunAudioChat checkpoints. # These checkpoints use `model_type="funaudiochat"`, which is not currently @@ -13,7 +13,7 @@ from transformers import CONFIG_MAPPING, PretrainedConfig # provides an `auto_map`) and vLLM can rely on `AutoConfig.from_pretrained()`. -class FunAudioChatAudioEncoderConfig(PretrainedConfig): +class FunAudioChatAudioEncoderConfig(PreTrainedConfig): model_type = "funaudiochat_audio_encoder" def __init__( @@ -74,7 +74,7 @@ class FunAudioChatAudioEncoderConfig(PretrainedConfig): self.pad_token_id = pad_token_id -class FunAudioChatConfig(PretrainedConfig): +class FunAudioChatConfig(PreTrainedConfig): model_type = "funaudiochat" attribute_map = { "audio_token_id": "audio_token_index", @@ -82,8 +82,8 @@ class FunAudioChatConfig(PretrainedConfig): def __init__( self, - audio_config: PretrainedConfig | dict | None = None, - text_config: PretrainedConfig | dict | None = None, + audio_config: PreTrainedConfig | dict | None = None, + text_config: PreTrainedConfig | dict | None = None, audio_token_index: int = 151646, ignore_index: int = -100, hidden_size: int | None = None, diff --git a/vllm/transformers_utils/configs/gemma4.py b/vllm/transformers_utils/configs/gemma4.py index c57ba5b7ec..69cbc06d04 100644 --- a/vllm/transformers_utils/configs/gemma4.py +++ b/vllm/transformers_utils/configs/gemma4.py @@ -4,12 +4,12 @@ from copy import copy -from transformers import PretrainedConfig +from transformers import PreTrainedConfig def gemma4_layer_config( - text_config: PretrainedConfig, layer_idx: int -) -> PretrainedConfig: + text_config: PreTrainedConfig, layer_idx: int +) -> PreTrainedConfig: """The Gemma4 text config as it applies to one layer. Gemma4 uses a larger head dimension on its full attention layers than on its diff --git a/vllm/transformers_utils/configs/glm5_next.py b/vllm/transformers_utils/configs/glm5_next.py index 95a0081bdd..4a565884a2 100644 --- a/vllm/transformers_utils/configs/glm5_next.py +++ b/vllm/transformers_utils/configs/glm5_next.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.logger import init_logger logger = init_logger(__name__) -class Glm5NextTextConfig(PretrainedConfig): +class Glm5NextTextConfig(PreTrainedConfig): model_type = "glm5_next_text" base_config_key = "text_config" keys_to_ignore_at_inference = ["past_key_values"] @@ -271,7 +271,7 @@ class Glm5NextTextConfig(PretrainedConfig): ] -class Glm5NextVisionConfig(PretrainedConfig): +class Glm5NextVisionConfig(PreTrainedConfig): model_type = "glm5_next_vision" base_config_key = "vision_config" @@ -324,7 +324,7 @@ class Glm5NextVisionConfig(PretrainedConfig): self.swiglu_limit = swiglu_limit -class Glm5NextConfig(PretrainedConfig): +class Glm5NextConfig(PreTrainedConfig): model_type = "glm5_next" sub_configs = { "vision_config": Glm5NextVisionConfig, @@ -345,7 +345,7 @@ class Glm5NextConfig(PretrainedConfig): **kwargs, ): # Init super() first so base-class defaults don't clobber text-config - # values set below (PretrainedConfig has many text-related defaults + # values set below (PreTrainedConfig has many text-related defaults # that differ from Glm5NextTextConfig). super().__init__(**kwargs) @@ -379,7 +379,7 @@ class Glm5NextConfig(PretrainedConfig): # and must NOT be mirrored onto text_config: ``architectures`` / # ``torch_dtype`` differ between the top-level config and the text # sub-config, and mirroring them makes the top-level ``architectures`` - # silently read back as None (PretrainedConfig initializes both to None), + # silently read back as None (PreTrainedConfig initializes both to None), # which then fails model-class resolution ("No model architectures are # specified"). _UNMIRRORED_KEYS = [ diff --git a/vllm/transformers_utils/configs/granite4_vision.py b/vllm/transformers_utils/configs/granite4_vision.py index a67e6ffeb8..8522bdd47a 100644 --- a/vllm/transformers_utils/configs/granite4_vision.py +++ b/vllm/transformers_utils/configs/granite4_vision.py @@ -6,7 +6,7 @@ from typing import Any import transformers -class Granite4VisionConfig(transformers.PretrainedConfig): +class Granite4VisionConfig(transformers.PreTrainedConfig): """Configuration for Granite 4 Vision model. This config is needed because the granite4_vision model type is not yet @@ -81,7 +81,7 @@ class Granite4VisionConfig(transformers.PretrainedConfig): **vision_config ) else: - self.vision_config = transformers.PretrainedConfig(**vision_config) + self.vision_config = transformers.PreTrainedConfig(**vision_config) text_model_type = text_config.get("model_type", "granite") if text_model_type in transformers.CONFIG_MAPPING: @@ -89,6 +89,6 @@ class Granite4VisionConfig(transformers.PretrainedConfig): **text_config ) else: - self.text_config = transformers.PretrainedConfig(**text_config) + self.text_config = transformers.PreTrainedConfig(**text_config) super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/hy_v3.py b/vllm/transformers_utils/configs/hy_v3.py index 0a67dd8af7..939dd5d78c 100644 --- a/vllm/transformers_utils/configs/hy_v3.py +++ b/vllm/transformers_utils/configs/hy_v3.py @@ -2,16 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class HYV3Config(PretrainedConfig): +class HYV3Config(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`HYV3Model`]. It is used to instantiate a HYV3 model (HY V3 MoE language model) according to the specified arguments. - Configuration objects inherit from [`PretrainedConfig`] and can be used to - control the model outputs. Read the documentation from [`PretrainedConfig`] + Configuration objects inherit from [`PreTrainedConfig`] and can be used to + control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: diff --git a/vllm/transformers_utils/configs/hyperclovax.py b/vllm/transformers_utils/configs/hyperclovax.py index 2033d59c11..5470408b58 100644 --- a/vllm/transformers_utils/configs/hyperclovax.py +++ b/vllm/transformers_utils/configs/hyperclovax.py @@ -17,16 +17,16 @@ # limitations under the License. """HyperCLOVA X model configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class HyperCLOVAXConfig(PretrainedConfig): +class HyperCLOVAXConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model according to the specified arguments, defining the model architecture. - Configuration objects inherit from [`PretrainedConfig`] and can be used + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from - [`PretrainedConfig`] for more information. + [`PreTrainedConfig`] for more information. Args: vocab_size (`int`, *optional*, defaults to 32000): diff --git a/vllm/transformers_utils/configs/kimi_k25.py b/vllm/transformers_utils/configs/kimi_k25.py index 095db1d8c4..ed3d05d7cc 100644 --- a/vllm/transformers_utils/configs/kimi_k25.py +++ b/vllm/transformers_utils/configs/kimi_k25.py @@ -7,10 +7,10 @@ A video-chunk is the smallest independently processable unit of video. """ from transformers import DeepseekV3Config -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class KimiK25VisionConfig(PretrainedConfig): +class KimiK25VisionConfig(PreTrainedConfig): model_type = "kimi_k25_vision" def __init__( @@ -59,7 +59,7 @@ class KimiK25VisionConfig(PretrainedConfig): self.projector_ln_eps = projector_ln_eps -class KimiK25Config(PretrainedConfig): +class KimiK25Config(PreTrainedConfig): """Kimi-K2.5 model configuration. Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks. diff --git a/vllm/transformers_utils/configs/kimi_k3.py b/vllm/transformers_utils/configs/kimi_k3.py index 62de6dc03b..6154a438bb 100644 --- a/vllm/transformers_utils/configs/kimi_k3.py +++ b/vllm/transformers_utils/configs/kimi_k3.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Kimi-K3 multimodal configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.logger import init_logger from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig @@ -10,7 +10,7 @@ from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig logger = init_logger(__name__) -class KimiK3VisionConfig(PretrainedConfig): +class KimiK3VisionConfig(PreTrainedConfig): model_type = "kimi_k3_vision" def __init__( @@ -83,7 +83,7 @@ class KimiK3VisionConfig(PretrainedConfig): self.intermediate_size = vt_intermediate_size -class KimiK3Config(PretrainedConfig): +class KimiK3Config(PreTrainedConfig): model_type = "kimi_k3" def __init__( diff --git a/vllm/transformers_utils/configs/kimi_linear.py b/vllm/transformers_utils/configs/kimi_linear.py index 6c2fd335ea..62b3963e1c 100644 --- a/vllm/transformers_utils/configs/kimi_linear.py +++ b/vllm/transformers_utils/configs/kimi_linear.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class KimiLinearConfig(PretrainedConfig): +class KimiLinearConfig(PreTrainedConfig): model_type = "kimi_linear" keys_to_ignore_at_inference = ["past_key_values"] diff --git a/vllm/transformers_utils/configs/kimi_vl.py b/vllm/transformers_utils/configs/kimi_vl.py index 6d992464cb..5733afd668 100644 --- a/vllm/transformers_utils/configs/kimi_vl.py +++ b/vllm/transformers_utils/configs/kimi_vl.py @@ -3,12 +3,12 @@ # Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py from transformers import DeepseekV2Config -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from vllm.transformers_utils.configs.moonvit import MoonViTConfig -class KimiVLConfig(PretrainedConfig): +class KimiVLConfig(PreTrainedConfig): model_type = "kimi_vl" def __init__( diff --git a/vllm/transformers_utils/configs/lfm2_moe.py b/vllm/transformers_utils/configs/lfm2_moe.py index 34682a62a1..903ae9d547 100644 --- a/vllm/transformers_utils/configs/lfm2_moe.py +++ b/vllm/transformers_utils/configs/lfm2_moe.py @@ -2,17 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class Lfm2MoeConfig(PretrainedConfig): +class Lfm2MoeConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`Lfm2MoeModel`]. It is used to instantiate a LFM2 Moe model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LFM2-8B-A1B model. e.g. [LiquidAI/LFM2-8B-A1B](https://huggingface.co/LiquidAI/LFM2-8B-A1B) - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: diff --git a/vllm/transformers_utils/configs/medusa.py b/vllm/transformers_utils/configs/medusa.py index f146c4c5f5..e8cef7a35f 100644 --- a/vllm/transformers_utils/configs/medusa.py +++ b/vllm/transformers_utils/configs/medusa.py @@ -3,12 +3,12 @@ import os -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.transformers_utils.utils import without_trust_remote_code -class MedusaConfig(PretrainedConfig): +class MedusaConfig(PreTrainedConfig): model_type = "medusa" def __init__( diff --git a/vllm/transformers_utils/configs/midashenglm.py b/vllm/transformers_utils/configs/midashenglm.py index f1bbd05710..76d8393e01 100644 --- a/vllm/transformers_utils/configs/midashenglm.py +++ b/vllm/transformers_utils/configs/midashenglm.py @@ -22,13 +22,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import ( Qwen2_5OmniTextConfig, ) -class DashengConfig(PretrainedConfig): +class DashengConfig(PreTrainedConfig): model_type = "midashenglm_dasheng_encoder" def __init__( @@ -80,7 +80,7 @@ class DashengConfig(PretrainedConfig): super().__init__(**kwargs) -class MiDashengLMConfig(PretrainedConfig): +class MiDashengLMConfig(PreTrainedConfig): model_type = "midashenglm" def __init__( diff --git a/vllm/transformers_utils/configs/mimo_v2_omni.py b/vllm/transformers_utils/configs/mimo_v2_omni.py index b87ca22a9a..c0e2f4286c 100644 --- a/vllm/transformers_utils/configs/mimo_v2_omni.py +++ b/vllm/transformers_utils/configs/mimo_v2_omni.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class Mimo_VLVisionConfig(PretrainedConfig): +class Mimo_VLVisionConfig(PreTrainedConfig): model_type = "mimovl" base_config_key = "vision_config" diff --git a/vllm/transformers_utils/configs/minimax_m3.py b/vllm/transformers_utils/configs/minimax_m3.py index c340dda85a..374b854a66 100644 --- a/vllm/transformers_utils/configs/minimax_m3.py +++ b/vllm/transformers_utils/configs/minimax_m3.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class MiniMaxM3TextConfig(PretrainedConfig): +class MiniMaxM3TextConfig(PreTrainedConfig): """Config for the MiniMax M3 text backbone (MiniMaxM3SparseForCausalLM). Defaults mirror the ``text_config`` of the MiniMax-M3-preview checkpoint. @@ -120,7 +120,7 @@ class MiniMaxM3MTPConfig(MiniMaxM3TextConfig): super().__init__(num_hidden_layers=num_hidden_layers, **kwargs) -class MiniMaxM3Config(PretrainedConfig): +class MiniMaxM3Config(PreTrainedConfig): """Top-level MiniMax M3 (VL) config. Holds the text backbone as ``text_config`` so that diff --git a/vllm/transformers_utils/configs/mistral.py b/vllm/transformers_utils/configs/mistral.py index 3ec94eb0be..aaeae9cbcf 100644 --- a/vllm/transformers_utils/configs/mistral.py +++ b/vllm/transformers_utils/configs/mistral.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers import PretrainedConfig, WhisperConfig +from transformers import PreTrainedConfig, WhisperConfig from vllm.logger import init_logger @@ -12,7 +12,7 @@ logger = init_logger(__name__) def adapt_config_dict( config_dict: dict[str, Any], defaults: dict[str, Any], -) -> PretrainedConfig: +) -> PreTrainedConfig: config_dict = _remap_general_mistral_args(config_dict) config_dict = _remap_mistral_sliding_window(config_dict) @@ -86,7 +86,7 @@ def adapt_config_dict( for k, v in defaults.items(): config_dict.setdefault(k, v) - config = PretrainedConfig.from_dict(config_dict) + config = PreTrainedConfig.from_dict(config_dict) logger.debug("Initialized config %s", config) @@ -103,8 +103,8 @@ def _remap_mistral_vision_args(config: dict) -> dict: config = { "model_type": "pixtral", "architectures": ["PixtralForConditionalGeneration"], - "text_config": PretrainedConfig.from_dict(config), - "vision_config": PretrainedConfig.from_dict(vision_config), + "text_config": PreTrainedConfig.from_dict(config), + "vision_config": PreTrainedConfig.from_dict(vision_config), } if quant_config: config["quantization_config"] = quant_config @@ -248,7 +248,7 @@ def _remap_mistral_audio_args(config: dict) -> dict: config = { "model_type": "voxtral", "architectures": [architecture], - "text_config": PretrainedConfig.from_dict(config), + "text_config": PreTrainedConfig.from_dict(config), "audio_config": WhisperConfig( num_mel_bins=encoder_args["audio_encoding_args"]["num_mel_bins"], window_size=encoder_args["audio_encoding_args"]["window_size"], diff --git a/vllm/transformers_utils/configs/mlp_speculator.py b/vllm/transformers_utils/configs/mlp_speculator.py index 6b5a70e3db..f668e5ef52 100644 --- a/vllm/transformers_utils/configs/mlp_speculator.py +++ b/vllm/transformers_utils/configs/mlp_speculator.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class MLPSpeculatorConfig(PretrainedConfig): +class MLPSpeculatorConfig(PreTrainedConfig): model_type = "mlp_speculator" attribute_map = { diff --git a/vllm/transformers_utils/configs/moondream3.py b/vllm/transformers_utils/configs/moondream3.py index 8bff5cd10e..e8a80082c1 100644 --- a/vllm/transformers_utils/configs/moondream3.py +++ b/vllm/transformers_utils/configs/moondream3.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Configuration for Moondream3 model.""" -from transformers import PretrainedConfig +from transformers import PreTrainedConfig -class Moondream3VisionConfig(PretrainedConfig): +class Moondream3VisionConfig(PreTrainedConfig): """Vision encoder configuration for Moondream3.""" model_type = "moondream3_vision" @@ -43,7 +43,7 @@ class Moondream3VisionConfig(PretrainedConfig): self.image_size = crop_size -class Moondream3TextConfig(PretrainedConfig): +class Moondream3TextConfig(PreTrainedConfig): """Text decoder configuration for Moondream3.""" model_type = "moondream3_text" @@ -101,7 +101,7 @@ class Moondream3TextConfig(PretrainedConfig): self.num_experts_per_tok = self.moe_experts_per_token -class Moondream3Config(PretrainedConfig): +class Moondream3Config(PreTrainedConfig): """Combined configuration for Moondream3 multimodal model.""" model_type = "moondream3" diff --git a/vllm/transformers_utils/configs/moonvit.py b/vllm/transformers_utils/configs/moonvit.py index 6e9b2897f4..c7cd34d41d 100644 --- a/vllm/transformers_utils/configs/moonvit.py +++ b/vllm/transformers_utils/configs/moonvit.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class MoonViTConfig(PretrainedConfig): +class MoonViTConfig(PreTrainedConfig): model_type = "moonvit" def __init__( diff --git a/vllm/transformers_utils/configs/moss_transcribe_diarize.py b/vllm/transformers_utils/configs/moss_transcribe_diarize.py index 20abbd8fee..6d57642f39 100644 --- a/vllm/transformers_utils/configs/moss_transcribe_diarize.py +++ b/vllm/transformers_utils/configs/moss_transcribe_diarize.py @@ -3,11 +3,11 @@ from typing import Any -from transformers import PretrainedConfig, Qwen3Config +from transformers import PreTrainedConfig, Qwen3Config from transformers.models.whisper.configuration_whisper import WhisperConfig -class MossTranscribeDiarizeConfig(PretrainedConfig): +class MossTranscribeDiarizeConfig(PreTrainedConfig): """Configuration for MOSS-Transcribe-Diarize.""" model_type = "moss_transcribe_diarize" diff --git a/vllm/transformers_utils/configs/muse_glimmer.py b/vllm/transformers_utils/configs/muse_glimmer.py index 9022c84cf0..80c4c4e171 100644 --- a/vllm/transformers_utils/configs/muse_glimmer.py +++ b/vllm/transformers_utils/configs/muse_glimmer.py @@ -15,7 +15,7 @@ and vision configs are consumed by the native multimodal serving path. from __future__ import annotations from transformers import Qwen3Config -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig def _default_no_rope_layers(num_hidden_layers: int) -> list[int]: @@ -27,7 +27,7 @@ def _default_no_rope_layers(num_hidden_layers: int) -> list[int]: ] -class MuseGlimmerTextConfig(PretrainedConfig): +class MuseGlimmerTextConfig(PreTrainedConfig): model_type = "muse_glimmer_text" keys_to_ignore_at_inference = ["past_key_values"] @@ -127,7 +127,7 @@ class MuseGlimmerTextConfig(PretrainedConfig): ) -class MuseGlimmerVisionConfig(PretrainedConfig): +class MuseGlimmerVisionConfig(PreTrainedConfig): model_type = "muse_glimmer_vision" def __init__( @@ -177,7 +177,7 @@ class MuseGlimmerVisionConfig(PretrainedConfig): self.layer_types = layer_types -class MuseGlimmerConfig(PretrainedConfig): +class MuseGlimmerConfig(PreTrainedConfig): model_type = "muse_glimmer" sub_configs = { "text_config": MuseGlimmerTextConfig, diff --git a/vllm/transformers_utils/configs/nemotron.py b/vllm/transformers_utils/configs/nemotron.py index bb15a8b433..135a0e1596 100644 --- a/vllm/transformers_utils/configs/nemotron.py +++ b/vllm/transformers_utils/configs/nemotron.py @@ -17,22 +17,22 @@ # limitations under the License. """Nemotron model configuration.""" -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.utils import logging logger = logging.get_logger(__name__) -class NemotronConfig(PretrainedConfig): +class NemotronConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`NemotronModel`]. It is used to instantiate a Nemotron model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the Nemotron-8B. - Configuration objects inherit from [`PretrainedConfig`] and can be + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from - [`PretrainedConfig`] for more information. + [`PreTrainedConfig`] for more information. Args: diff --git a/vllm/transformers_utils/configs/nemotron_h.py b/vllm/transformers_utils/configs/nemotron_h.py index c83313edfb..c25a5d54e3 100644 --- a/vllm/transformers_utils/configs/nemotron_h.py +++ b/vllm/transformers_utils/configs/nemotron_h.py @@ -18,13 +18,13 @@ """NemotronH model configuration.""" import regex as re -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.utils import logging logger = logging.get_logger(__name__) -class NemotronHConfig(PretrainedConfig): +class NemotronHConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a NemotronH model according to the specified arguments, defining the model architecture. Instantiating diff --git a/vllm/transformers_utils/configs/olmo_hybrid.py b/vllm/transformers_utils/configs/olmo_hybrid.py index 6e12cdd274..62d3c4e0d5 100644 --- a/vllm/transformers_utils/configs/olmo_hybrid.py +++ b/vllm/transformers_utils/configs/olmo_hybrid.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class OlmoHybridConfig(PretrainedConfig): +class OlmoHybridConfig(PreTrainedConfig): r"""Configuration class for [`OlmoHybridModel`]. It is used to instantiate an OLMo Hybrid model according to the specified arguments, defining the model architecture. Instantiating a diff --git a/vllm/transformers_utils/configs/openvla.py b/vllm/transformers_utils/configs/openvla.py index 897af8f705..dba2f7f5a2 100644 --- a/vllm/transformers_utils/configs/openvla.py +++ b/vllm/transformers_utils/configs/openvla.py @@ -9,10 +9,10 @@ configuration without executing Hugging Face remote code. from typing import Any -from transformers import LlamaConfig, PretrainedConfig +from transformers import LlamaConfig, PreTrainedConfig -class OpenVLAConfig(PretrainedConfig): +class OpenVLAConfig(PreTrainedConfig): """Configuration class for OpenVLA models.""" model_type = "openvla" diff --git a/vllm/transformers_utils/configs/ovis.py b/vllm/transformers_utils/configs/ovis.py index 64629566a3..bc84bfe812 100644 --- a/vllm/transformers_utils/configs/ovis.py +++ b/vllm/transformers_utils/configs/ovis.py @@ -7,10 +7,10 @@ # Ovis Config with AimV2 config registration removed for Transformers compatibility from typing import Any -from transformers import AutoConfig, PretrainedConfig +from transformers import AutoConfig, PreTrainedConfig -class AIMv2Config(PretrainedConfig): +class AIMv2Config(PreTrainedConfig): """This is the configuration class to store the configuration of an [`AIMv2Model`]. Instantiating a configuration with the defaults will yield a similar configuration to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224). @@ -29,7 +29,7 @@ class AIMv2Config(PretrainedConfig): projection_dropout: Dropout ratio for the projection layer after the attention. qkv_bias: Whether to add a bias to the queries, keys and values. use_bias: Whether to add a bias in the feed-forward and projection layers. - kwargs: Keyword arguments for the [`PretrainedConfig`]. + kwargs: Keyword arguments for the [`PreTrainedConfig`]. """ @@ -70,7 +70,7 @@ class AIMv2Config(PretrainedConfig): # ---------------------------------------------------------------------- # Visual Tokenizer Configuration # ---------------------------------------------------------------------- -class BaseVisualTokenizerConfig(PretrainedConfig): +class BaseVisualTokenizerConfig(PreTrainedConfig): def __init__( self, vocab_size=16384, @@ -78,7 +78,7 @@ class BaseVisualTokenizerConfig(PretrainedConfig): tau=1.0, depths=None, drop_cls_token=False, - backbone_config: PretrainedConfig | dict | None = None, + backbone_config: PreTrainedConfig | dict | None = None, hidden_stride: int = 1, **kwargs, ): @@ -92,10 +92,10 @@ class BaseVisualTokenizerConfig(PretrainedConfig): self.backbone_kwargs = dict[str, Any]() self.drop_cls_token = drop_cls_token if backbone_config is not None: - assert isinstance(backbone_config, (PretrainedConfig, dict)), ( - f"expect `backbone_config` to be instance of PretrainedConfig or dict, but got {type(backbone_config)} type" + assert isinstance(backbone_config, (PreTrainedConfig, dict)), ( + f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type" ) - if not isinstance(backbone_config, PretrainedConfig): + if not isinstance(backbone_config, PreTrainedConfig): model_type = backbone_config["model_type"] if model_type != "aimv2": backbone_config.pop("model_type") @@ -139,13 +139,13 @@ AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig) # ---------------------------------------------------------------------- # Ovis Configuration # ---------------------------------------------------------------------- -class OvisConfig(PretrainedConfig): +class OvisConfig(PreTrainedConfig): model_type = "ovis" def __init__( self, - llm_config: PretrainedConfig | dict | None = None, - visual_tokenizer_config: PretrainedConfig | dict | None = None, + llm_config: PreTrainedConfig | dict | None = None, + visual_tokenizer_config: PreTrainedConfig | dict | None = None, multimodal_max_length=8192, hidden_size=None, conversation_formatter_class=None, @@ -155,10 +155,10 @@ class OvisConfig(PretrainedConfig): ): super().__init__(**kwargs) if llm_config is not None: - assert isinstance(llm_config, (PretrainedConfig, dict)), ( - f"expect `llm_config` to be instance of PretrainedConfig or dict, but got {type(llm_config)} type" + assert isinstance(llm_config, (PreTrainedConfig, dict)), ( + f"expect `llm_config` to be instance of PreTrainedConfig or dict, but got {type(llm_config)} type" ) - if not isinstance(llm_config, PretrainedConfig): + if not isinstance(llm_config, PreTrainedConfig): model_type = llm_config["model_type"] llm_config.pop("model_type") llm_config = AutoConfig.for_model(model_type, **llm_config) @@ -166,10 +166,10 @@ class OvisConfig(PretrainedConfig): # map llm_config to text_config self.text_config = llm_config if visual_tokenizer_config is not None: - assert isinstance(visual_tokenizer_config, (PretrainedConfig, dict)), ( - f"expect `visual_tokenizer_config` to be instance of PretrainedConfig or dict, but got {type(visual_tokenizer_config)} type" + assert isinstance(visual_tokenizer_config, (PreTrainedConfig, dict)), ( + f"expect `visual_tokenizer_config` to be instance of PreTrainedConfig or dict, but got {type(visual_tokenizer_config)} type" ) - if not isinstance(visual_tokenizer_config, PretrainedConfig): + if not isinstance(visual_tokenizer_config, PreTrainedConfig): model_type = visual_tokenizer_config["model_type"] visual_tokenizer_config.pop("model_type") visual_tokenizer_config = AutoConfig.for_model( diff --git a/vllm/transformers_utils/configs/parakeet.py b/vllm/transformers_utils/configs/parakeet.py index febd1da446..c574317f81 100644 --- a/vllm/transformers_utils/configs/parakeet.py +++ b/vllm/transformers_utils/configs/parakeet.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass -from transformers import ParakeetEncoderConfig, PretrainedConfig +from transformers import ParakeetEncoderConfig, PreTrainedConfig class ParakeetConfig(ParakeetEncoderConfig): @@ -24,9 +24,9 @@ class ParakeetConfig(ParakeetEncoderConfig): @staticmethod def from_hf_config( - config: PretrainedConfig, *, llm_hidden_size: int, max_model_len: int + config: PreTrainedConfig, *, llm_hidden_size: int, max_model_len: int ) -> "ParakeetConfig": - assert isinstance(config, PretrainedConfig) + assert isinstance(config, PreTrainedConfig) return ParakeetConfig( **config.to_dict(), scale_input=False, @@ -55,8 +55,8 @@ class ExtractorConfig: padding_value: float = 0.0 @classmethod - def from_hf_config(cls, config: PretrainedConfig) -> "ExtractorConfig": - assert isinstance(config, PretrainedConfig) + def from_hf_config(cls, config: PreTrainedConfig) -> "ExtractorConfig": + assert isinstance(config, PreTrainedConfig) defaults = ("hop_length", "win_length", "preemphasis", "n_fft", "padding_value") optional_kwargs = { name: getattr(config, name) for name in defaults if hasattr(config, name) diff --git a/vllm/transformers_utils/configs/qianfan_ocr.py b/vllm/transformers_utils/configs/qianfan_ocr.py index da004bb90f..73df82c8fe 100644 --- a/vllm/transformers_utils/configs/qianfan_ocr.py +++ b/vllm/transformers_utils/configs/qianfan_ocr.py @@ -3,11 +3,11 @@ from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from transformers.models.auto import CONFIG_MAPPING -class QianfanOCRVisionConfig(PretrainedConfig): +class QianfanOCRVisionConfig(PreTrainedConfig): model_type = "qianfan_ocr_vision" def __init__( @@ -53,7 +53,7 @@ class QianfanOCRVisionConfig(PretrainedConfig): self.use_mean_pooling = use_mean_pooling -class QianfanOCRConfig(PretrainedConfig): +class QianfanOCRConfig(PreTrainedConfig): model_type = "qianfan_ocr" def __init__( diff --git a/vllm/transformers_utils/configs/qwen3_5.py b/vllm/transformers_utils/configs/qwen3_5.py index e30cbe6502..57f64f0f78 100644 --- a/vllm/transformers_utils/configs/qwen3_5.py +++ b/vllm/transformers_utils/configs/qwen3_5.py @@ -16,10 +16,10 @@ # limitations under the License. """Qwen3.5 model configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class Qwen3_5TextConfig(PretrainedConfig): +class Qwen3_5TextConfig(PreTrainedConfig): model_type = "qwen3_5_text" keys_to_ignore_at_inference = ["past_key_values"] @@ -108,7 +108,7 @@ class Qwen3_5TextConfig(PretrainedConfig): self.linear_num_value_heads = linear_num_value_heads super().__init__(**kwargs) # Set these AFTER super().__init__() because transformers v4's - # PretrainedConfig.__init__ has these as explicit params with different + # PreTrainedConfig.__init__ has these as explicit params with different # defaults (e.g. tie_word_embeddings=True) that would overwrite our values. self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id @@ -116,7 +116,7 @@ class Qwen3_5TextConfig(PretrainedConfig): self.tie_word_embeddings = tie_word_embeddings -class Qwen3_5VisionConfig(PretrainedConfig): +class Qwen3_5VisionConfig(PreTrainedConfig): model_type = "qwen3_5" base_config_key = "vision_config" @@ -152,7 +152,7 @@ class Qwen3_5VisionConfig(PretrainedConfig): self.initializer_range = initializer_range -class Qwen3_5Config(PretrainedConfig): +class Qwen3_5Config(PreTrainedConfig): model_type = "qwen3_5" sub_configs = { "vision_config": Qwen3_5VisionConfig, @@ -186,7 +186,7 @@ class Qwen3_5Config(PretrainedConfig): self.vision_start_token_id = vision_start_token_id self.vision_end_token_id = vision_end_token_id super().__init__(**kwargs) - # Set after super().__init__() to avoid v4 PretrainedConfig overwrite + # Set after super().__init__() to avoid v4 PreTrainedConfig overwrite self.tie_word_embeddings = tie_word_embeddings diff --git a/vllm/transformers_utils/configs/qwen3_5_moe.py b/vllm/transformers_utils/configs/qwen3_5_moe.py index 938272d028..34e66c825a 100644 --- a/vllm/transformers_utils/configs/qwen3_5_moe.py +++ b/vllm/transformers_utils/configs/qwen3_5_moe.py @@ -16,10 +16,10 @@ # limitations under the License. """Qwen3.5-MoE model configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class Qwen3_5MoeTextConfig(PretrainedConfig): +class Qwen3_5MoeTextConfig(PreTrainedConfig): model_type = "qwen3_5_moe_text" keys_to_ignore_at_inference = ["past_key_values"] @@ -120,7 +120,7 @@ class Qwen3_5MoeTextConfig(PretrainedConfig): self.router_aux_loss_coef = router_aux_loss_coef super().__init__(**kwargs) # Set these AFTER super().__init__() because transformers v4's - # PretrainedConfig.__init__ has these as explicit params with different + # PreTrainedConfig.__init__ has these as explicit params with different # defaults (e.g. tie_word_embeddings=True) that would overwrite our values. self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id @@ -128,7 +128,7 @@ class Qwen3_5MoeTextConfig(PretrainedConfig): self.tie_word_embeddings = tie_word_embeddings -class Qwen3_5MoeVisionConfig(PretrainedConfig): +class Qwen3_5MoeVisionConfig(PreTrainedConfig): model_type = "qwen3_5_moe" base_config_key = "vision_config" @@ -164,7 +164,7 @@ class Qwen3_5MoeVisionConfig(PretrainedConfig): self.initializer_range = initializer_range -class Qwen3_5MoeConfig(PretrainedConfig): +class Qwen3_5MoeConfig(PreTrainedConfig): model_type = "qwen3_5_moe" sub_configs = { "vision_config": Qwen3_5MoeVisionConfig, @@ -198,7 +198,7 @@ class Qwen3_5MoeConfig(PretrainedConfig): self.vision_start_token_id = vision_start_token_id self.vision_end_token_id = vision_end_token_id super().__init__(**kwargs) - # Set after super().__init__() to avoid v4 PretrainedConfig overwrite + # Set after super().__init__() to avoid v4 PreTrainedConfig overwrite self.tie_word_embeddings = tie_word_embeddings diff --git a/vllm/transformers_utils/configs/qwen3_asr.py b/vllm/transformers_utils/configs/qwen3_asr.py index 652c02525f..77125e4b22 100644 --- a/vllm/transformers_utils/configs/qwen3_asr.py +++ b/vllm/transformers_utils/configs/qwen3_asr.py @@ -17,14 +17,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.modeling_rope_utils import rope_config_validation from transformers.utils import logging logger = logging.get_logger(__name__) -class Qwen3ASRAudioEncoderConfig(PretrainedConfig): +class Qwen3ASRAudioEncoderConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a @@ -33,8 +33,8 @@ class Qwen3ASRAudioEncoderConfig(PretrainedConfig): e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: num_mel_bins (`int`, *optional*, defaults to 128): @@ -130,15 +130,15 @@ class Qwen3ASRAudioEncoderConfig(PretrainedConfig): self.downsample_hidden_size = downsample_hidden_size -class Qwen3ASRTextConfig(PretrainedConfig): +class Qwen3ASRTextConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: vocab_size (`int`, *optional*, defaults to 151936): @@ -285,7 +285,7 @@ class Qwen3ASRTextConfig(PretrainedConfig): super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) -class Qwen3ASRThinkerConfig(PretrainedConfig): +class Qwen3ASRThinkerConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a @@ -294,8 +294,8 @@ class Qwen3ASRThinkerConfig(PretrainedConfig): e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: audio_config (`dict`, *optional*): @@ -365,7 +365,7 @@ class Qwen3ASRThinkerConfig(PretrainedConfig): self.audio_token_id = audio_token_id -class Qwen3ASRConfig(PretrainedConfig): +class Qwen3ASRConfig(PreTrainedConfig): """ This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR model according to the specified sub-models configurations, defining the model architecture. @@ -373,8 +373,8 @@ class Qwen3ASRConfig(PretrainedConfig): Instantiating a configuration with the defaults will yield a similar configuration to that of the [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture. - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model. @@ -437,7 +437,7 @@ class Qwen3ASRConfig(PretrainedConfig): self.timestamp_segment_time = timestamp_segment_time super().__init__(**kwargs) - def get_text_config(self, decoder=False) -> "PretrainedConfig": + def get_text_config(self, decoder=False) -> "PreTrainedConfig": """ Returns the config that is meant to be used with text IO. On most models, it is the original config instance itself. On specific composite models, it is under a set of valid names. diff --git a/vllm/transformers_utils/configs/qwen3_next.py b/vllm/transformers_utils/configs/qwen3_next.py index 8460725bfd..8b083ff963 100644 --- a/vllm/transformers_utils/configs/qwen3_next.py +++ b/vllm/transformers_utils/configs/qwen3_next.py @@ -16,20 +16,20 @@ # limitations under the License. """Qwen3-Next model configuration.""" -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.utils import logging logger = logging.get_logger(__name__) -class Qwen3NextConfig(PretrainedConfig): +class Qwen3NextConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a Qwen3-Next model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct). - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. Args: diff --git a/vllm/transformers_utils/configs/radio.py b/vllm/transformers_utils/configs/radio.py index 4a3e468f84..89a56a85ef 100644 --- a/vllm/transformers_utils/configs/radio.py +++ b/vllm/transformers_utils/configs/radio.py @@ -4,7 +4,7 @@ from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig from transformers.utils import logging logger = logging.get_logger(__name__) @@ -20,7 +20,7 @@ OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711) -class RadioConfig(PretrainedConfig): +class RadioConfig(PreTrainedConfig): r"""This is the configuration class to store the configuration of a Radio vision model. It is used to instantiate a Radio model according to the specified arguments, defining the model architecture. diff --git a/vllm/transformers_utils/configs/speculators/base.py b/vllm/transformers_utils/configs/speculators/base.py index a867749dba..cb188a1ef0 100644 --- a/vllm/transformers_utils/configs/speculators/base.py +++ b/vllm/transformers_utils/configs/speculators/base.py @@ -4,7 +4,7 @@ import os from dataclasses import fields from typing import Any -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm.transformers_utils.configs.speculators.algos import ( SUPPORTED_SPECULATORS_TYPES, @@ -12,13 +12,13 @@ from vllm.transformers_utils.configs.speculators.algos import ( from vllm.transformers_utils.utils import without_trust_remote_code -class SpeculatorsConfig(PretrainedConfig): +class SpeculatorsConfig(PreTrainedConfig): model_type = "speculators" def __init__(self, **kwargs): # super().__init__ performs some validation before setting all kwargs as # attributes, so we set them first to be safe - pre_trained_config_fields = {f.name for f in fields(PretrainedConfig)} + pre_trained_config_fields = {f.name for f in fields(PreTrainedConfig)} super_kwargs = dict() for key, value in kwargs.items(): if key == "model_type": diff --git a/vllm/transformers_utils/configs/step3_vl.py b/vllm/transformers_utils/configs/step3_vl.py index 0ee650a704..c19030e554 100644 --- a/vllm/transformers_utils/configs/step3_vl.py +++ b/vllm/transformers_utils/configs/step3_vl.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class Step3VisionEncoderConfig(PretrainedConfig): +class Step3VisionEncoderConfig(PreTrainedConfig): model_type = "step3_vision_encoder" def __init__( @@ -35,7 +35,7 @@ class Step3VisionEncoderConfig(PretrainedConfig): super().__init__(**kwargs) -class Step3TextConfig(PretrainedConfig): +class Step3TextConfig(PreTrainedConfig): model_type = "step3_text" architectures = ["Step3TextForCausalLM"] @@ -146,7 +146,7 @@ class Step3TextConfig(PretrainedConfig): super().__init__(**kwargs) -class Step3VLConfig(PretrainedConfig): +class Step3VLConfig(PreTrainedConfig): model_type = "step3_vl" def __init__( diff --git a/vllm/transformers_utils/configs/step3p5.py b/vllm/transformers_utils/configs/step3p5.py index d67b9dfff5..1b8ac82a79 100644 --- a/vllm/transformers_utils/configs/step3p5.py +++ b/vllm/transformers_utils/configs/step3p5.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from transformers.configuration_utils import PretrainedConfig +from transformers.configuration_utils import PreTrainedConfig -class Step3p5Config(PretrainedConfig): +class Step3p5Config(PreTrainedConfig): model_type = "step3p5" def __init__( diff --git a/vllm/transformers_utils/configs/ultravox.py b/vllm/transformers_utils/configs/ultravox.py index b8bd003199..669fe7be48 100644 --- a/vllm/transformers_utils/configs/ultravox.py +++ b/vllm/transformers_utils/configs/ultravox.py @@ -7,14 +7,14 @@ from typing import Any import transformers -class UltravoxConfig(transformers.PretrainedConfig): +class UltravoxConfig(transformers.PreTrainedConfig): r"""This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an Ultravox model according to the specified arguments, defining the model architecture. - Configuration objects inherit from [`PretrainedConfig`] and can be used to - control the model outputs. Read the documentation from [`PretrainedConfig`] + Configuration objects inherit from [`PreTrainedConfig`] and can be used to + control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: @@ -74,7 +74,7 @@ class UltravoxConfig(transformers.PretrainedConfig): self.num_projector_layers = num_projector_layers # N.B. May set the wrapped_model_config below. - self.wrapped_model_config: transformers.PretrainedConfig + self.wrapped_model_config: transformers.PreTrainedConfig self.text_model_id = text_model_id if text_model_id is None: text_config = text_config or {} @@ -113,7 +113,7 @@ class UltravoxConfig(transformers.PretrainedConfig): return super().__setattr__(key, value) @property - def text_config(self) -> transformers.PretrainedConfig: + def text_config(self) -> transformers.PreTrainedConfig: # When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate # the full model, but the text config is the text config of the inner # model. diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 72f2920480..9386e9a328 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -5,7 +5,7 @@ from typing import final import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE -from transformers import PretrainedConfig +from transformers import PreTrainedConfig from vllm import envs from vllm.config.model_arch import ( @@ -26,8 +26,8 @@ logger = init_logger(__name__) class ModelArchConfigConvertorBase: def __init__( self, - hf_config: PretrainedConfig, - hf_text_config: PretrainedConfig, + hf_config: PreTrainedConfig, + hf_text_config: PreTrainedConfig, revision: str | None = None, ): self.hf_config = hf_config @@ -36,7 +36,7 @@ class ModelArchConfigConvertorBase: def get_per_layer_hf_configs( self, - ) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None: + ) -> list[tuple[PreTrainedConfig, PreTrainedConfig]] | None: """`(hf_config, hf_text_config)` per layer, or `None` if homogeneous. This is the only place that decides whether a checkpoint is heterogeneous @@ -219,7 +219,7 @@ class ModelArchConfigConvertorBase: @classmethod def get_torch_dtype( cls, - hf_config: PretrainedConfig, + hf_config: PreTrainedConfig, model_id: str, revision: str | None, config_format: str | ConfigFormat, @@ -257,7 +257,7 @@ class ModelArchConfigConvertorBase: return config_dtype - def _normalize_quantization_config(self, config: PretrainedConfig): + def _normalize_quantization_config(self, config: PreTrainedConfig): quant_cfg = getattr(config, "quantization_config", None) if quant_cfg is None: # compressed-tensors uses a "compression_config" key @@ -585,8 +585,8 @@ class DeepSeekMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): class DeepseekV4ModelArchConfigConvertor(ModelArchConfigConvertorBase): def __init__( self, - hf_config: PretrainedConfig, - hf_text_config: PretrainedConfig, + hf_config: PreTrainedConfig, + hf_text_config: PreTrainedConfig, revision: str | None = None, ): # DeepSeek-V4-Flash-Vision-Exp ships the same architectures/model_type @@ -622,7 +622,7 @@ class MimoMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def _strip_mimo_v2_attention_chunk_size( - hf_config: PretrainedConfig, hf_text_config: PretrainedConfig + hf_config: PreTrainedConfig, hf_text_config: PreTrainedConfig ) -> None: # MiMo-V2-Flash's config.json sets `attention_chunk_size=128` but the # architecture does not actually use chunked local attention. Leaving it @@ -635,8 +635,8 @@ def _strip_mimo_v2_attention_chunk_size( class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase): def __init__( self, - hf_config: PretrainedConfig, - hf_text_config: PretrainedConfig, + hf_config: PreTrainedConfig, + hf_text_config: PreTrainedConfig, revision: str | None = None, ): if getattr(hf_config, "vision_config", None): @@ -648,8 +648,8 @@ class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase): class MimoV2MTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def __init__( self, - hf_config: PretrainedConfig, - hf_text_config: PretrainedConfig, + hf_config: PreTrainedConfig, + hf_text_config: PreTrainedConfig, revision: str | None = None, ): super().__init__(hf_config, hf_text_config, revision) @@ -742,7 +742,7 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase): def get_per_layer_hf_configs( self, - ) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None: + ) -> list[tuple[PreTrainedConfig, PreTrainedConfig]] | None: # Gemma4 uses a larger head dimension, and sometimes more KV heads, on its # full attention layers than on its sliding ones. Transformers >= 5.15.0 # says so in the config; before that the values are flat attributes picked @@ -763,7 +763,7 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase): class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase): - def _language_config(self) -> PretrainedConfig: + def _language_config(self) -> PreTrainedConfig: return self.hf_config.language_config def get_num_hidden_layers(self) -> int: @@ -813,6 +813,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = { "dbrx": DbrxModelArchConfigConvertor, "deepseek_mtp": DeepSeekMTPModelArchConfigConvertor, "deepseek_v4": DeepseekV4ModelArchConfigConvertor, + "diffusion_gemma": Gemma4ModelArchConfigConvertor, "diffusion_gemma_text": Gemma4ModelArchConfigConvertor, "ernie_mtp": ErnieMTPModelArchConfigConvertor, "falcon": FalconModelArchConfigConvertor, diff --git a/vllm/transformers_utils/processors/funasr.py b/vllm/transformers_utils/processors/funasr.py index aaa0bec48c..0a2f9ca6e1 100644 --- a/vllm/transformers_utils/processors/funasr.py +++ b/vllm/transformers_utils/processors/funasr.py @@ -164,56 +164,6 @@ class WavFrontend(nn.Module): feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) return feats_pad, feats_lens - def forward_fbank( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - ) - - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - def forward_lfr_cmvn( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - mat = input[i, : input_lengths[i], :] - if self.lfr_m != 1 or self.lfr_n != 1: - mat = apply_lfr(mat, self.lfr_m, self.lfr_n) - if self.cmvn is not None: - mat = apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - class FunASRFeatureExtractor(SequenceFeatureExtractor): r"""Constructs a FunASR feature extractor. diff --git a/vllm/transformers_utils/processors/glm4v.py b/vllm/transformers_utils/processors/glm4v.py index e9ee497cea..605cd7e34a 100644 --- a/vllm/transformers_utils/processors/glm4v.py +++ b/vllm/transformers_utils/processors/glm4v.py @@ -4,12 +4,12 @@ # Adapted from # https://github.com/zai-org/CogAgent from transformers import PythonBackend -from transformers.image_processing_utils_fast import BaseImageProcessorFast +from transformers.image_processing_backends import TorchvisionBackend from transformers.image_utils import PILImageResampling from transformers.processing_utils import ProcessorMixin -class GLM4VImageProcessorFast(BaseImageProcessorFast): +class GLM4VImageProcessorFast(TorchvisionBackend): """Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177 to HF Transformers. """ diff --git a/vllm/transformers_utils/processors/glm5next.py b/vllm/transformers_utils/processors/glm5next.py index b6988d3b8f..a6cba9d77f 100644 --- a/vllm/transformers_utils/processors/glm5next.py +++ b/vllm/transformers_utils/processors/glm5next.py @@ -7,12 +7,12 @@ import math import numpy as np import torch from torchvision.transforms.v2 import functional as tvF -from transformers.image_processing_utils import BatchFeature -from transformers.image_processing_utils_fast import ( - BaseImageProcessorFast, +from transformers.image_processing_backends import ( + TorchvisionBackend, group_images_by_shape, reorder_images, ) +from transformers.image_processing_utils import BatchFeature from transformers.image_utils import ( OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, @@ -327,7 +327,7 @@ class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[c max_image_tokens: int | None -class Glm5NextImageProcessor(BaseImageProcessorFast): +class Glm5NextImageProcessor(TorchvisionBackend): """Fast torchvision image processor for GLM-5.3-Flash. ``patch_expand_factor`` multiplies into the ``smart_resize`` spatial @@ -759,7 +759,7 @@ class Glm5NextProcessor(ProcessorMixin): attributes = ["image_processor", "tokenizer", "video_processor"] image_processor_class = "AutoImageProcessor" video_processor_class = "AutoVideoProcessor" - tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast") + tokenizer_class = ("PythonBackend", "TokenizersBackend") def __init__( self, diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py index e59a50ccbe..17a915b2d6 100644 --- a/vllm/transformers_utils/processors/minicpmo.py +++ b/vllm/transformers_utils/processors/minicpmo.py @@ -17,12 +17,11 @@ """Processor class for MiniCPMO.""" import math -from typing import Literal, TypeAlias +from typing import TypeAlias import numpy as np import regex import torch -import torchaudio from transformers.image_processing_utils import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessorMixin @@ -522,66 +521,3 @@ class MiniCPMOProcessor(ProcessorMixin): padding_length.append(tensor.shape[-1] - len(item)) return tensor, padding_length - - -class MelSpectrogramFeatures(torch.nn.Module): - def __init__( - self, - sample_rate=24000, - n_fft=1024, - hop_length=256, - n_mels=100, - padding: Literal["center", "same"] = "center", - ): - super().__init__() - if padding not in ["center", "same"]: - raise ValueError("Padding must be 'center' or 'same'.") - self.padding = padding - self.mel_spec = torchaudio.transforms.MelSpectrogram( - sample_rate=sample_rate, - n_fft=n_fft, - hop_length=hop_length, - n_mels=n_mels, - center=padding == "center", - power=1, - ) - - def __call__(self, audio: torch.Tensor) -> torch.Tensor: - """audio: Tensor([num_channels, num_samples]).""" - return super().__call__(audio) - - def forward(self, audio: torch.Tensor) -> torch.Tensor: - """audio: Tensor([num_channels, num_samples]).""" - mel: torch.Tensor = self.mel_spec(audio) - features = torch.log(torch.clip(mel, min=1e-5)) - return features - - -class ChatTTSProcessor: - def __init__(self, text_tokenizer): - self.audio_processor = MelSpectrogramFeatures() - self.text_tokenizer = text_tokenizer - - def __call__(self, text_list, audio_list): - assert len(text_list) == len(audio_list) - input_ids_varlen = [] - for text in text_list: - input_ids_ = self.text_tokenizer.encode( - text, return_tensors="pt", add_special_tokens=False - ) # [1, seq_len] - input_ids_ = input_ids_.squeeze(0) # [seq_len] - input_ids_varlen.append(input_ids_) - - audio_features_varlen = [] - for audio in audio_list: - assert audio.shape.__len__() == 1 # [seq_len] - try: - mel = self.audio_processor(audio) # [100(num_mel_bins), seq_len_mel] - except Exception as e: - raise e - audio_features_varlen.append(mel) - - return { - "tts_input_ids_varlen": input_ids_varlen, # return List[Tensor] - "tts_input_features_varlen": audio_features_varlen, # return List[Tensor] - } diff --git a/vllm/transformers_utils/processors/minimax_m3.py b/vllm/transformers_utils/processors/minimax_m3.py index 60fa7ed536..237b1b5f1e 100644 --- a/vllm/transformers_utils/processors/minimax_m3.py +++ b/vllm/transformers_utils/processors/minimax_m3.py @@ -18,8 +18,8 @@ import math import torch from torchvision.transforms import InterpolationMode from transformers import AutoTokenizer, BatchFeature -from transformers.image_processing_utils_fast import ( - BaseImageProcessorFast, +from transformers.image_processing_backends import ( + TorchvisionBackend, group_images_by_shape, reorder_images, ) @@ -156,7 +156,7 @@ class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): # type: ignor max_long_side_pixel: int -class MiniMaxM3VLImageProcessor(BaseImageProcessorFast): +class MiniMaxM3VLImageProcessor(TorchvisionBackend): do_resize = True resample = PILImageResampling.BICUBIC # required by base-class validation, not used as the resize bound @@ -522,9 +522,14 @@ class MiniMaxVLProcessor(ProcessorMixin): # register() API now stores classes as {"pil": cls} dicts in # _extra_content, but get_possibly_dynamic_module() still calls # .__name__ on the raw value, crashing with AttributeError on dicts. - tokenizer = AutoTokenizer.from_pretrained( - pretrained_model_name_or_path, **kwargs - ) + # + # Reuse the tokenizer passed by vLLM instead of loading another one, + # and keep it out of the kwargs forwarded to the sub-processor loaders. + tokenizer = kwargs.pop("tokenizer", None) + if tokenizer is None: + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, **kwargs + ) image_processor = MiniMaxM3VLImageProcessor.from_pretrained( pretrained_model_name_or_path, **kwargs ) diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py index 22bdeef21a..321fc95171 100644 --- a/vllm/transformers_utils/processors/moondream3.py +++ b/vllm/transformers_utils/processors/moondream3.py @@ -209,7 +209,6 @@ class Moondream3Processor(ProcessorMixin): "revision", "subfolder", "token", - "use_fast", ): if key in kwargs: tokenizer_kwargs[key] = kwargs[key] diff --git a/vllm/transformers_utils/processors/muse_glimmer.py b/vllm/transformers_utils/processors/muse_glimmer.py index c1a5326ea1..baacbcf0c1 100644 --- a/vllm/transformers_utils/processors/muse_glimmer.py +++ b/vllm/transformers_utils/processors/muse_glimmer.py @@ -437,7 +437,7 @@ class MuseGlimmerProcessor(ProcessorMixin): attributes = ["image_processor", "video_processor", "tokenizer"] image_processor_class = "AutoImageProcessor" video_processor_class = "AutoVideoProcessor" - tokenizer_class = "PreTrainedTokenizerFast" + tokenizer_class = "TokenizersBackend" def __init__( self, diff --git a/vllm/transformers_utils/processors/nano_nemotron_vl.py b/vllm/transformers_utils/processors/nano_nemotron_vl.py index 0e29cd4cec..9572b7c4a7 100644 --- a/vllm/transformers_utils/processors/nano_nemotron_vl.py +++ b/vllm/transformers_utils/processors/nano_nemotron_vl.py @@ -20,7 +20,7 @@ import numpy.typing as npt import regex as re import torch from PIL import Image -from transformers import BatchFeature, PretrainedConfig, TensorType +from transformers import BatchFeature, PreTrainedConfig, TensorType from vllm.model_executor.models.parakeet import ParakeetExtractor from vllm.multimodal.inputs import AudioItem @@ -584,7 +584,7 @@ class BaseNanoNemotronVLProcessor(ABC): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, tokenizer: HfTokenizer, *args, max_model_len: int, @@ -623,7 +623,7 @@ class BaseNanoNemotronVLProcessor(ABC): self.dtype: torch.dtype = getattr(config, "dtype", torch.float32) @staticmethod - def use_dynamic_resolution(config: PretrainedConfig) -> bool: + def use_dynamic_resolution(config: PreTrainedConfig) -> bool: return "min_num_patches" in config.vision_config.args @property @@ -771,7 +771,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor): def __init__( self, - config: PretrainedConfig, + config: PreTrainedConfig, tokenizer: HfTokenizer, *, max_model_len: int, diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py index a77072745f..9ad320be39 100644 --- a/vllm/transformers_utils/processors/ovis.py +++ b/vllm/transformers_utils/processors/ovis.py @@ -110,7 +110,7 @@ class OvisProcessor(ProcessorMixin): ) -> BatchFeature: """Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode - the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to + the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. Args: diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py index c31ea91520..9e4e47738a 100644 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ b/vllm/transformers_utils/processors/ovis2_5.py @@ -96,9 +96,14 @@ class Ovis2_5Processor(ProcessorMixin): # missing from the vocab. Re-add them to restore the expected ids. self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True) + # in TokenizerPool tokenizer.get_added_vocab() gets values from an instance + # that was updated with tokenizer.add_tokens(), so it is favourable over + # self.tokenizer.convert_tokens_to_ids(), which refers to a copy before + # mutation by tokenizer.add_tokens() + added_vocab = self.tokenizer.get_added_vocab() + return { - key: self.tokenizer.convert_tokens_to_ids(token_name) - for key, token_name in required_tokens.items() + key: added_vocab[token_name] for key, token_name in required_tokens.items() } def __call__( @@ -115,7 +120,7 @@ class Ovis2_5Processor(ProcessorMixin): and image(s). This method forwards the `text`and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode the text. To prepare the vision inputs, - this method forwards the `vision_infos` and `kwrags` arguments to + this method forwards the `vision_infos` and `kwargs` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. diff --git a/vllm/transformers_utils/repo_utils.py b/vllm/transformers_utils/repo_utils.py index 6b9ff9ec5e..b6a23b007a 100644 --- a/vllm/transformers_utils/repo_utils.py +++ b/vllm/transformers_utils/repo_utils.py @@ -63,6 +63,11 @@ def resolve_revision( requested revision that also carries the commit hash, so downstream error messages stay readable and offline loads reuse the cached `refs/` entry. + A commit hash only means something for the repo it was resolved against, so + a repo that is not `repo_id` needs its own call. Passing a revision resolved + for another repo back in is safe: the revision initially requested is + resolved again, against `repo_id` this time. + Returns: The resolved revision, or `revision` unchanged if it cannot be resolved (local path, ModelScope, or any Hub error). diff --git a/vllm/triton_utils/dispatcher.py b/vllm/triton_utils/dispatcher.py new file mode 100644 index 0000000000..021ed97214 --- /dev/null +++ b/vllm/triton_utils/dispatcher.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Registry for overriding Triton kernels with platform implementations. + +Non-CUDA platforms can replace Triton kernels defined in vLLM core +without modifying core code: + + from vllm.triton_utils.dispatcher import register_kernels + + register_kernels({ + "vllm.v1.sample.rejection_sampler.expand_kernel": my_expand_impl, + "vllm.v1.worker.mamba_utils.batch_memcpy_kernel": my_memcpy_impl, + }) +""" + +import importlib +import inspect +import sys +from collections.abc import Callable, Mapping +from typing import Any + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_registry: dict[str, tuple[Callable[..., Any], Callable[..., Any]]] = {} + + +def _kernel_arg_names(kernel: Callable[..., Any]) -> tuple[str, ...]: + """Return the argument names of a Triton kernel or its fallback.""" + names = getattr(kernel, "arg_names", None) + if names is None: + # No real Triton (TritonPlaceholder): read the plain function. + fn = getattr(kernel, "func", kernel) + names = tuple(inspect.signature(fn).parameters) + return tuple(names) + + +class KernelOverride: + """Stands in for a Triton kernel and routes launches to ``impl``. + + The JIT warmup infrastructure launches kernels as + ``kernel[grid](**kwargs)`` where ``kwargs`` are keyed by the original + kernel's argument names. This wrapper forwards those launch arguments + to the platform implementation: by keyword when the implementation's + parameter names match the kernel's, otherwise positionally in the + kernel's parameter order. + """ + + def __init__(self, kernel: Callable[..., Any], impl: Callable[..., Any]) -> None: + self._impl = impl + # Mirror the original kernel's names so launch binding and warmup + # introspection keep working against this wrapper. + self.arg_names = _kernel_arg_names(kernel) + self.constexprs = getattr(kernel, "constexprs", None) + self.func = impl + self.__name__ = getattr(impl, "__name__", "kernel_override") + self.__module__ = getattr(kernel, "__module__", "") + self._forward_by_name = ( + tuple(inspect.signature(impl).parameters) == self.arg_names + ) + + def __getitem__(self, grid: Any) -> Callable[..., Any]: + return self._launch + + def _launch(self, *args: Any, **kwargs: Any) -> Any: + if args: + return self._impl(*args, **kwargs) + if self._forward_by_name: + return self._impl(**kwargs) + unexpected = [name for name in kwargs if name not in self.arg_names] + if unexpected: + raise RuntimeError( + f"Override for {self.__name__!r} received unexpected launch " + f"arguments: {unexpected}" + ) + return self._impl(*(kwargs[name] for name in self.arg_names if name in kwargs)) + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self._impl(*args, **kwargs) + + +def _resolve_kernel(name: str) -> tuple[Any, str]: + """Return the (host object, attribute) holding the kernel `name`.""" + module_name, attr = name.rsplit(".", 1) + parts = module_name.split(".") + if parts[-1][:1].isupper(): + class_module = importlib.import_module(".".join(parts[:-1])) + return getattr(class_module, parts[-1]), attr + importlib.import_module(module_name) + module = sys.modules[module_name] + if hasattr(module, attr): + return module, attr + raise ValueError(f"Kernel {name!r} not found for override") + + +def _rebind_kernels( + overrides: list[tuple[Callable[..., Any], KernelOverride]], +) -> None: + """Point every reference to each original kernel at its wrapper. + + A kernel object can be captured in two kinds of places besides its + defining module, and both must be patched for the overrides to take + effect everywhere: + + * plain attributes of other modules (``from mod import kernel`` + copies), rebound to the wrapper; + * JIT warmup owners, whose ``kernel`` attribute is the kernel they + launch; swapped, and their cached ``_kernel_arg_names`` invalidated + so the launch binding re-derives from the wrapper. + + One pass over ``sys.modules`` handles all kernels in `overrides`. + """ + originals = {original: wrapper for original, wrapper in overrides} + + def lookup(value: Any) -> KernelOverride | None: + # Only compare by identity: this scan runs over every module + # namespace, and some attribute values (e.g. PlaceholderModule + # sentinels) trigger imports or errors on hash/eq. + for original in originals: + if value is original: + return originals[original] + return None + + for module in list(sys.modules.values()): + namespace = getattr(module, "__dict__", None) + if not namespace: + continue + for attr, value in list(namespace.items()): + wrapper = lookup(value) + if wrapper is not None: + setattr(module, attr, wrapper) + continue + # JIT warmup owners (e.g. _DecoratedTritonJitKernel) hold the + # kernel as an instance attribute; read the instance dict directly + # so module-level __getattr__ hooks are never triggered, and so + # this scan needs no dependency on the warmup machinery. + if hasattr(value, "__dict__"): + wrapper = lookup(vars(value).get("kernel")) + if wrapper is None: + continue + value.kernel = wrapper + if not isinstance(value, type): + value.__dict__.pop("_kernel_arg_names", None) + + +def register_kernels(overrides: Mapping[str, Callable[..., Any]]) -> None: + """Register platform-specific implementations for Triton kernels. + + Args: + overrides: Mapping from fully qualified kernel name to the + platform implementation. Kernel names look like + "vllm.v1.sample.rejection_sampler.expand_kernel"; for kernels + owned by a JIT warmup class, include the attribute path, e.g. + "vllm.v1.worker.block_table.ComputeSlotMappingKernel.kernel". + Implementations are invoked like the kernels themselves, with + the launch arguments in the original kernel's parameter order + (or by keyword when their parameter names match the kernel's). + + """ + resolved: list[tuple[str, Any, str, Callable[..., Any], KernelOverride]] = [] + for name, impl in overrides.items(): + if name in _registry: + logger.warning("Kernel %s is already registered. Overwriting.", name) + host, attr = _resolve_kernel(name) + original = getattr(host, attr) + wrapper = KernelOverride(original, impl) + _registry[name] = (original, impl) + resolved.append((name, host, attr, original, wrapper)) + # Rebind other holders first: once the defining attributes are swapped + # below, the scan could no longer find the original kernels. + _rebind_kernels([(original, wrapper) for *_, original, wrapper in resolved]) + for name, host, attr, _, wrapper in resolved: + setattr(host, attr, wrapper) + logger.debug("Registered kernel override %s", name) diff --git a/vllm/utils/cpu_triton_utils.py b/vllm/utils/cpu_triton_utils.py index d1696317aa..d8e4cd389d 100644 --- a/vllm/utils/cpu_triton_utils.py +++ b/vllm/utils/cpu_triton_utils.py @@ -4,19 +4,10 @@ """Contains replacement functions to fallback Triton usages in CPU backend.""" import ctypes -from collections.abc import Callable import torch -class _FuncWrapper: - def __init__(self, func: Callable) -> None: - self.func = func - - def __getitem__(self, *args, **kwargs) -> Callable: - return self.func - - # For _compute_slot_mapping_kernel in vllm/v1/worker/block_table.py def _compute_slot_mapping_kernel_impl( num_tokens: int, @@ -47,9 +38,6 @@ def _compute_slot_mapping_kernel_impl( ) -compute_slot_mapping_kernel = _FuncWrapper(_compute_slot_mapping_kernel_impl) - - def _ensure_int64(t: torch.Tensor) -> torch.Tensor: return t if t.dtype == torch.int64 else t.to(torch.int64) @@ -454,31 +442,7 @@ def _sample_recovered_tokens_kernel_impl( output_token_ids.copy_(output_i64.to(orig_dtype)) -eagle_prepare_inputs_padded_kernel = _FuncWrapper( - _eagle_prepare_inputs_padded_kernel_impl -) -eagle_prepare_next_token_padded_kernel = _FuncWrapper( - _eagle_prepare_next_token_padded_kernel_impl -) -copy_and_expand_eagle_inputs_kernel = _FuncWrapper( - _copy_and_expand_eagle_inputs_kernel_impl -) -copy_and_expand_dflash_inputs_kernel = _FuncWrapper( - _copy_and_expand_dflash_inputs_kernel_impl -) -eagle_step_slot_mapping_metadata_kernel = _FuncWrapper( - _eagle_step_slot_mapping_metadata_kernel_impl -) -rejection_greedy_sample_kernel = _FuncWrapper(_rejection_greedy_sample_kernel_impl) -rejection_random_sample_kernel = _FuncWrapper(_rejection_random_sample_kernel_impl) -expand_kernel = _FuncWrapper(_expand_kernel_impl) -sample_recovered_tokens_kernel = _FuncWrapper(_sample_recovered_tokens_kernel_impl) - - def _batch_memcpy_impl(src_ptrs, dst_ptrs, sizes, BLOCK_SIZE=None): # BLOCK_SIZE is unused; kept for signature parity with the Triton kernel. for src, dst, size in zip(src_ptrs.tolist(), dst_ptrs.tolist(), sizes.tolist()): ctypes.memmove(dst, src, size) - - -batch_memcpy_kernel = _FuncWrapper(_batch_memcpy_impl) diff --git a/vllm/utils/diffusion.py b/vllm/utils/diffusion.py new file mode 100644 index 0000000000..4980f940d8 --- /dev/null +++ b/vllm/utils/diffusion.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Request validation for discrete diffusion models.""" + +from typing import TYPE_CHECKING + +from vllm.exceptions import VLLMValidationError + +if TYPE_CHECKING: + from vllm.sampling_params import SamplingParams + + +def validate_diffusion_sampling_params( + params: "SamplingParams", + *, + canvas_length: int | None, + vocab_size: int, + async_scheduling: bool | None, +) -> None: + """Validate request options and normalize one-canvas reads at admission.""" + extra = params.extra_args + if not extra: + return + + def check(valid: bool, message: str) -> None: + if not valid: + raise VLLMValidationError(message, parameter="extra_args") + + width = extra.get("diffusion_canvas_length") + if width is not None: + check( + isinstance(width, int) + and not isinstance(width, bool) + and width > 0 + and (canvas_length is None or width <= canvas_length), + "diffusion_canvas_length must be a positive integer no larger " + "than the served canvas.", + ) + # The request's canvas: its own width, else the served one. + expected_len = width or canvas_length + + seed = extra.get("diffusion_seed_canvas") + if seed is not None: + check( + isinstance(seed, (list, tuple)) + and all(isinstance(t, int) and not isinstance(t, bool) for t in seed), + "diffusion_seed_canvas must be a list of token ids.", + ) + check( + all(0 <= t < vocab_size for t in seed), + f"diffusion_seed_canvas ids must be in [0, {vocab_size}).", + ) + check( + expected_len is None or len(seed) == expected_len, + f"diffusion_seed_canvas must hold exactly {expected_len} ids, " + f"got {len(seed)}.", + ) + + pins = extra.get("diffusion_pinned") + if pins is not None: + check( + isinstance(pins, (list, tuple)) + and all(isinstance(p, int) and not isinstance(p, bool) for p in pins), + "diffusion_pinned must be a list of canvas positions.", + ) + check( + seed is not None, + "diffusion_pinned needs a diffusion_seed_canvas to hold.", + ) + check( + all(p >= 0 and (expected_len is None or p < expected_len) for p in pins), + "diffusion_pinned positions must lie inside the canvas.", + ) + + cap = extra.get("diffusion_max_steps") + if cap is not None: + check( + isinstance(cap, int) and not isinstance(cap, bool) and cap > 0, + "diffusion_max_steps must be a positive integer.", + ) + + # The OpenAI server's vllm_xargs narrows JSON booleans to 0/1. + read_only = extra.get("diffusion_read_only") + if read_only is not None: + check( + isinstance(read_only, (bool, int)) and read_only in (0, 1), + "diffusion_read_only must be a boolean (or 0/1).", + ) + if read_only: + # One canvas is the whole output. Cap max_tokens so the scheduler + # ends the request there, and ignore EOS so an end-of-turn token + # drawn into a noise slot does not end it early. + if expected_len is not None: + params.max_tokens = min(params.max_tokens or expected_len, expected_len) + params.ignore_eos = True + + constrained = extra.get("diffusion_constrained") + if constrained is not None: + check( + isinstance(constrained, (bool, int)) and constrained in (0, 1), + "diffusion_constrained must be a boolean (or 0/1).", + ) + if constrained: + check( + bool(params.logprob_token_ids), + "diffusion_constrained needs logprob_token_ids: they are the allowed set.", + ) + + check( + bool(async_scheduling) + or width is None + or canvas_length is None + or width == canvas_length, + "A diffusion_canvas_length smaller than the served canvas " + "requires --async-scheduling.", + ) diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index cc83a62a16..ae09ed2dbe 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -4,18 +4,18 @@ from __future__ import annotations import contextlib +import logging import multiprocessing import os import signal import sys from collections.abc import Callable, Iterator from pathlib import Path -from typing import TextIO import psutil import vllm.envs as envs -from vllm.logger import init_logger +from vllm.logger import init_logger, set_vllm_process_name from vllm.platforms import current_platform from vllm.platforms.interface import in_wsl from vllm.ray.lazy_utils import is_in_ray_actor @@ -24,10 +24,6 @@ from .platform_utils import cuda_is_initialized, xpu_is_initialized logger = init_logger(__name__) -CYAN = "\033[0;36m" -RESET = "\033[0;0m" - - # Environment variable utilities @@ -63,7 +59,7 @@ def suppress_stdout(): """Suppress stdout from C libraries at the file descriptor level. Only suppresses stdout, not stderr, to preserve error messages. - Suppression is disabled when VLLM_LOGGING_LEVEL is set to DEBUG. + Suppression is disabled when vLLM debug logging is enabled. Example: with suppress_stdout(): @@ -72,7 +68,7 @@ def suppress_stdout(): """ # Don't suppress if logging level is DEBUG - if envs.VLLM_LOGGING_LEVEL == "DEBUG": + if logger.isEnabledFor(logging.DEBUG): yield return @@ -197,59 +193,14 @@ def set_process_title( setproctitle.setproctitle(f"{prefix}::{name}") -def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None: - """Add colored prefix to file output for log decoration.""" - is_tty = hasattr(file, "isatty") and file.isatty() - if ( - envs.NO_COLOR - or envs.VLLM_LOGGING_COLOR == "0" - or (envs.VLLM_LOGGING_COLOR != "1" and not is_tty) - ): - prefix = f"({worker_name} pid={pid}) " - else: - prefix = f"{CYAN}({worker_name} pid={pid}){RESET} " - # Use the original write to avoid nesting prefixes on repeated calls. - file_write = getattr(file, "_original_write", file.write) - - def write_with_prefix(s: str): - if not s: - return - if file.start_new_line: # type: ignore[attr-defined] - file_write(prefix) - idx = 0 - while (next_idx := s.find("\n", idx)) != -1: - next_idx += 1 - file_write(s[idx:next_idx]) - if next_idx == len(s): - file.start_new_line = True # type: ignore[attr-defined] - return - file_write(prefix) - idx = next_idx - file_write(s[idx:]) - file.start_new_line = False # type: ignore[attr-defined] - - file.start_new_line = True # type: ignore[attr-defined] - file._original_write = file_write # type: ignore[attr-defined] - file.write = write_with_prefix # type: ignore[method-assign] - - def decorate_logs( process_name: str | None = None, *, skip_if_decorated: bool = False ) -> None: - """Decorate stdout/stderr with process name and PID prefix.""" - # Respect VLLM_CONFIGURE_LOGGING environment variable - if not envs.VLLM_CONFIGURE_LOGGING: - return - - if skip_if_decorated and hasattr(sys.stdout, "_original_write"): - return - + """Add vLLM process metadata to subsequent log records.""" if process_name is None: process_name = get_mp_context().current_process().name - pid = os.getpid() - _add_prefix(sys.stdout, process_name, pid) - _add_prefix(sys.stderr, process_name, pid) + set_vllm_process_name(process_name, skip_if_set=skip_if_decorated) def kill_process_tree(pid: int): diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index afafa3c925..5d661634d7 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -69,7 +69,9 @@ class AttentionBackend(ABC): forward_includes_kv_cache_update: bool = True @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes( + kv_cache_spec: "KVCacheSpec | None" = None, + ) -> list[int | MultipleOf]: return [MultipleOf(1)] @staticmethod @@ -571,6 +573,26 @@ class AttentionCGSupport(Enum): """NO cudagraph support""" +def max_decode_query_len(vllm_config: "VllmConfig") -> int: + """Widest request a spec-as-decode builder treats as a decode. + + On model runner V2 this is a verification request, 1 + + num_speculative_tokens: no V2 speculator builds a wider query. The reorder + threshold and the code that mirrors the builders' decode/prefill split all + read this, so they cannot drift apart. + """ + speculative_config = getattr(vllm_config, "speculative_config", None) + if speculative_config is None or speculative_config.num_speculative_tokens is None: + return 1 + num_speculative_tokens = speculative_config.num_speculative_tokens + max_query_len = 1 + num_speculative_tokens + if speculative_config.parallel_drafting and not vllm_config.use_v2_model_runner: + # Model runner V1 only; remove with it. Its parallel drafter appends up + # to num_speculative_tokens mask slots to each verified request. + max_query_len += num_speculative_tokens + return max_query_len + + class AttentionMetadataBuilder(ABC, Generic[M]): # Does this backend/builder support CUDA Graphs for attention (default: no). # Do not access directly. Call get_cudagraph_support() instead. @@ -614,6 +636,30 @@ class AttentionMetadataBuilder(ABC, Generic[M]): """Get the cudagraph support level of this builder class.""" return cls._cudagraph_support + @classmethod + def get_varlen_cudagraph_max_query_len( + cls: type["AttentionMetadataBuilder"], + vllm_config: "VllmConfig", + kv_cache_spec: "KVCacheSpec", + ) -> int | None: + """Get the largest per-request query length L of the variable-length + decode batches a FULL cudagraph of this builder class can replay. + + The graph must replay any decode batch in which every real request has + between 1 and L query tokens and every padding request has 0. Lengths + come from the device query_start_loc; host metadata carries only the + token count and an upper bound on the per-request length. Batches with + a prefill never replay these graphs. Whether host metadata may + understate device query lengths at all is a separate backend question; + see supports_device_cpu_query_lens_mismatch(). + + Returns: + L, or None for builders reporting ALWAYS, which replay any batch, + and for builders that cannot replay variable-length batches. + + """ + return None + def _init_reorder_batch_threshold( self, reorder_batch_threshold: int | None = 1, @@ -630,14 +676,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]): speculative_config is not None and speculative_config.num_speculative_tokens is not None ): - max_num_queries_for_spec = ( - 1 - + (2 if speculative_config.parallel_drafting else 1) - * speculative_config.num_speculative_tokens - ) self.reorder_batch_threshold = max( self.reorder_batch_threshold, - max_num_queries_for_spec, + max_decode_query_len(self.vllm_config), ) if ( @@ -829,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]): def __new__(cls, *args, **kwargs): # use __new__ so that all subclasses will call this self = super().__new__(cls) - try: - from vllm.distributed.parallel_state import get_dcp_group + from vllm.config import get_current_vllm_config_or_none + from vllm.distributed.parallel_state import get_dcp_world_size_and_rank - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + # Replicated drafts run at DCP=1 inside a DCP target's process group. + config = get_current_vllm_config_or_none() + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + config is None or config.parallel_config.decode_context_parallel_size > 1 + ) try: from vllm.distributed.parallel_state import get_pcp_group @@ -926,6 +966,10 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ return False + def fused_qk_norm_mrope_kvcache_supported(self): + """Whether this implementation supports fused QKNorm+MRoPE+KVCache.""" + return False + def fused_rope_kvcache_supported(self): """Does this attention implementation support RoPE+KVCache fusion. This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops @@ -955,6 +999,26 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ raise NotImplementedError + def do_qk_norm_mrope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + rotary_dim: int, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + """Apply QK-norm and MRoPE, then write K/V to the cache.""" + raise NotImplementedError + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/attention/backends/b12x.py b/vllm/v1/attention/backends/b12x.py index d2b28eed0a..833372ab44 100644 --- a/vllm/v1/attention/backends/b12x.py +++ b/vllm/v1/attention/backends/b12x.py @@ -183,7 +183,7 @@ class B12xPagedAttentionBackend(AttentionBackend): return B12xPagedMetadataBuilder @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return list(_B12X_SUPPORTED_PAGE_SIZES) @classmethod diff --git a/vllm/v1/attention/backends/composite.py b/vllm/v1/attention/backends/composite.py index d091a29d87..0ba40e99f2 100644 --- a/vllm/v1/attention/backends/composite.py +++ b/vllm/v1/attention/backends/composite.py @@ -116,14 +116,16 @@ class MMPrefixAttentionRouting(CompositeAttentionRouting): def _intersect_kernel_block_sizes( - first: type[AttentionBackend], second: type[AttentionBackend] + first: type[AttentionBackend], + second: type[AttentionBackend], + kv_cache_spec: KVCacheSpec | None = None, ): from vllm.v1.attention.backend import MultipleOf result: list[int | MultipleOf] = [] seen: set[tuple[type, int]] = set() - for lhs in first.get_supported_kernel_block_sizes(): - for rhs in second.get_supported_kernel_block_sizes(): + for lhs in first.get_supported_kernel_block_sizes(kv_cache_spec): + for rhs in second.get_supported_kernel_block_sizes(kv_cache_spec): candidate: int | MultipleOf | None = None if isinstance(lhs, MultipleOf) and isinstance(rhs, MultipleOf): candidate = MultipleOf(math.lcm(lhs.base, rhs.base)) @@ -378,7 +380,9 @@ def create_composite_attention_backend( return CompositeAttentionMetadataBuilder @staticmethod - def get_supported_kernel_block_sizes(): + def get_supported_kernel_block_sizes( + kv_cache_spec: KVCacheSpec | None = None, + ): if kernel_block_sizes: return [ size @@ -388,7 +392,9 @@ def create_composite_attention_backend( for backend in (general_backend, causal_backend) ) ] - return _intersect_kernel_block_sizes(general_backend, causal_backend) + return _intersect_kernel_block_sizes( + general_backend, causal_backend, kv_cache_spec + ) @classmethod def supports_block_size(cls, block_size): diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index 24be5b8ec9..46b01cc551 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -13,6 +13,7 @@ from vllm import _custom_ops as ops from vllm import envs from vllm.config import ( VllmConfig, + get_current_vllm_config, get_layers_from_vllm_config, ) from vllm.logger import init_logger @@ -31,6 +32,10 @@ from vllm.v1.attention.backend import ( from vllm.v1.attention.backends.utils import ( get_num_attention_heads_from_layers, ) +from vllm.v1.attention.backends.zentorch_sdpa import ( + should_use_zentorch_sdpa, + zentorch_sdpa_attn, +) from vllm.v1.kv_cache_interface import ( AttentionSpec, CrossAttentionSpec, @@ -57,7 +62,7 @@ class CPUAttentionBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(32)] @classmethod @@ -334,6 +339,12 @@ class CPUAttentionBackendImpl(AttentionImpl): "heads in the layer" ) + vllm_config = get_current_vllm_config() + self.use_zentorch_sdpa = should_use_zentorch_sdpa( + attn_type, + vllm_config.model_config.dtype, + ) + def forward( self, layer: AttentionLayer, @@ -383,6 +394,21 @@ class CPUAttentionBackendImpl(AttentionImpl): AttentionType.ENCODER, ) if is_encoder_attention: + if self.use_zentorch_sdpa: + # Encoder attention never reads the KV cache back, so the + # zentorch path attends the packed QKV directly instead of + # staging it through the scratch encoder cache. + zentorch_sdpa_attn( + query[:num_actual_tokens], + key[:num_actual_tokens], + value[:num_actual_tokens], + output[:num_actual_tokens], + attn_metadata, + self.scale, + self.sliding_window, + self.alibi_slopes, + ) + return output # For encoder attention, kv_cache = attn_metadata.encoder_cache @@ -406,6 +432,15 @@ class CPUAttentionBackendImpl(AttentionImpl): kv_cache_dtype=self.kv_cache_dtype, ) + # The CPU kernel executes attention sinks natively in bf16. If the + # sinks tensor is anything other than bf16, cast it to fp32 so it is + # executed in full float precision (done lazily here, after weights + # are loaded, rather than at __init__ time). + if self.sinks is not None and self.sinks.dtype not in [ + torch.bfloat16, + torch.float32, + ]: + self.sinks = self.sinks.to(torch.float32) ops.cpu_attention_with_kv_cache( query=query[:num_actual_tokens], key_cache=key_cache, diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index 650fc1dea0..83348bb94a 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -303,7 +303,11 @@ def flash_attn_supports_kv_cache_dtype( kv_cache_block_size=kv_cache_block_size, supports_fa4_hd256=supports_fa4_hd256, ) - return (fa_version == 3 and current_platform.is_device_capability_family(90)) or ( + is_sm90 = current_platform.is_device_capability_family(90) + sm90_fp8_kv_supported = fa_version == 3 or ( + fa_version == 4 and head_size in (None, 512) + ) + return (sm90_fp8_kv_supported and is_sm90) or ( fa_version == 4 and current_platform.is_device_capability_family(100) ) diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index b57af3fd29..6035678394 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -60,7 +60,10 @@ from vllm.config import ( get_layers_from_vllm_config, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.platforms.interface import DeviceCapability from vllm.utils.math_utils import cdiv, round_up @@ -291,18 +294,56 @@ class FlashAttentionBackend(AttentionBackend): ] head_size_v: int | None = None - @classmethod - def _get_fa4_hd256_block_size(cls) -> int | None: - vllm_config = get_current_vllm_config_or_none() - if vllm_config is None or vllm_config.model_config is None: - return None + @staticmethod + def _get_sm90_fa4_fp8_kv_block_size( + kv_cache_spec: KVCacheSpec | None = None, + ) -> int | None: + if kv_cache_spec is None: + vllm_config = get_current_vllm_config_or_none() + if vllm_config is None or vllm_config.model_config is None: + return None + head_size = vllm_config.model_config.get_head_size() + uses_fp8_kv = vllm_config.cache_config.cache_dtype in ( + "fp8", + "fp8_e4m3", + ) + else: + if not isinstance(kv_cache_spec, AttentionSpec): + return None + head_size = kv_cache_spec.head_size + uses_fp8_kv = kv_cache_spec.kv_quant_mode == KVQuantMode.FP8_PER_TENSOR - head_size = vllm_config.model_config.get_head_size() if ( - uses_fa4_hd256_kernel(head_size, cls.head_size_v) + current_platform.is_device_capability_family(90) + and uses_fp8_kv + and head_size == 512 + and get_flash_attn_version(head_size=head_size) == 4 + ): + # The SM90 FP8-KV-dequant kernel uses a 64-token TMA tile/page. + return 64 + return None + + @classmethod + def _get_fa4_hd256_block_size( + cls, kv_cache_spec: KVCacheSpec | None = None + ) -> int | None: + if kv_cache_spec is None: + vllm_config = get_current_vllm_config_or_none() + if vllm_config is None or vllm_config.model_config is None: + return None + head_size = vllm_config.model_config.get_head_size() + head_size_v = cls.head_size_v + else: + if not isinstance(kv_cache_spec, AttentionSpec): + return None + head_size = kv_cache_spec.head_size + head_size_v = kv_cache_spec.head_size_v + + if ( + uses_fa4_hd256_kernel(head_size, head_size_v) and get_flash_attn_version( head_size=head_size, - head_size_v=cls.head_size_v, + head_size_v=head_size_v, supports_fa4_hd256=True, ) == 4 @@ -311,8 +352,12 @@ class FlashAttentionBackend(AttentionBackend): return None @classmethod - def get_supported_kernel_block_sizes(cls) -> list[int | MultipleOf]: - if block_size := cls._get_fa4_hd256_block_size(): + def get_supported_kernel_block_sizes( + cls, kv_cache_spec: KVCacheSpec | None = None + ) -> list[int | MultipleOf]: + if block_size := cls._get_sm90_fa4_fp8_kv_block_size(kv_cache_spec): + return [block_size] + if block_size := cls._get_fa4_hd256_block_size(kv_cache_spec): # Sliding-window specs select the smallest advertised size. return [block_size] return [MultipleOf(16)] @@ -321,6 +366,8 @@ class FlashAttentionBackend(AttentionBackend): @classmethod def get_preferred_block_size(cls, default_block_size: int) -> int: + if block_size := cls._get_sm90_fa4_fp8_kv_block_size(): + return max(default_block_size, block_size) if block_size := cls._get_fa4_hd256_block_size(): return max(default_block_size, block_size) if current_platform.is_xpu(): @@ -419,6 +466,13 @@ class FlashAttentionBackend(AttentionBackend): ) -> str | None: if has_sink and device_capability < DeviceCapability(9, 0): return "sink not supported on compute capability < 9.0" + if ( + use_mm_prefix + and kv_cache_dtype is not None + and is_quantized_kv_cache(kv_cache_dtype) + and device_capability == DeviceCapability(9, 0) + ): + return "SM90 FP8 KV with mm_prefix requires Triton" if ( kv_cache_dtype is not None and is_quantized_kv_cache(kv_cache_dtype) @@ -431,7 +485,10 @@ class FlashAttentionBackend(AttentionBackend): supports_fa4_hd256=True, ) ): - return "FP8 KV cache requires FA3 on SM90 or FA4 on SM100" + return ( + "FP8 KV cache requires FA3 on SM90, FA4 with head_size=512 " + "on SM90, or FA4 on SM100" + ) if ( use_mm_prefix and get_flash_attn_version( @@ -514,15 +571,16 @@ def _get_sliding_window_configs( ) -> set[tuple[int, int] | None]: """Get the set of all sliding window configs used in the model. - Only inspects FlashAttentionImpl layers. Other backends (e.g. - TurboQuant, MLA) use their own metadata builders and are skipped. + Only inspects FlashAttentionImpl layers, including implementations wrapped + by a composite backend. Other backends (e.g. TurboQuant, MLA) use their own + metadata builders and are skipped. """ sliding_window_configs: set[tuple[int, int] | None] = set() layers = get_layers_from_vllm_config(vllm_config, Attention) for layer in layers.values(): - if not isinstance(layer.impl, FlashAttentionImpl): - continue - sliding_window_configs.add(layer.impl.sliding_window) + for impl in layer.impl.get_impl_variants(): + if isinstance(impl, FlashAttentionImpl): + sliding_window_configs.add(impl.sliding_window) return sliding_window_configs @@ -671,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad num_qo_heads=self.num_heads_q, ) - try: - from vllm.distributed.parallel_state import get_dcp_group - - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) # Fused draft decode reuses the captured metadata object across draft # steps. For DCP, build-time host-side decisions such as @@ -1145,7 +1197,17 @@ class FlashAttentionImpl(AttentionImpl): "heads in the layer" ) - self.supports_quant_query_input = flash_attn_supports_quant_query_input() + # FA4's SM90 FP8-KV path consumes native FP16/BF16 Q and dequantizes + # FP8 K/V in-kernel. Other FA4 paths (notably SM100) still require Q, + # K, and V to have the same FP8 dtype. + uses_sm90_fa4_fp8_kv_dequant = ( + self.vllm_flash_attn_version == 4 + and current_platform.is_device_capability_family(90) + and self.kv_cache_dtype in ("fp8", "fp8_e4m3") + ) + self.supports_quant_query_input = flash_attn_supports_quant_query_input() and ( + not uses_sm90_fa4_fp8_kv_dequant + ) dcp_a2a = ( vllm_config is not None diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 3113c261dd..b3d40b1c7f 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -30,7 +30,10 @@ from vllm.config import ( get_current_vllm_config_or_none, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -67,6 +70,7 @@ from vllm.v1.attention.backend import ( AttentionType, CommonAttentionMetadata, MultipleOf, + max_decode_query_len, ) from vllm.v1.attention.backends.utils import ( get_dcp_local_seq_lens, @@ -423,7 +427,7 @@ class FlashInferBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # Page sizes >= 128 only run on the trtllm-gen dynamic kernel (GQA/MQA # on Blackwell); advertise them only when usable so selection never # picks a large kernel block we cannot serve. @@ -640,6 +644,7 @@ class FlashInferTrtllmAPIDecode: @dataclass class FlashInferMetadata: + kv_cache_layout: KVCacheLayout num_actual_tokens: int """Total number of tokens in the batch (excluding padding).""" @@ -746,17 +751,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self._decode_cudagraph_max_bs, self.compilation_config.max_cudagraph_capture_size, ) - try: - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - self.dcp_kv_cache_interleave_size = ( - vllm_config.parallel_config.cp_kv_cache_interleave_size - ) - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 - self.dcp_kv_cache_interleave_size = 1 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) + self.dcp_kv_cache_interleave_size = ( + vllm_config.parallel_config.cp_kv_cache_interleave_size + ) self.use_dcp = self.dcp_world_size > 1 self.dcp_a2a = ( self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a" @@ -856,15 +856,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.use_xqa = ( self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA ) - # Adaptive verification trims drafts on device, so decode query lengths - # must come from the device qo_indptr; only trtllm-gen supports that - # (the selector already rejects the other configurations). - self.use_trtllm_gen_varlen_decode = ( - speculative_config is not None - and speculative_config.enable_adaptive_verification - and self.flashinfer_trtllm_api_decode_kernel - == FlashInferDecodeKernel.TRTLLM_GEN - and not self.use_dcp + self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode( + vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp ) self._init_reorder_batch_threshold( 1, @@ -1011,6 +1004,31 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): else: return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + @override # type: ignore[misc] + @classmethod + def get_varlen_cudagraph_max_query_len( + cls: type["FlashInferMetadataBuilder"], + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> int | None: + # The uniform gate already covers DCP, head geometry, the XQA head-dim + # limit and causality, and yields None for subclasses that force NEVER. + if ( + cls.get_cudagraph_support(vllm_config, kv_cache_spec) + != AttentionCGSupport.UNIFORM_BATCH + ): + return None + # The kernel selector asserts off the trtllm-capable architectures, so + # call it only past the gate, as get_cudagraph_support does. + if not cls._uses_trtllm_gen_varlen_decode( + vllm_config, + cls._get_flashinfer_trtllm_api_decode_kernel(), + use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1, + ): + return None + # Longer requests are split off as prefills. + return max_decode_query_len(vllm_config) + def _get_workspace_buffer(self): if self._workspace_buffer is None: buffer_size = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE @@ -1039,6 +1057,24 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): def set_workspace_buffer(self, workspace_buffer: torch.Tensor): self._workspace_buffer = workspace_buffer + @staticmethod + def _uses_trtllm_gen_varlen_decode( + vllm_config: VllmConfig, + decode_kernel: FlashInferDecodeKernel | None, + use_dcp: bool, + ) -> bool: + """Whether decode reads per-request query lengths from the device + qo_indptr. Adaptive verification trims drafts on device, and only + trtllm-gen supports that (the selector already rejects the other + configurations).""" + speculative_config = vllm_config.speculative_config + return ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + and decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN + and not use_dcp + ) + @staticmethod def _get_flashinfer_trtllm_api_decode_kernel() -> FlashInferDecodeKernel: if current_platform.is_device_capability( @@ -1302,7 +1338,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): split_decodes_and_prefills( common_attn_metadata, decode_threshold=self.reorder_batch_threshold, - require_uniform=not self.use_xqa, + require_uniform=not ( + self.use_xqa or self.use_trtllm_gen_varlen_decode + ), ) ) else: @@ -1385,6 +1423,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Leave prefill/decode/cascade_wrapper empty, to be populated # case by case depending on the batch contents and backend selection. attn_metadata = FlashInferMetadata( + kv_cache_layout=self.kv_cache_layout, num_actual_tokens=num_actual_tokens, slot_mapping=common_attn_metadata.slot_mapping, q_data_type_prefill=self.q_data_type_prefill, @@ -1563,8 +1602,30 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): qo_indptr_prefill_cpu[1:] - qo_indptr_prefill_cpu[:-1] ) max_q_len_prefill = int(query_lens_prefill_cpu.max().item()) + prefill_block_tables = block_table_tensor[prefill_start:] + if ( + self.q_data_type_prefill != FP8_DTYPE + and self.cache_dtype.startswith("fp8") + ): + seq_lens_cpu_upper_bound = ( + common_attn_metadata.seq_lens_cpu_upper_bound + ) + max_prefill_seq_len = max_seq_len + if seq_lens_cpu_upper_bound is not None: + max_prefill_seq_len = int( + seq_lens_cpu_upper_bound[prefill_start:num_reqs] + .max() + .item() + ) + # Dequantization allocates one page per table entry and + # indexes rows by their logical width. + prefill_block_tables = canonicalize_singleton_dim_strides( + prefill_block_tables[ + :, : cdiv(max_prefill_seq_len, page_size) + ].contiguous() + ) attn_metadata.prefill = TRTLLMPrefill( - block_tables=block_table_tensor[prefill_start:], + block_tables=prefill_block_tables, seq_lens=prefill_seq_lens, cum_seq_lens_q=qo_indptr_prefill_gpu, cum_seq_lens_kv=paged_kv_indptr_prefill_gpu, @@ -1682,15 +1743,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): bool(causal), ) elif self.use_trtllm_gen_varlen_decode: - # CPU lens are only an upper bound, device qo_indptr is - # the truth. - decode_q_lens = ( - qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes] + # CPU placeholders may underestimate query lengths; + # use a safe upper bound. + q_len_per_req = min( + common_attn_metadata.max_query_len, self.reorder_batch_threshold ) - max_q_upper = int(decode_q_lens.max()) - if max_q_upper > 1: - q_len_per_req = max_q_upper - q_cu_seq_lens = qo_indptr[: num_decodes + 1] + # Always pass GPU offsets so a graph captured with uniform + # query lengths can replay batches with variable query lengths. + q_cu_seq_lens = qo_indptr[: num_decodes + 1] attn_metadata.decode = FlashInferTrtllmAPIDecode( kernel=self.flashinfer_trtllm_api_decode_kernel, block_tables=block_table_tensor[:num_decodes], @@ -1844,8 +1904,6 @@ class FlashInferImpl(AttentionImpl): num_heads, num_kv_heads, is_prefill=False ) vllm_config = get_current_vllm_config_or_none() - # The layout is resolved after model construction, so read it lazily. - self.cache_config = vllm_config.cache_config if vllm_config else None # Query pre-quantization needs a single dtype for the whole query tensor. # SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill, # so only enable this for SM100 trtllm-gen where both use FP8-Q. @@ -1881,11 +1939,6 @@ class FlashInferImpl(AttentionImpl): else: self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False) - @property - def kv_cache_layout(self) -> KVCacheLayout: - assert self.cache_config is not None - return self.cache_config.get_resolved_kv_cache_layout() - def fused_output_quant_supported(self, quant_key: QuantKey): if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4: logger.warning_once( @@ -2051,6 +2104,7 @@ class FlashInferImpl(AttentionImpl): # performance to make sure it does not introduce any overhead. num_actual_tokens = attn_metadata.num_actual_tokens + kv_cache_layout = attn_metadata.kv_cache_layout # FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache # as uint8 bytes, so pass FP8 caches with their logical dtype. @@ -2074,7 +2128,7 @@ class FlashInferImpl(AttentionImpl): if attn_metadata.use_cascade: # Cascade attention (rare case). assert attn_metadata.cascade_wrapper is not None - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order if self.is_kvcache_nvfp4: kv_cache_views = tuple( cache.permute(*stride_order) @@ -2094,7 +2148,7 @@ class FlashInferImpl(AttentionImpl): num_decode_tokens = attn_metadata.num_decode_tokens num_prefill_tokens = attn_metadata.num_prefill_tokens - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous # Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1 # with TP=8). PyTorch permits non-canonical strides on size-1 dims; @@ -2251,7 +2305,7 @@ class FlashInferImpl(AttentionImpl): seq_lens_prefill = attn_metadata.prefill.seq_lens # This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" assert is_strictly_contiguous(prefill_query) assert is_strictly_contiguous(workspace_buffer) assert is_strictly_contiguous(block_tables_prefill) @@ -2441,7 +2495,7 @@ class FlashInferImpl(AttentionImpl): # trtllm-gen needs HND layout on SM100. XQA is selected # separately on SM90 and does not use this SM100 layout gate. if decode_with_trtllm_gen: - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" else: assert decode_with_xqa assert is_strictly_contiguous(decode_query) @@ -2487,7 +2541,7 @@ class FlashInferImpl(AttentionImpl): window_left=self.window_left, out=output_padded[:decode_query_tokens], sinks=self.sinks, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), q_len_per_req=q_len_per_req, mask=attn_metadata.decode.mask, q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens, @@ -2554,7 +2608,7 @@ class FlashInferImpl(AttentionImpl): sinks=self.sinks, o_sf_scale=self.o_sf_scale, out=out, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), backend=attn_metadata.decode.kernel.value, q_len_per_req=q_len_per_req, max_q_len=max_q_len, diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 9d0ac08172..1b9a21b655 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -154,7 +154,7 @@ class FlexAttentionBackend(AttentionBackend): return [] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(16)] diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index c2cfd003d9..85173dc2e4 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -64,6 +64,7 @@ class GDNAttentionMetadata: non_spec_token_indx: torch.Tensor | None = None num_accepted_tokens: torch.Tensor | None = None # shape: [batch,] + uniform_spec_sequence_length: int | None = None # None for ragged batches # Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices) chunk_indices: torch.Tensor | None = None @@ -226,6 +227,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] self.vllm_config.cache_config.mamba_cache_mode, ) + uniform_spec_sequence_length = None spec_sequence_masks_cpu: torch.Tensor | None = None if not self.use_spec_decode or num_decode_draft_tokens_cpu is None: spec_sequence_masks = None @@ -289,6 +291,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] assert spec_sequence_masks_cpu is not None non_spec_sequence_masks_cpu = ~spec_sequence_masks_cpu query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + spec_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu] + if spec_query_lens_cpu.numel() > 0: + first_spec_sequence_length = int(spec_query_lens_cpu[0]) + if first_spec_sequence_length > 0 and bool( + torch.all(spec_query_lens_cpu == first_spec_sequence_length) + ): + uniform_spec_sequence_length = first_spec_sequence_length # Use CPU tensors to avoid CPU-GPU sync non_spec_query_lens_cpu = query_lens_cpu[non_spec_sequence_masks_cpu] @@ -540,6 +549,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] spec_token_indx=spec_token_indx, non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, + uniform_spec_sequence_length=uniform_spec_sequence_length, nums_dict=nums_dict, batch_ptr=batch_ptr, token_chunk_offset_ptr=token_chunk_offset_ptr, diff --git a/vllm/v1/attention/backends/hpc_attn.py b/vllm/v1/attention/backends/hpc_attn.py index 14545307fe..cbe97da45c 100644 --- a/vllm/v1/attention/backends/hpc_attn.py +++ b/vllm/v1/attention/backends/hpc_attn.py @@ -288,7 +288,7 @@ class HpcAttentionBackend(AttentionBackend): forward_includes_kv_cache_update: bool = True @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64] @staticmethod diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index 0a4584632d..fc71e3485f 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -156,7 +156,10 @@ class Mamba2AttentionMetadataBuilder( # data so it needs no D2H. `seq_lens_cpu_upper_bound` is precise # for prefill rows, which is all this slice covers. num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata( - common, common_attn_metadata + common_attn_metadata, + common.num_reqs, + common.num_prefills, + common.num_decode_tokens, ) prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any()) diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index b5f0b9b182..63f98f1786 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -70,6 +70,9 @@ class BaseMambaAttentionMetadata: # last_chunk_indices_p is a tensor of shape (batch,) that contains the # index of the last chunk for every sequence in the (prefill) batch. last_chunk_indices_p: torch.Tensor | None = None + # Number of block-aligned SSM states the prefill path writes, i.e. + # sum(block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p). + num_state_writes_p: int = 0 # The following attributes are for triton implementation of causal_conv1d nums_dict: dict | None = None @@ -348,8 +351,10 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): def _prefill_cpu_metadata( self, - common: M, common_attn_metadata: CommonAttentionMetadata, + num_reqs: int, + num_prefills: int, + num_decode_tokens: int, ) -> tuple[torch.Tensor, torch.Tensor]: """Prefill context lengths and query offsets, from CPU data only. @@ -361,11 +366,9 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): """ seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound assert seq_lens_cpu is not None - num_reqs = common.num_reqs - num_prefills = common.num_prefills query_start_loc_p_cpu = ( common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :] - - common.num_decode_tokens + - num_decode_tokens ) prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1] num_computed_tokens_p_cpu = ( @@ -385,7 +388,10 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_prefills = common.num_prefills num_computed_tokens_p_cpu, query_start_loc_p_cpu = self._prefill_cpu_metadata( - common, common_attn_metadata + common_attn_metadata, + common.num_reqs, + num_prefills, + common.num_decode_tokens, ) cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata( @@ -533,6 +539,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): block_idx_last_computed_token = None block_idx_last_scheduled_token = None block_idx_last_scheduled_token_prev_step = None + num_state_writes_p = 0 # for causal_conv1d nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None @@ -625,6 +632,23 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_reqs - num_prefills : num_reqs ] + # How many block-aligned states prefill will write: the same + # (block_idx_last - block_idx_first) as above, from CPU data so + # the mixer needs no D2H read. The -1 in both indices cancels. + block_size = self.kv_cache_spec.block_size + seq_lens_p_cpu = seq_lens_cpu[num_reqs - num_prefills : num_reqs] + num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata( + common_attn_metadata, num_reqs, num_prefills, num_decode_tokens + ) + num_state_writes_p = int( + ( + (seq_lens_p_cpu + block_size - 1) // block_size + - (num_computed_tokens_p_cpu + block_size) // block_size + ) + .clamp(min=0) + .sum() + ) + if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0: decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound @@ -728,6 +752,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): query_start_loc_d=query_start_loc_d, block_idx_last_scheduled_token=block_idx_last_scheduled_token, block_idx_first_scheduled_token_p=block_idx_first_scheduled_token_p, + num_state_writes_p=num_state_writes_p, block_idx_last_computed_token=block_idx_last_computed_token, block_idx_last_scheduled_token_prev_step=( block_idx_last_scheduled_token_prev_step diff --git a/vllm/v1/attention/backends/mla/amx_mla.py b/vllm/v1/attention/backends/mla/amx_mla.py index 38c8b68af9..16689887af 100644 --- a/vllm/v1/attention/backends/mla/amx_mla.py +++ b/vllm/v1/attention/backends/mla/amx_mla.py @@ -130,7 +130,7 @@ class AMXMLABackend(MLACommonBackend): return [576] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(32)] @classmethod diff --git a/vllm/v1/attention/backends/mla/cpu_mla.py b/vllm/v1/attention/backends/mla/cpu_mla.py index 91d3cb3339..431f6c3185 100644 --- a/vllm/v1/attention/backends/mla/cpu_mla.py +++ b/vllm/v1/attention/backends/mla/cpu_mla.py @@ -64,7 +64,7 @@ class CPUMLABackend(MLACommonBackend): return [576] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [16] @classmethod diff --git a/vllm/v1/attention/backends/mla/cutlass_mla.py b/vllm/v1/attention/backends/mla/cutlass_mla.py index 575b9d32a9..b88a4a17a8 100644 --- a/vllm/v1/attention/backends/mla/cutlass_mla.py +++ b/vllm/v1/attention/backends/mla/cutlass_mla.py @@ -46,7 +46,7 @@ class CutlassMLABackend(MLACommonBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [128] @staticmethod diff --git a/vllm/v1/attention/backends/mla/flashattn_mla.py b/vllm/v1/attention/backends/mla/flashattn_mla.py index 084a1f4ce6..b685be102e 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla.py @@ -49,7 +49,7 @@ class FlashAttnMLABackend(MLACommonBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(16)] @staticmethod @@ -194,8 +194,11 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata] query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> FlashAttnMLADecodeMetadata: + # Measured, not the promised bound: _schedule_decode asserts metadata size + # against a persistent buffer. query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] max_query_len = query_lens_cpu.max().item() diff --git a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py index 2d1e218640..0c3f0f4288 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py @@ -40,7 +40,7 @@ class FlashAttnMLASparseBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64] @staticmethod @@ -178,6 +178,11 @@ class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]): assert self.topk_indices_buffer is not None, ( "Indexer or topk_indices_buffer required for sparse MLA" ) + self.cu_seqlens_q_buffer = torch.arange( + self.topk_indices_buffer.shape[0] + 1, + dtype=torch.int32, + device=self.topk_indices_buffer.device, + ) self.supports_quant_query_input = False def forward_mqa( @@ -306,9 +311,7 @@ class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]): kv_cache if cache_is_flat else flat_kv_row_view(kv_cache, block_size)[0] ) - cu_seqlens_q = torch.arange( - 0, q_rope.shape[0] + 1, dtype=torch.int32, device=q_rope.device - ) + cu_seqlens_q = self.cu_seqlens_q_buffer[: q_rope.shape[0] + 1] v_cache = kv_rows[:, : self.kv_lora_rank].unsqueeze(1).unsqueeze(1) if self.qk_rope_head_dim == 0: # FA3's QV path requires the 64-wide Q/K specialization. For NoPE diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 61414bba49..ca78961eb5 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar import torch @@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( MLACommonBackend, + MLACommonDecodeMetadata, MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, @@ -115,9 +117,28 @@ def _get_multi_ctas_kv_counter_buffer( return _fi_multi_ctas_kv_counter -class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH - query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM +@dataclass +class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata): + flattened_block_table: torch.Tensor | None = None + flattened_seq_lens: torch.Tensor | None = None + flattened_row_req: torch.Tensor | None = None + # Row count the flattened tensors were built for (0 = not built), not a query len. + query_len: int = 0 + query_start_loc: torch.Tensor | None = None + max_query_len: int = 1 + + +@dataclass +class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]): + pass + + +class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]): + # Adaptive verification requires ALWAYS from every builder, matching upstream's + # DeepseekV4FlashMLAMetadataBuilder. The kernels tile ragged queries from the device + # query offsets (flashinfer #3238), so one k+1 graph replays any 1..k+1 mix. + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.VARLEN # Non-causal DSpark blocks are flattened to single-token rows in forward_mqa. supports_non_causal_multi_token_decode: ClassVar[bool] = True supports_non_causal_multi_token_dcp: ClassVar[bool] = True @@ -143,10 +164,31 @@ class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): layer_names, vllm_config, device, - MLACommonMetadata, + FlashInferMLAMetadata, supports_dcp_with_varlen=True, ) + def _build_decode( + self, + block_table_tensor: torch.Tensor, + seq_lens_device: torch.Tensor, + max_seq_len: int, + query_start_loc_cpu: torch.Tensor, + query_start_loc_device: torch.Tensor, + num_decode_tokens: int, + max_query_len: int, + dcp_tot_seq_lens_device: torch.Tensor | None, + ) -> FlashInferMLADecodeMetadata: + # Promised bound, not measured: a capture dummy puts one token/req, so + # measuring would bake an undersized graph. Ragged only when max_query_len>1. + return FlashInferMLADecodeMetadata( + block_table=block_table_tensor, + seq_lens=seq_lens_device, + dcp_tot_seq_lens=dcp_tot_seq_lens_device, + query_start_loc=query_start_loc_device, + max_query_len=max_query_len, + ) + class FlashInferMLABackend(MLACommonBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -159,7 +201,7 @@ class FlashInferMLABackend(MLACommonBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [32, 64] @staticmethod @@ -208,7 +250,7 @@ class FlashInferMLABackend(MLACommonBackend): return None -class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): +class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): can_return_lse_for_decode: bool = True supports_dcp: bool = True # DCP is the only path that consumes LSE. It uses monolithic CuTeDSL, @@ -276,7 +318,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], kv_c_and_k_pe_cache: torch.Tensor, - attn_metadata: MLACommonMetadata, + attn_metadata: FlashInferMLAMetadata, layer: AttentionLayer, ) -> tuple[torch.Tensor, torch.Tensor | None]: assert kv_c_and_k_pe_cache.numel() > 0 @@ -289,14 +331,32 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens + # Led by the promised bound (baked into the graph), not + # num_decode_tokens // num_decodes -- that average is a per-request length + # only for uniform batches. + multi_token_decode = ( + attn_metadata.decode.max_query_len > 1 + or attn_metadata.num_decode_tokens > attn_metadata.num_decodes + ) + cum_seq_lens_q: torch.Tensor | None = None + max_q_len: int | None = None + row_req: torch.Tensor | None = None + if not attn_metadata.causal: # FlashInfer decode has no causal flag. Flatten each non-causal # query block into independent single-token rows. - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes q = q.unsqueeze(1) - if query_len > 1: - block_table = block_table.repeat_interleave(query_len, dim=0) - seq_lens = seq_lens.repeat_interleave(query_len) + if multi_token_decode: + block_table, seq_lens, row_req = self._flattened_decode_metadata( + attn_metadata, q.shape[0] + ) + elif attn_metadata.decode.max_query_len > 1 and self.dcp_world_size == 1: + # Causal spec decode: keep q compact and let the kernel tile each + # request's length (uniform 1+k and adaptive ragged). Mirrors vllm #52157. + # DCP keeps the uniform reshape below: it needs LSE, which the kernels + # do not return on the ragged path (flashinfer #3238). + cum_seq_lens_q = attn_metadata.decode.query_start_loc + max_q_len = attn_metadata.decode.max_query_len # trtllm API requires extra dimension q_len_per_request for MTP elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0: logger.warning_once( @@ -327,10 +387,8 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): if self.dcp_world_size > 1: causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens assert causal_seqlens_kv_global is not None - if not attn_metadata.causal and query_len > 1: - causal_seqlens_kv_global = causal_seqlens_kv_global.repeat_interleave( - query_len - ) + if row_req is not None: + causal_seqlens_kv_global = causal_seqlens_kv_global[row_req] extra_kwargs.update( enable_dcp=True, cp_world=self.dcp_world_size, @@ -342,6 +400,16 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): # trtllm-gen rejects MLA head counts it can't tile (e.g. 96); # fall back to cute-dsl for those. decode_backend = _select_mla_decode_backend(runtime_num_heads) + if cum_seq_lens_q is not None: + # Neither decode backend returns LSE on the ragged path + # (flashinfer #3238); DCP, the only LSE consumer, took the uniform + # branch above, so this only guards a future caller wiring the two. + assert not return_lse, ( + "FlashInferMLA ragged decode cannot return LSE; DCP and adaptive " + "variable-length decode are mutually exclusive." + ) + extra_kwargs["cum_seq_lens_q"] = cum_seq_lens_q + extra_kwargs["max_q_len"] = max_q_len if decode_backend: extra_kwargs["backend"] = decode_backend elif kv_c_and_k_pe_cache.shape[-2] in (32, 64): @@ -386,3 +454,37 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): o = o.view(-1, o.shape[-2], o.shape[-1]) return o, lse + + def _flattened_decode_metadata( + self, + attn_metadata: FlashInferMLAMetadata, + num_rows: int, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Expand per-request decode tensors to one row per query token. + + Cached across the layers of a group, which all see the same batch. + """ + decode = attn_metadata.decode + assert decode is not None + if decode.query_len != num_rows: + cu = decode.query_start_loc + assert cu is not None + # searchsorted on the device offsets, not repeat_interleave(uniform_len): + # the latter only lines up for uniform batches and hands ragged rows + # another request's KV (silent garbage). + rows = torch.arange(num_rows, device=cu.device, dtype=cu.dtype) + row_req = torch.searchsorted(cu[1:], rows, right=True).clamp_( + max=decode.block_table.shape[0] - 1 + ) + decode.flattened_row_req = row_req + decode.flattened_block_table = decode.block_table[row_req] + decode.flattened_seq_lens = decode.seq_lens[row_req] + decode.query_len = num_rows + assert decode.flattened_block_table is not None + assert decode.flattened_seq_lens is not None + assert decode.flattened_row_req is not None + return ( + decode.flattened_block_table, + decode.flattened_seq_lens, + decode.flattened_row_req, + ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 142a63ec72..5af50ef1bd 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -38,7 +38,6 @@ from vllm.v1.kv_cache_interface import AttentionSpec if TYPE_CHECKING: from vllm.model_executor.models.deepseek_v2 import Indexer - from vllm.v1.attention.backend import CommonAttentionMetadata logger = init_logger(__name__) @@ -83,7 +82,7 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [32, 64] @staticmethod @@ -172,7 +171,7 @@ class FlashInferMLASparseSM120Backend(_FlashInferMLASparseBackendBase): return "FLASHINFER_MLA_SPARSE_SM120" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64, 256] @staticmethod @@ -301,12 +300,6 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde ), ) - def _build_req_id_per_token( - self, - common_attn_metadata: "CommonAttentionMetadata", - ) -> torch.Tensor: - return common_attn_metadata.token_to_req_indices(self.req_id_per_token_buffer) - # Global workspace buffer (lazily initialized) _fi_sparse_workspace: torch.Tensor | None = None diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py index 830bc52ab9..9dc899c903 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py @@ -43,6 +43,7 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import ( SparseMLACommonImpl, ) from vllm.platforms.interface import DeviceCapability +from vllm.triton_utils import tl, triton from vllm.utils.flashinfer import has_flashinfer_sm90_nope_mla from vllm.v1.attention.backend import ( AttentionBackend, @@ -64,6 +65,23 @@ _FP8_KV_DTYPES = ("fp8", "fp8_e4m3") _WORKSPACE_BYTES = 128 * 1024 * 1024 +@triton.jit +def _pack_topk_indices( + slots, + indptr, + indices, + WIDTH: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + start = tl.load(indptr + row) + end = tl.load(indptr + row + 1) + mask = (cols < end - start) & (cols < WIDTH) + values = tl.load(slots + row * WIDTH + cols, mask=mask, other=0) + tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask) + + class FlashInferMLASparseSM90Backend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ @@ -74,7 +92,7 @@ class FlashInferMLASparseSM90Backend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(64)] @staticmethod @@ -192,10 +210,11 @@ class _SM90State: self.kv_len_arr = torch.full( (max_tokens,), topk_width, dtype=torch.int32, device=device ) + self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32, device=device) self.wrapper = BatchMLAPagedAttentionWrapper( self.workspace, qo_indptr=torch.zeros(max_tokens + 1, dtype=torch.int32, device=device), - kv_indptr=torch.zeros(max_tokens + 1, dtype=torch.int32, device=device), + kv_indptr=self.kv_indptr, kv_indices=self.kv_indices, kv_len_arr=self.kv_len_arr, use_cuda_graph=True, @@ -234,12 +253,13 @@ class _SM90State: # fixed (max_tokens+1)-sized buffers with exact-size copy_, so the # indptr must always be full-size. Rows past num_tokens are padded # empty (qo_indptr flat at num_tokens) — zero-query rows read no q - # and schedule no work. Padded rows keep the full width lens; the - # value is never dereferenced. + # and schedule no work. FlashInfer 0.7 requires packed CSR offsets + # matching the exact lengths, including zero-length padded rows. torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu) - torch.mul(self._qo_cpu, self.topk_width, out=self._kv_cpu) - self._lens_cpu.fill_(self.topk_width) + self._lens_cpu.zero_() self._lens_cpu[:num_tokens] = kv_lens.to(torch.int32) + self._kv_cpu[0] = 0 + torch.cumsum(self._lens_cpu, dim=0, out=self._kv_cpu[1:]) self.wrapper.plan( self._qo_cpu, self._kv_cpu, @@ -255,6 +275,12 @@ class _SM90State: kv_data_type=self.kv_dtype, ) + def pack_indices(self, topk_slots: torch.Tensor) -> None: + num_tokens, width = topk_slots.shape + _pack_topk_indices[(num_tokens, triton.cdiv(width, 256))]( + topk_slots, self.kv_indptr, self.kv_indices, width, 256 + ) + @dataclass class FlashInferMLASparseSM90Metadata(FlashInferMLASparseMetadata): @@ -304,7 +330,7 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): hf_config = vllm_config.model_config.hf_text_config assert hf_config.index_topk is not None self._index_topk = int(hf_config.index_topk) - self._index_kpool = int(kv_cache_spec.tokens_per_state) + self._index_kpool = int(getattr(hf_config, "index_kpool", 1) or 1) def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]: """Exact per-row KV lengths, host-side (the flashinfer wrapper bakes @@ -362,8 +388,7 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): + 1 ) topk = self._index_topk - kpool = max(self._index_kpool, 1) - lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool) + lens = torch.where(ctx <= topk, ctx, topk + ctx % self._index_kpool) return num_rows, lens.to(torch.int32) def build( @@ -462,12 +487,8 @@ class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Met ) state = attn_metadata.state assert state is not None - # Refresh top-k rows in graph and clamp masked tails to a valid slot; - # per-row lengths are already baked into the host-side plan. - width = topk_slots.shape[1] - state.kv_indices[: num_tokens * width].copy_( - topk_slots.reshape(-1).clamp_(min=0).to(torch.int32) - ) + # Pack valid prefixes at the offsets refreshed by plan() before replay. + state.pack_indices(topk_slots) flat = ( kv_c_and_k_pe_cache.view(torch.float8_e4m3fn) diff --git a/vllm/v1/attention/backends/mla/flashmla.py b/vllm/v1/attention/backends/mla/flashmla.py index 8ebfbbba79..c8cd31d564 100644 --- a/vllm/v1/attention/backends/mla/flashmla.py +++ b/vllm/v1/attention/backends/mla/flashmla.py @@ -55,7 +55,7 @@ class FlashMLABackend(MLACommonBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64] @staticmethod @@ -158,6 +158,7 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> FlashMLADecodeMetadata: query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 2e71f0a35c..014358f530 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -139,7 +139,7 @@ class FlashMLASparseBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64] @staticmethod diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 1b8455a232..a3c6df6fe6 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -12,6 +12,7 @@ from vllm import _custom_ops as ops from vllm.config import VllmConfig from vllm.forward_context import in_piecewise_cudagraph from vllm.utils.torch_utils import current_stream +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, ) @@ -496,17 +497,8 @@ class SparseMLAIndexGroupBuilder: def get_sparse_mla_index_group_max_rows(vllm_config: VllmConfig) -> int: - max_query_len = 1 - speculative_config = vllm_config.speculative_config - if ( - speculative_config is not None - and speculative_config.num_speculative_tokens is not None - ): - max_query_len += speculative_config.num_speculative_tokens * ( - 2 if speculative_config.parallel_drafting else 1 - ) scheduler_config = vllm_config.scheduler_config return min( scheduler_config.max_num_batched_tokens, - scheduler_config.max_num_seqs * max_query_len, + scheduler_config.max_num_seqs * max_decode_query_len(vllm_config), ) diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 1ede0442bc..e7eb67b61f 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -200,7 +200,7 @@ class DeepseekV32IndexerBackend(AttentionBackend): return "DEEPSEEK_V32_INDEXER" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [1, MultipleOf(16)] if current_platform.is_rocm() else [64] @classmethod @@ -228,7 +228,7 @@ class KpoolTailBackend(DeepseekV32IndexerBackend): return [] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(1)] @staticmethod @@ -257,7 +257,7 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend): return (KVCacheLayout.BLHNC, KVCacheLayout.BLNHC) @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # Block sizes count uncompressed tokens: C4 indexer pages hold 64 rows. return [256] @@ -274,7 +274,7 @@ class DeepseekV41IndexerBackend(DeepseekV4IndexerBackend): return "DEEPSEEK_V41_INDEXER" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [64 if current_platform.is_device_capability_family(90) else 128] diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py index 1cf1761a77..f7ec31e153 100644 --- a/vllm/v1/attention/backends/mla/prefill/registry.py +++ b/vllm/v1/attention/backends/mla/prefill/registry.py @@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): "AiterFlashAttnPrefillBackend" ) CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend" + ZEN_CPU = ( + "vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa." + "ZenCPUSDPAMLAPrefillBackend" + ) # Placeholder for third-party/custom backends - must be registered before use # set to None to avoid alias with other backend, whose value is an empty string CUSTOM = None diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py index a2b74e4327..0cb29f51b4 100644 --- a/vllm/v1/attention/backends/mla/prefill/selector.py +++ b/vllm/v1/attention/backends/mla/prefill/selector.py @@ -110,8 +110,18 @@ def get_mla_prefill_backend( from vllm.platforms import current_platform if current_platform.is_cpu(): - logger.info_once("Using CPU SDPA MLA prefill backend.") - return MLAPrefillBackendEnum.CPU.get_class() + # CPUs have no compute capability, so the capability-driven priority + # path below does not apply. Prefer an accelerator-specific CPU backend + # when its kernels are present, else the generic SDPA one. + for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU): + try: + cpu_backend_cls = backend_enum.get_class() + except ImportError: + continue + if cpu_backend_cls.is_available(): + logger.info_once("Using %s MLA prefill backend.", backend_enum.name) + return cpu_backend_cls + raise ValueError("No valid CPU MLA prefill backend found.") device_capability = current_platform.get_device_capability() if device_capability is None: diff --git a/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py new file mode 100644 index 0000000000..88c6ce98a0 --- /dev/null +++ b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs. + +Subclasses the generic CPU SDPA backend and replaces the per-request attention +with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both +matmuls of the base implementation into a single call. + +``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need +one (context chunks, prefix/suffix merge) are delegated to the base backend. + +Availability is reported through :meth:`is_available`, so the prefill selector +gates this backend once at selection time rather than per attention call. +""" + +import torch + +from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op +from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend + + +class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend): + """MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``.""" + + @staticmethod + def get_name() -> str: + return "ZEN_CPU_SDPA_MLA" + + @classmethod + def is_available(cls) -> bool: + return has_zentorch_op(["zentorch_sdpa"]) + + @staticmethod + def _sdpa_layout( + q_r: torch.Tensor, + k_r: torch.Tensor, + v_r: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + """Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout. + + MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not + accept, so V is padded up to the query head dim. The original V head + dim is returned so the caller can slice the padding off afterwards. + """ + h_q = q_r.shape[-1] + h_v = v_r.shape[-1] + pad = h_q - h_v + + qs = q_r.transpose(0, 1).unsqueeze(0).contiguous() + ks = k_r.transpose(0, 1).unsqueeze(0).contiguous() + vs = v_r.transpose(0, 1).unsqueeze(0).contiguous() + if pad > 0: + vs = torch.nn.functional.pad(vs, (0, pad)) + return qs, ks, vs, h_v + + def _ragged_sdpa( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_cu_seq_lens: torch.Tensor, + kv_cu_seq_lens: torch.Tensor, + causal: bool, + return_softmax_lse: bool, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if return_softmax_lse: + # zentorch_sdpa's second return is not a usable LSE, so fall back + # to the base fp32 implementation whenever one is required. + return super()._ragged_sdpa( + q, + k, + v, + q_cu_seq_lens, + kv_cu_seq_lens, + causal, + return_softmax_lse, + ) + + num_reqs = q_cu_seq_lens.numel() - 1 + assert num_reqs == kv_cu_seq_lens.numel() - 1 + + out_chunks: list[torch.Tensor] = [] + for r in range(num_reqs): + q_start = int(q_cu_seq_lens[r]) + q_end = int(q_cu_seq_lens[r + 1]) + kv_start = int(kv_cu_seq_lens[r]) + kv_end = int(kv_cu_seq_lens[r + 1]) + + qs, ks, vs, h_v = self._sdpa_layout( + q[q_start:q_end], + k[kv_start:kv_end], + v[kv_start:kv_end], + ) + + attn, _ = torch.ops.zentorch.zentorch_sdpa( + qs, + ks, + vs, + 0.0, + causal, + attn_mask=None, + scale=self.scale, + ) + out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype)) + + return torch.cat(out_chunks, dim=0) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 75173e59a3..1ab4df0664 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -178,6 +178,30 @@ def _gluon_mla_decode_supported() -> bool: return on_gfx950() +# Past 2 GiB, mla_gluon swaps its masked buffer_load for an unmasked +# global_load that reads outside the cache and aborts the process. Mirrors the +# kernel's own within_2gb bound, so the fallback engages exactly where it does. +_GLUON_MAX_KV_CACHE_BYTES = 1 << 31 + + +def _gluon_kv_cache_in_bounds(kv_cache_bytes: int | None) -> bool: + """Whether this layer's KV cache keeps Gluon on its bounds-checked path. + + The cache is one flat tensor per layer, so the bound disqualifies the layer + outright; no batch shape brings an oversized cache back in range. ``None`` + is the not-yet-sized case during profiling, before any kernel runs. + """ + if kv_cache_bytes is None or kv_cache_bytes <= _GLUON_MAX_KV_CACHE_BYTES: + return True + logger.warning_once( + "KV cache is %.1f GiB per layer, past the 2 GiB where the Gluon MLA " + "kernel drops its KV bounds mask; using the padded ASM decode instead. " + "Lower --gpu-memory-utilization or --max-model-len to get Gluon back.", + kv_cache_bytes / (1 << 30), + ) + return False + + @functools.lru_cache(maxsize=1) def _get_segmented_mla_decode(): """Load AITER's segmented MLA decode with unreduced partial output.""" @@ -258,7 +282,7 @@ class AiterMLABackend(MLACommonBackend): return [] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # The aiter MLA decode kernel always operates with page_size=1 # internally (the wrapper flattens kv_buffer via .view(-1, 1, 1, H)). # We support any kernel_block_size by expanding block-level indices @@ -564,6 +588,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # _build_decode needs the cache dtype to pick the decode kernel; keep # the normalized string instead of dropping it at the end of __init__. self._kv_cache_dtype_str = kv_cache_dtype_str + # Sized per layer, matching the flat KV tensor the kernel is handed. + num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks + self._kv_cache_bytes = ( + None + if num_gpu_blocks is None + else num_gpu_blocks * kv_cache_spec.page_size_bytes + ) # MLAAttention quantizes decode Q to FP8 before calling this backend # whenever the KV cache is FP8 and supports_quant_query_input is true. q_dtype = ( @@ -772,9 +803,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device=device, ) + from vllm.platforms import current_platform from vllm.v1.worker.workspace import current_workspace_manager - max_num_partial_tiles = reduce_partial_map_size + # AITER metadata sizing assumes all requests can carry max_prefill_qlen tokens, + # which is a loose worst case in the number of QO tiles. Rather, the sum of + # qlens is bounded by max_num_batched_tokens. Manually compute the number of + # qo tiles to avoid OOM at startup. + # TODO: AITER should give us this budget constrained value + qo_tile_cnt = ( + cdiv(max_num_batched_tokens, _FP8_PREFILL_TILE_Q) + max_num_reqs - 1 + ) + max_num_partial_tiles = qo_tile_cnt + current_platform.num_compute_units() reservations: list[tuple[tuple[int, ...], torch.dtype]] = [ ( (max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k, v_head_dim), @@ -1007,6 +1047,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> AiterMLADecodeMetadata: causal = self._decode_causal @@ -1055,6 +1096,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._decode_num_heads, int(max_qo_len), self._kv_cache_dtype_str, + self._kv_cache_bytes, ) use_gluon_verify = AiterMLAHelper.use_gluon_verify( self._decode_num_heads, @@ -1062,6 +1104,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._kv_cache_dtype_str, self.dcp_world_size, causal, + self._kv_cache_bytes, ) use_segmented_dcp_verify = ( self._supports_segmented_dcp_verify and max_qo_len > 1 @@ -1471,7 +1514,12 @@ class AiterMLAHelper: return AiterMLAHelper._get_mla_unpadded_heads(num_heads, lse) @staticmethod - def use_gluon_decode(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool: + def use_gluon_decode( + num_heads: int, + max_qo_len: int, + kv_cache_dtype: str, + kv_cache_bytes: int | None = None, + ) -> bool: # Small-head (<16) single-token decode takes either the Gluon kernel or # the padded asm persistent decode, selected by # VLLM_ROCM_AITER_MLA_ASM_PADDING and the arch (Gluon is gfx950 only). @@ -1487,10 +1535,12 @@ class AiterMLAHelper: mode = _aiter_mla_small_head_mode() if mode == "asm": return False - gluon_supported = _gluon_mla_decode_supported() - if mode == "gluon": - return gluon_supported - return m % num_heads == 0 and gluon_supported + if not _gluon_mla_decode_supported(): + return False + if mode != "gluon" and m % num_heads != 0: + return False + # Last, so the size warning only fires where Gluon would have run. + return _gluon_kv_cache_in_bounds(kv_cache_bytes) @staticmethod def use_gluon_verify( @@ -1499,6 +1549,7 @@ class AiterMLAHelper: kv_cache_dtype: str, dcp_world_size: int = 1, causal: bool = True, + kv_cache_bytes: int | None = None, ) -> bool: """Whether a small-head multi-token verify uses native Gluon MTP. @@ -1526,8 +1577,10 @@ class AiterMLAHelper: return False if not _gluon_mla_decode_supported(): return False - # Same arch and mode gating as use_gluon_decode. - return _aiter_mla_small_head_mode() != "asm" + # Same arch, mode and cache-size gating as use_gluon_decode. + if _aiter_mla_small_head_mode() == "asm": + return False + return _gluon_kv_cache_in_bounds(kv_cache_bytes) @staticmethod def dcp_local_verify_row_lens( diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 69b6e342cd..f6258759d0 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -276,48 +276,6 @@ def generate_sparse_seqlen_triton( return out -@triton.jit -def fetch_id_to_ragged_kernel( - in_tensor_ptr, # [num_seq, topk] - cumsum_ptr, # [num_seq + 1] - out_tensor_ptr, # [max_num_seq * topk] - in_tensor_ptr_stride, - TOPK: tl.constexpr, - TOKEN_NUM: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - seq_id = tl.program_id(0) - block_id = tl.program_id(1) - offset = tl.arange(0, BLOCK_SIZE) - token_start = tl.load(cumsum_ptr + seq_id) - token_end = tl.load(cumsum_ptr + seq_id + 1) - token_num = token_end - token_start - row_offset = block_id * BLOCK_SIZE - if row_offset >= token_num: - return - in_tensor_offset = seq_id * in_tensor_ptr_stride + row_offset + offset - in_tensor_mask = (row_offset + offset) < TOPK - in_tensor_val = tl.load(in_tensor_ptr + in_tensor_offset, mask=in_tensor_mask) - out_tensor_offset = token_start + row_offset + offset - out_tensor_mask = (out_tensor_offset < token_end) & in_tensor_mask - tl.store(out_tensor_ptr + out_tensor_offset, in_tensor_val, mask=out_tensor_mask) - - -def fetch_id_to_ragged_triton( - in_tensor: torch.Tensor, cumsum: torch.Tensor, out_tensor: torch.Tensor, topk -): - num_tokens = in_tensor.size(0) - block_size = 64 - num_block_per_row = triton.cdiv(topk, block_size) - grid = ( - num_tokens, - num_block_per_row, - ) - fetch_id_to_ragged_kernel[grid]( - in_tensor, cumsum, out_tensor, in_tensor.stride(0), topk, num_tokens, block_size - ) - - class ROCMAiterMLASparseBackend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ @@ -329,7 +287,7 @@ class ROCMAiterMLASparseBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [1, MultipleOf(16)] @staticmethod diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 7b07dae71c..b0ef9e65a3 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -26,6 +26,7 @@ from vllm.v1.attention.backend import ( AttentionMetadataBuilder, CommonAttentionMetadata, MultipleOf, + max_decode_query_len, ) from vllm.v1.attention.backends.mla.compressor_utils import ( get_dspark_swa_index_width, @@ -148,7 +149,7 @@ class DeepseekSparseSWABackend(AttentionBackend): return "DEEPSEEK_SPARSE_SWA" @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(32)] @classmethod @@ -456,14 +457,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): self.num_speculative_tokens = ( spec_config.num_speculative_tokens if spec_config else 0 ) - # Decode can have query_len up to - # 1 + (2 if parallel drafting else 1) * num_speculative_tokens. # sparse_swa has no MQA-vs-dense-MHA routing, so multi-token queries take - # the prefill path and the decode/prefill split stays at that width. - spec_mult = ( - 2 if (spec_config is not None and spec_config.parallel_drafting) else 1 - ) - self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens + # the prefill path and the decode/prefill split stays at the widest + # decode. + self.decode_threshold = max_decode_query_len(self.vllm_config) self.reorder_batch_threshold = None hf_config = self.vllm_config.model_config.hf_config diff --git a/vllm/v1/attention/backends/mla/tokenspeed_mla.py b/vllm/v1/attention/backends/mla/tokenspeed_mla.py index 20cddfa4ca..b3d982eac3 100644 --- a/vllm/v1/attention/backends/mla/tokenspeed_mla.py +++ b/vllm/v1/attention/backends/mla/tokenspeed_mla.py @@ -87,7 +87,7 @@ class TokenspeedMLABackend(MLACommonBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [32, 64] @staticmethod diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index 3deb996a5b..950d798f22 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -107,7 +107,7 @@ class TritonMLABackend(MLACommonBackend): return [] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(16)] @classmethod diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 4ce8b23d31..3df3b26bba 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -852,7 +852,7 @@ class AiterFlashAttentionBackend(AttentionBackend): return attn_type in (AttentionType.DECODER,) @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: if not rocm_aiter_ops.is_shuffle_kv_cache_enabled(): return [16, 32] # Only gluon serves 128-token pages; the pa_fwd_asm/ll4mi decode diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index 9e3267c526..6aa456e22c 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -7,7 +7,7 @@ from typing import ClassVar import torch from vllm import _custom_ops as ops -from vllm._aiter_ops import rocm_aiter_ops +from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops from vllm.config import get_current_vllm_config_or_none from vllm.config.cache import CacheDType from vllm.logger import init_logger @@ -64,7 +64,7 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(16)] @classmethod @@ -359,6 +359,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): def fused_qk_norm_rope_kvcache_supported(self): return rocm_aiter_ops.is_enabled() + def fused_qk_norm_mrope_kvcache_supported(self) -> bool: + return IS_AITER_FOUND + def do_qk_norm_rope_kvcache_update( self, layer: AttentionLayer, @@ -397,6 +400,47 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): use_shuffle_layout=False, ) + def do_qk_norm_mrope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + rotary_dim: int, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + key_cache, value_cache = self._split_kv_cache(kv_cache) + rocm_aiter_ops.do_qk_norm_mrope_kvcache_update( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=self.num_heads, + num_heads_k=self.num_kv_heads, + head_dim=self.head_size, + is_neox=is_neox, + mrope_section=list(mrope_section), + is_interleaved=is_interleaved, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + key_cache=key_cache, + value_cache=value_cache, + slot_mapping=layer_slot_mapping, + k_scale=layer._k_scale_cpu, + v_scale=layer._v_scale_cpu, + kv_cache_dtype=self.kv_cache_dtype, + rotary_dim=rotary_dim, + ) + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index 2671249fb2..76382aa75e 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -176,7 +176,7 @@ class RocmAttentionBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: # ROCM paged attention native C++ kernel only supports block sizes 16 and 32 # due to shared memory (LDS) constraints on AMD GPUs. # See csrc/rocm/attention.cu CALL_CUSTOM_LAUNCHER_BLK macro. diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 1b290c0ae9..8a2fab4cc7 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -304,7 +304,7 @@ class TritonAttentionBackend(AttentionBackend): ] @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [MultipleOf(16)] @classmethod diff --git a/vllm/v1/attention/backends/triton_flash_attn.py b/vllm/v1/attention/backends/triton_flash_attn.py index 0853b291cd..152a92048e 100644 --- a/vllm/v1/attention/backends/triton_flash_attn.py +++ b/vllm/v1/attention/backends/triton_flash_attn.py @@ -15,7 +15,7 @@ from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend -class _FA4Backend(FlashAttentionBackend): +class _FABackend(FlashAttentionBackend): @classmethod def supports_combination( cls, @@ -42,22 +42,20 @@ class _FA4Backend(FlashAttentionBackend): ) if reason is not None: return reason - if ( - get_flash_attn_version( - head_size=head_size, - has_sinks=has_sink, - kv_cache_block_size=block_size, - supports_fa4_hd256=True, - ) - != 4 - ): - return "causal route requires FlashAttention v4" + fa_version = get_flash_attn_version( + head_size=head_size, + has_sinks=has_sink, + kv_cache_block_size=block_size, + supports_fa4_hd256=True, + ) + if fa_version not in (3, 4): + return "causal route requires FlashAttention v3 or v4" return None TritonFlashAttentionBackend = create_composite_attention_backend( TritonAttentionBackend, - _FA4Backend, + _FABackend, name="TritonFlashAttentionBackend", backend_name="TRITON_FLASH_ATTN", module=__name__, diff --git a/vllm/v1/attention/backends/turboquant_attn.py b/vllm/v1/attention/backends/turboquant_attn.py index 8542e6164f..056e42cf62 100644 --- a/vllm/v1/attention/backends/turboquant_attn.py +++ b/vllm/v1/attention/backends/turboquant_attn.py @@ -161,7 +161,7 @@ class TurboQuantAttentionBackend(AttentionBackend): return (KVCacheLayout.LBNHC,) @staticmethod - def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: return [16, 32, 64, 128] @classmethod diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 3413c5ad00..c97483912d 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor -from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + KVCacheLayout, + KVCacheSpec, + MambaSpec, +) if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput @@ -286,9 +291,10 @@ def resolve_kv_cache_layout( # specs can re-interpret HNC with different sizes as long as the total number of # bytes is the same. If not block-compact, each spec must agree on HNC to alias # the same page (this aliasing is done by the Hybrid Memory Allocator, HMA). + kv_cache_specs = tuple(kv_cache_specs or ()) hnc_shapes = { (spec.num_heads, spec.num_states, spec.page_size_bytes) - for spec in kv_cache_specs or () + for spec in kv_cache_specs } if len(hnc_shapes) > 1: candidates = [m for m in candidates if m.is_block_compact] @@ -298,6 +304,20 @@ def resolve_kv_cache_layout( f"none is in every supported set: {supported_layouts}." ) + dcp_sharding = { + spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec) + } + page_sizes = {spec.page_size_bytes for spec in kv_cache_specs} + if len(dcp_sharding) > 1 and len(page_sizes) > 1: + # Sharded target and replicated draft caches need independent groups. + # Block-outer layouts can pack those groups without equalizing pages. + candidates = [layout for layout in candidates if layout.is_block_outermost] + if not candidates: + raise ValueError( + "DCP with a replicated draft and mixed KV page sizes requires " + f"a block-outer KV cache layout; supported sets: {supported_layouts}." + ) + if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None: layout = _layout_from_name(requested) if layout not in candidates: @@ -828,7 +848,7 @@ def split_decodes_and_prefills( (query_lens == query_lens[0]) | (query_lens == 0) ): return num_reqs, 0, num_tokens, 0 # all decodes - is_prefill = query_lens != query_lens[0] + is_prefill = (query_lens != query_lens[0]) & (query_lens != 0) else: is_prefill = query_lens > decode_threshold @@ -1096,6 +1116,7 @@ def get_dcp_local_seq_lens( Only consider dcp now, we can extend the case of cp based on this. """ seq_lens_i32 = seq_lens.to(torch.int32) + rank_offsets: int | torch.Tensor if dcp_rank is None: rank_offsets = torch.arange( dcp_size, @@ -1113,7 +1134,8 @@ def get_dcp_local_seq_lens( rank_offsets = dcp_rank seq_lens_tiled = seq_lens_i32 else: - rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device) + # Use the Python scalar directly to avoid a synchronizing H2D copy. + rank_offsets = dcp_rank seq_lens_tiled = seq_lens_i32 base = ( seq_lens_tiled diff --git a/vllm/v1/attention/backends/zentorch_sdpa.py b/vllm/v1/attention/backends/zentorch_sdpa.py new file mode 100644 index 0000000000..cafb524b0e --- /dev/null +++ b/vllm/v1/attention/backends/zentorch_sdpa.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Encoder attention through the zentorch SDPA kernel on Zen CPUs.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op +from vllm.v1.attention.backend import AttentionType + +if TYPE_CHECKING: + from vllm.v1.attention.backends.cpu_attn import CPUAttentionMetadata + +__all__ = ["should_use_zentorch_sdpa", "zentorch_sdpa_attn"] + + +def should_use_zentorch_sdpa( + attn_type: str, + dtype: torch.dtype, +) -> bool: + """True when encoder attention should dispatch to zentorch_sdpa. + + Full, sliding-window, and ALiBi encoder layers all dispatch. A window is + a single-head [1, 1, S, S] additive mask; ALiBi is a per-query-head + [1, H, S, S] bias. The zentorch_sdpa_attn op builds both. + + dtype must match the op's ISA gate (bf16: AVX512-BF16, fp32: AVX512); + fp16 needs AVX512-FP16, which torch exposes no query for, so it stays + native. + """ + if attn_type not in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER): + return False + if not has_zentorch_op(["zentorch_sdpa_attn"]): + return False + if dtype == torch.bfloat16: + return torch.cpu._is_avx512_bf16_supported() + if dtype == torch.float32: + return torch.cpu._is_avx512_supported() + return False + + +def zentorch_sdpa_attn( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + attn_metadata: CPUAttentionMetadata, + scale: float, + sliding_window: int = -1, + alibi_slopes: torch.Tensor | None = None, +) -> torch.Tensor: + """Run encoder / encoder-only attention with zentorch_sdpa. + + Encoder attention reads no KV cache, so the packed + [num_tokens, num_heads, head_size] query/key/value are attended in place + and written straight into `output`. + + Args: + query: Query of shape [num_tokens, num_heads, head_size]. + key: Key of shape [num_tokens, num_kv_heads, head_size]. + value: Value of shape [num_tokens, num_kv_heads, head_size]. + output: Pre-allocated output, same shape as `query`. + attn_metadata: Metadata carrying the per-sequence token offsets and + whether attention is causal. + scale: Softmax scale. + sliding_window: Encoder window size, or -1 for full bidirectional + attention. Handed to the op, which masks token i to + [i-(W-1), i+(W-1)]. + alibi_slopes: Per-query-head ALiBi slopes, or None. Handed to the op, + which applies them as an additive per-head bias. + + Returns: + `output`, filled in place. + + """ + # zentorch_sdpa_attn builds the ALiBi mask in query.dtype. A bf16 per-head + # mask is wrong when the query head count is not a power of two, so run + # that path in fp32 and write bf16 back. + if alibi_slopes is not None and query.dtype == torch.bfloat16: + out32 = torch.empty_like(query, dtype=torch.float32) + zentorch_sdpa_attn( + query.float(), + key.float(), + value.float(), + out32, + attn_metadata, + scale, + sliding_window, + alibi_slopes, + ) + output.copy_(out32.to(dtype=output.dtype)) + return output + + # The op counts window tokens on each side of the diagonal; vLLM's window + # includes the diagonal itself. + window = -1 if sliding_window in (None, -1) else sliding_window - 1 + + # zentorch_sdpa takes no enable_gqa argument but derives the KV head count + # from the key tensor and maps each query head onto its KV head, so GQA/MQA + # needs no expansion here. + query = query.movedim(0, query.dim() - 2) + key = key.movedim(0, key.dim() - 2) + value = value.movedim(0, value.dim() - 2) + + start_loc = attn_metadata.query_start_loc.numpy() + seq_lens = start_loc[1:] - start_loc[:-1] + + # Encoder sequences are packed along the token dimension. When they all + # have the same length, recover a dense BHSD batch and invoke zentorch_sdpa + # once for the whole scheduler batch instead of once per sequence. + if len(seq_lens) > 0 and (seq_lens == seq_lens[0]).all(): + batch_size = len(seq_lens) + seq_len = int(seq_lens[0]) + + def _packed_hsd_to_bhsd(tensor: torch.Tensor) -> torch.Tensor: + return tensor.unflatten(1, (batch_size, seq_len)).permute(1, 0, 2, 3) + + torch.ops.zentorch.zentorch_sdpa_attn( + _packed_hsd_to_bhsd(query), + _packed_hsd_to_bhsd(key), + _packed_hsd_to_bhsd(value), + # The op writes [B, H, S, D]; hand it the matching view of the + # packed buffer so it stores there instead of into a temporary. + output.unflatten(0, (batch_size, seq_len)).permute(0, 2, 1, 3), + scale=scale, + is_causal=attn_metadata.causal, + alibi_slopes=alibi_slopes, + left_window_size=window, + right_window_size=window, + ) + return output + + for start_q, end_q in zip(start_loc[:-1], start_loc[1:], strict=True): + torch.ops.zentorch.zentorch_sdpa_attn( + query[None, :, start_q:end_q, :], + key[None, :, start_q:end_q, :], + value[None, :, start_q:end_q, :], + output[start_q:end_q, :, :].movedim(0, 1).unsqueeze(0), + scale=scale, + is_causal=attn_metadata.causal, + alibi_slopes=alibi_slopes, + left_window_size=window, + right_window_size=window, + ) + return output diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index 8f844b6476..71e1fdae07 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -27,6 +27,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( triton_scalar_specialization_rep, ) from vllm.triton_utils import tl, triton +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.attention.ops.cp_common import ( DirectCPWorkspace, direct_cp_enabled, @@ -1011,16 +1012,9 @@ def dcp_a2a_lse_reduce( def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int: scheduler_config = vllm_config.scheduler_config - speculative_config = vllm_config.speculative_config - speculative_tokens = vllm_config.num_speculative_tokens - tokens_per_seq = ( - 1 - + ( - 2 - if speculative_config is not None and speculative_config.parallel_drafting - else 1 - ) - * speculative_tokens + # num_speculative_tokens also counts a diffusion canvas. + tokens_per_seq = max( + 1 + vllm_config.num_speculative_tokens, max_decode_query_len(vllm_config) ) return min( scheduler_config.max_num_batched_tokens, diff --git a/vllm/v1/attention/ops/pcp.py b/vllm/v1/attention/ops/pcp.py index 75ab1c9e8e..7e7dbe2245 100644 --- a/vllm/v1/attention/ops/pcp.py +++ b/vllm/v1/attention/ops/pcp.py @@ -18,8 +18,15 @@ def _gather_prefill_cache_inputs( assert all(tensor.shape[0] == local_num_tokens for tensor in tensors) assert 0 <= num_decode_tokens <= local_num_tokens - if num_decode_tokens == local_num_tokens: - return tensors, slot_mapping[:num_decode_tokens] + # Replicated draft decodes use unexpanded slot mappings, even with DP padding. + if ( + num_decode_tokens == local_num_tokens + or slot_mapping.shape[0] <= local_num_tokens + ): + return ( + tuple(tensor[:num_decode_tokens] for tensor in tensors), + slot_mapping[:num_decode_tokens], + ) pcp_group = get_pcp_group() gathered_prefills = tuple( diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 6b1530d74e..a3608a9c7e 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -16,7 +16,7 @@ from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import LayerNameType +from vllm.utils.torch_utils import LayerNameType, direct_register_custom_op from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton from vllm.v1.worker.workspace import current_workspace_manager @@ -1572,8 +1572,13 @@ def _fused_inverse_rope_gptj( positions: torch.Tensor, cos_sin_cache: torch.Tensor, rope_head_dim: int, + out: torch.Tensor | None = None, ) -> torch.Tensor: - """bf16 inverse GPT-J RoPE via a single fused Triton kernel.""" + """bf16 inverse GPT-J RoPE via a single fused Triton kernel. + + ``out`` may alias ``o``: the rotation is a per-row bijection whose kernel + reads both lanes of a pair before storing either. + """ assert o.dim() == 3 and o.stride(-1) == 1, ( "_fused_inverse_rope_gptj expects a [T, H, D] input with a contiguous last dim" ) @@ -1585,9 +1590,14 @@ def _fused_inverse_rope_gptj( f"[P, {rope_head_dim}] = cos | sin, got {tuple(cos_sin_cache.shape)}" ) num_tokens, num_heads, head_dim = o.shape - out = torch.empty( - (num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device - ) + if out is None: + out = torch.empty( + (num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device + ) + else: + assert out.dtype == torch.bfloat16, ( + f"inverse RoPE writes bf16, got an output buffer of {out.dtype}" + ) if num_tokens == 0: return out _inverse_rope_gptj_kernel[(num_tokens, num_heads)]( @@ -1608,6 +1618,109 @@ def _fused_inverse_rope_gptj( return out +def rocm_inverse_rope_rows_( + o: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + rope_head_dim: int, +) -> None: + """Inverse-RoPE attention output rows in place. + + For rows no attention kernel rotated in its epilogue. Call it from the + eager attention segment: which rows still owe a rotation depends on the + prefill/decode split, and the o_proj that used to do this runs inside the + compiled region, where a batch-dependent Python value would be frozen at + trace time. + """ + if o.shape[0] == 0: + return + _fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o) + + +@triton.jit +def _inverse_rope_mxfp8_quant_kernel( + o_ptr, # [T, H, D] bf16 + q_ptr, # [T, H * D] e4m3 + s_ptr, # [T, H * D // 32] uint8 (E8M0) + pos_ptr, + cos_sin_ptr, + s_t, + s_h, + qs_t, + ss_t, + cs_stride, + HEAD_DIM: tl.constexpr, + NOPE: tl.constexpr, + HALF: tl.constexpr, +): + t = tl.program_id(0) + h = tl.program_id(1) + lanes = tl.arange(0, HEAD_DIM) + x = tl.load(o_ptr + t * s_t + h * s_h + lanes).to(tl.float32) + # Same lane-pair formulation as the reduce epilogue: NoPE pairs take + # cos=1 / sin=0, so one expression rotates the whole row. + pos = tl.load(pos_ptr + t).to(tl.int64) + pair_idx = tl.arange(0, HEAD_DIM // 2) - (NOPE // 2) + is_rope = pair_idx >= 0 + k = tl.where(is_rope, pair_idx, 0) + cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0) + sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0) + even, odd = tl.split(tl.reshape(x, (HEAD_DIM // 2, 2))) + x = tl.reshape( + tl.join(even * cos + odd * sin, odd * cos - even * sin), (1, HEAD_DIM) + ) + xq, bits = _mxfp8_quantize_rows(x, 1, HEAD_DIM) + tl.store( + q_ptr + t * qs_t + h * HEAD_DIM + lanes, + tl.reshape(xq, (HEAD_DIM,)).to(q_ptr.dtype.element_ty), + ) + scale_offsets = tl.arange(0, HEAD_DIM // 32) + tl.store( + s_ptr + t * ss_t + h * (HEAD_DIM // 32) + scale_offsets, + tl.reshape(bits, (HEAD_DIM // 32,)).to(tl.uint8), + ) + + +def rocm_inverse_rope_mxfp8_rows( + o: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + rope_head_dim: int, + out_data: torch.Tensor, + out_scale: torch.Tensor, +) -> None: + """Inverse-RoPE bf16 attention rows and MXFP8-quantize them for wo_a. + + The counterpart of ``rocm_inverse_rope_rows_`` for layers whose attention + output is MXFP8: rows the decode reduce did not emit (prefill) go through + here. ``o`` is [T, H, D]; ``out_data`` [T, H * D] e4m3 and ``out_scale`` + [T, H * D // 32] E8M0, the layout the reduce epilogue writes. + """ + num_tokens, num_heads, head_dim = o.shape + if num_tokens == 0: + return + assert o.stride(-1) == 1 and out_data.stride(-1) == 1 and out_scale.stride(-1) == 1 + assert out_data.shape == (num_tokens, num_heads * head_dim) + assert out_scale.shape == (num_tokens, num_heads * head_dim // 32) + assert cos_sin_cache.shape[-1] == rope_head_dim + _inverse_rope_mxfp8_quant_kernel[(num_tokens, num_heads)]( + o, + out_data, + out_scale, + positions, + cos_sin_cache, + o.stride(0), + o.stride(1), + out_data.stride(0), + out_scale.stride(0), + cos_sin_cache.stride(0), + HEAD_DIM=head_dim, + NOPE=head_dim - rope_head_dim, + HALF=rope_head_dim // 2, + num_warps=4, + ) + + def _get_cached_wo_a_bf16( wo_a: torch.nn.Module, n_local_groups: int, @@ -1664,16 +1777,27 @@ def rocm_inv_rope_einsum( n_local_groups: int, o_lora_rank: int, wo_a: torch.nn.Module, + inverse_rope: bool = True, ) -> torch.Tensor: """Inverse-RoPE + WO_A bmm path used on ROCm. Fuses the inverse GPT-J RoPE into one Triton kernel and caches the bf16 - wo_a weight so the per-step dequant disappears. + wo_a weight so the per-step dequant disappears. Callers whose attention + already rotated every row pass ``inverse_rope=False``; that is a property + of the attention backend, not of the batch, so it stays constant across + steps and is safe to read from compiled code. """ - o_ref = _fused_inverse_rope_gptj( - o, positions, rotary_emb.cos_sin_cache, rope_head_dim - ) - o_ref = o_ref.view(o.shape[0], n_local_groups, -1) + if inverse_rope: + o_ref = _fused_inverse_rope_gptj( + o, positions, rotary_emb.cos_sin_cache, rope_head_dim + ) + else: + assert o.dtype == torch.bfloat16, ( + "a pre-rotated attention output feeds the wo_a bmm directly, so it " + f"must already be bf16, got {o.dtype}" + ) + o_ref = o + o_ref = o_ref.reshape(o.shape[0], n_local_groups, -1) wo_a_weight = _get_cached_wo_a_bf16( wo_a, n_local_groups, o_lora_rank, o_ref.shape[-1] @@ -1682,6 +1806,178 @@ def rocm_inv_rope_einsum( return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight) +@triton.jit +def _mxfp8_wo_a_bmm_kernel( + a_ptr, # [T, G * K] e4m3 + as_ptr, # [T, G * K // 32] E8M0 + w_ptr, # [G * R, K] e4m3 + ws_ptr, # [G * R // SCALE_ROWS, K // 32] E8M0 + out_ptr, # [T, G * R] + num_tokens, + stride_at, + stride_ast, + stride_wn, + stride_wsn, + stride_out, + R: tl.constexpr, + K: tl.constexpr, + SCALE_ROWS: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + # Output columns are group-major, so a BLOCK_N tile never straddles two + # groups (R % BLOCK_N == 0) and picks its group's K slice of A. + pid_n = tl.program_id(0) + pid_m = tl.program_id(1) + group = (pid_n * BLOCK_N) // R + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + m_mask = offs_m < num_tokens + offs_k = tl.arange(0, BLOCK_K) + offs_sk = tl.arange(0, BLOCK_K // 32) + + a_ptrs = a_ptr + offs_m[:, None] * stride_at + (group * K + offs_k)[None, :] + as_ptrs = ( + as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :] + ) + w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] + ws_ptrs = ws_ptr + (offs_n // SCALE_ROWS)[:, None] * stride_wsn + offs_sk[None, :] + + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for _ in range(K // BLOCK_K): + a = tl.load(a_ptrs, mask=m_mask[:, None], other=0.0) + a_s = tl.load(as_ptrs, mask=m_mask[:, None], other=127) + w = tl.load(w_ptrs) + w_s = tl.load(ws_ptrs) + acc += tl.dot_scaled(a, a_s, "e4m3", w.T, w_s, "e4m3") + a_ptrs += BLOCK_K + as_ptrs += BLOCK_K // 32 + w_ptrs += BLOCK_K + ws_ptrs += BLOCK_K // 32 + + tl.store( + out_ptr + offs_m[:, None] * stride_out + offs_n[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=m_mask[:, None], + ) + + +def _mxfp8_wo_a_bmm_config(num_tokens: int, n_groups: int) -> tuple[int, ...]: + """(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) for gfx950. + + Tuned under HIP graphs with a cold weight at G = 4 and 2, over every + decode shape of conc 1-128 x 0-5 spec tokens plus prefill chunks up to + 8K tokens. The best tile tracks the total work T * G, so the tiers are + keyed on it. + + This will be replaced after new GEMM kernel from AITER with proper 32x32 scale + shape GEMM fp8 enabled. + """ + work = num_tokens * n_groups + if work <= 64: + return 16, 16, 1024, 2, 3 + if work <= 128: + return 32, 16, 1024, 2, 3 + if work <= 256: + return 32, 32, 512, 2, 3 + if work <= 512: + return 64, 32, 512, 2, 3 + if work <= 1024: + return 64, 64, 512, 4, 2 + if work <= 2048: + return 64, 64, 256, 4, 2 + if work <= 3072: + return 64, 64, 256, 2, 1 + if work <= 4096: + return 128, 128, 256, 8, 2 + return 128, 128, 128, 4, 2 + + +def _rocm_mxfp8_wo_a_bmm_impl( + a: torch.Tensor, + a_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + n_groups: int, + o_lora_rank: int, +) -> torch.Tensor: + num_tokens = a.shape[0] + group_dim = a.shape[1] // n_groups + out = torch.empty( + (num_tokens, n_groups * o_lora_rank), dtype=torch.bfloat16, device=a.device + ) + if num_tokens == 0: + return out + block_m, block_n, block_k, num_warps, num_stages = _mxfp8_wo_a_bmm_config( + num_tokens, n_groups + ) + grid = (n_groups * o_lora_rank // block_n, triton.cdiv(num_tokens, block_m)) + _mxfp8_wo_a_bmm_kernel[grid]( + a, + a_scale, + weight, + weight_scale, + out, + num_tokens, + a.stride(0), + a_scale.stride(0), + weight.stride(0), + weight_scale.stride(0), + out.stride(0), + R=o_lora_rank, + K=group_dim, + SCALE_ROWS=weight.shape[0] // weight_scale.shape[0], + BLOCK_M=block_m, + BLOCK_N=block_n, + BLOCK_K=block_k, + num_warps=num_warps, + num_stages=num_stages, + ) + return out + + +def _rocm_mxfp8_wo_a_bmm_fake( + a: torch.Tensor, + a_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + n_groups: int, + o_lora_rank: int, +) -> torch.Tensor: + return a.new_empty((a.shape[0], n_groups * o_lora_rank), dtype=torch.bfloat16) + + +# An opaque op: the tile choice branches on the token count, which the +# compiled o_proj must not freeze at its trace-time value. +direct_register_custom_op( + op_name="rocm_dsv41_mxfp8_wo_a_bmm", + op_func=_rocm_mxfp8_wo_a_bmm_impl, + fake_impl=_rocm_mxfp8_wo_a_bmm_fake, +) + + +def rocm_mxfp8_wo_a_bmm( + a: torch.Tensor, + a_scale: torch.Tensor, + wo_a: torch.nn.Module, + n_groups: int, + o_lora_rank: int, +) -> torch.Tensor: + """Grouped MXFP8 wo_a: ``out[t, g, :] = a[t, g, :] @ W[g].T``, bf16 out. + + ``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its + [T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them. + The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with + either [G * R // 32, K // 32] block scales or [G * R, K // 32] per-row + scales, so there is no dequantized copy to keep. + Returns [T, G * R]. + """ + return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm( + a, a_scale, wo_a.weight, wo_a.weight_scale, n_groups, o_lora_rank + ) + + _DSV4_SPARSE_NOPE_DIM = 448 _DSV4_SPARSE_ROPE_DIM = 64 @@ -2882,6 +3178,33 @@ def _sparse_attn_decode_gfx950_partial_kernel( ) +@triton.jit +def _mxfp8_scale_bits(amax): + """Biased E8M0 exponent that puts ``amax`` at the top of the e4m3 range. + + Same rounding as ``mxfp8_e4m3_quantize``, so the output is bit-identical to + quantizing the tensor there. + """ + amax = tl.maximum(amax, 1.1754943508222875e-38) + bits = tl.ceil(tl.log2(amax / 448.0)) + 127.0 + return tl.minimum(tl.maximum(bits, 0.0), 254.0) + + +@triton.jit +def _mxfp8_quantize_rows(x, ROWS: tl.constexpr, COLS: tl.constexpr): + """MXFP8-quantize ``x`` [ROWS, COLS] in registers, one scale per 32 lanes. + + Returns the rescaled fp32 values (to be cast to e4m3 on store) and the + [ROWS, COLS // 32] biased E8M0 exponents. + """ + blocks = tl.reshape(x, (ROWS, COLS // 32, 32)) + bits = _mxfp8_scale_bits(tl.max(tl.abs(blocks), axis=2)) + # Multiply by the reciprocal: a divisor of 2**-127 would be subnormal and + # flush to zero, turning an all-zero block into NaN. + q = blocks * tl.exp2(127.0 - bits)[:, :, None] + return tl.reshape(q, (ROWS, COLS)), bits + + @triton.jit def _sparse_attn_decode_reduce_kernel( part_m_ptr, @@ -2889,13 +3212,19 @@ def _sparse_attn_decode_reduce_kernel( part_acc_ptr, attn_sink_ptr, out_ptr, + pos_ptr, + cos_sin_ptr, + out_scale_ptr, out_stride0, out_stride1, + os_stride0, + os_stride1, pm_stride0, pm_stride_s, pa_stride0, pa_stride_s, pa_stride_h, + cs_stride, num_heads, HAS_ATTN_SINK: tl.constexpr, ADAPTIVE_SPLITS: tl.constexpr, @@ -2903,6 +3232,10 @@ def _sparse_attn_decode_reduce_kernel( BLOCK_H: tl.constexpr, NUM_SPLITS: tl.constexpr, SPLITS_PAD: tl.constexpr, + FUSE_INV_ROPE: tl.constexpr, + NOPE: tl.constexpr, + HALF: tl.constexpr, + QUANT_OUT: tl.constexpr, ): query_idx = tl.program_id(0) pid_h = tl.program_id(1) @@ -2992,12 +3325,45 @@ def _sparse_attn_decode_reduce_kernel( out = tl.where(l_final[:, None] > 0.0, acc / denom[:, None], 0.0) + if FUSE_INV_ROPE: + # Inverse GPT-J RoPE on the trailing rope lanes, straight out of the + # combine registers: out_even = a*cos + b*sin, out_odd = b*cos - a*sin. + # NoPE lanes take cos=1/sin=0 so one expression covers the whole row + # and the o_proj rotation pass disappears for these tokens. + pos = tl.load(pos_ptr + query_idx).to(tl.int64) + pair_idx = tl.arange(0, COMB_DIM // 2) - (NOPE // 2) + is_rope = pair_idx >= 0 + k = tl.where(is_rope, pair_idx, 0) + cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0) + sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0) + even, odd = tl.split(tl.reshape(out, (BLOCK_H, COMB_DIM // 2, 2))) + out = tl.reshape( + tl.join( + even * cos[None, :] + odd * sin[None, :], + odd * cos[None, :] - even * sin[None, :], + ), + (BLOCK_H, COMB_DIM), + ) + out_row_ptr = ( out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1 ) + if QUANT_OUT: + # Emit wo_a's MXFP8 input directly instead of a bf16 row that a + # separate pass would read back to quantize. + out, scale_bits = _mxfp8_quantize_rows(out, BLOCK_H, COMB_DIM) + scale_offsets = tl.arange(0, COMB_DIM // 32) + tl.store( + out_scale_ptr + + query_idx * os_stride0 + + head_offsets[:, None] * os_stride1 + + scale_offsets[None, :], + scale_bits.to(tl.uint8), + mask=head_mask[:, None], + ) tl.store( out_row_ptr + comb_offsets[None, :], - out, + out.to(out_ptr.dtype.element_ty), mask=head_mask[:, None], ) @@ -3312,7 +3678,16 @@ def _rocm_sparse_attn_decode_ragged_triton( out: torch.Tensor | None = None, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, + out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> torch.Tensor: + """Split-K sparse decode; returns the attention output. + + With ``out_mxfp8 = (data, scale)`` the reduce writes MXFP8 instead of + bf16: ``data`` is [b, h * d] e4m3 and ``scale`` [b, h * d // 32] E8M0, and + ``data`` viewed as [b, h, d] is returned. + """ assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}" assert main_cache.ndim == 3, ( f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}" @@ -3377,7 +3752,28 @@ def _rocm_sparse_attn_decode_ragged_triton( extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32) block_h = 16 - if out is None: + out_scale = None + if out_mxfp8 is not None: + assert out is None, "out and out_mxfp8 are mutually exclusive" + assert _ON_GFX950, "the MXFP8 reduce epilogue is gfx950-only" + assert inv_rope_positions is not None, ( + "the MXFP8 output feeds wo_a, so it must be inverse-RoPE'd first" + ) + out_data, out_scale = out_mxfp8 + assert out_data.dtype == torch.float8_e4m3fn and out_scale.dtype == ( + torch.uint8 + ), f"expected e4m3/uint8 MXFP8 buffers, got {out_data.dtype}/{out_scale.dtype}" + assert out_data.shape == (num_queries, num_heads * head_dim), ( + f"expected MXFP8 data [{num_queries}, {num_heads * head_dim}], " + f"got {tuple(out_data.shape)}" + ) + assert out_scale.shape == (num_queries, num_heads * head_dim // 32), ( + f"expected MXFP8 scale [{num_queries}, {num_heads * head_dim // 32}], " + f"got {tuple(out_scale.shape)}" + ) + assert out_data.stride(-1) == 1 and out_scale.stride(-1) == 1 + out = out_data.view(num_queries, num_heads, head_dim) + elif out is None: out = torch.empty_like(q, dtype=torch.bfloat16) else: assert out.shape == q.shape, f"expected out shape {q.shape}, got {out.shape}" @@ -3554,19 +3950,38 @@ def _rocm_sparse_attn_decode_ragged_triton( num_warps=4, ) + fuse_inv_rope = inv_rope_positions is not None + if fuse_inv_rope: + assert inv_rope_cos_sin_cache is not None + assert inv_rope_cos_sin_cache.shape[-1] == rope_head_dim, ( + "fused inverse RoPE expects cos_sin_cache laid out as " + f"[P, {rope_head_dim}] = cos | sin, got " + f"{tuple(inv_rope_cos_sin_cache.shape)}" + ) + assert nope_head_dim % 2 == 0 and rope_head_dim % 2 == 0, ( + "fused inverse RoPE pairs adjacent lanes, so both head dims must " + f"be even, got nope={nope_head_dim} rope={rope_head_dim}" + ) + _sparse_attn_decode_reduce_kernel[(num_queries, num_heads)]( part_m, part_l, part_acc, attn_sink, out, + inv_rope_positions, + inv_rope_cos_sin_cache, + out_scale, out.stride(0), out.stride(1), + out_scale.stride(0) if out_scale is not None else 0, + head_dim // 32, part_m.stride(0), part_m.stride(1), part_acc.stride(0), part_acc.stride(1), part_acc.stride(2), + inv_rope_cos_sin_cache.stride(0) if inv_rope_cos_sin_cache is not None else 0, num_heads, HAS_ATTN_SINK=has_attn_sink, ADAPTIVE_SPLITS=adaptive_splits, @@ -3574,6 +3989,10 @@ def _rocm_sparse_attn_decode_ragged_triton( BLOCK_H=1, NUM_SPLITS=num_splits, SPLITS_PAD=triton.next_power_of_2(num_splits), + FUSE_INV_ROPE=fuse_inv_rope, + NOPE=nope_head_dim, + HALF=rope_head_dim // 2, + QUANT_OUT=out_scale is not None, num_warps=4, ) return out @@ -3598,6 +4017,9 @@ def _rocm_sparse_attn_decode_triton( out: torch.Tensor | None = None, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, + out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> torch.Tensor: if main_ragged_indices is None or main_ragged_indptr is None: main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense( @@ -3636,6 +4058,9 @@ def _rocm_sparse_attn_decode_triton( out=out, extra_cache_nan_free=extra_cache_nan_free, adaptive_splits=adaptive_splits, + inv_rope_positions=inv_rope_positions, + inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, + out_mxfp8=out_mxfp8, ) @@ -3734,10 +4159,25 @@ def rocm_sparse_attn_decode( head_dim: int, nope_head_dim: int, rope_head_dim: int, - output: torch.Tensor, + output: torch.Tensor | None, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, -) -> None: + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, + output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None, +) -> int: + """Run sparse MLA decode into ``output``. + + Passing ``inv_rope_positions`` folds the inverse RoPE into the reduce + epilogue. Returns how many leading rows of ``output`` came back rotated, + so a caller mixing in a decode path that does not fuse still knows what it + owes the standalone pass. Read it from the eager attention segment only. + + ``output_mxfp8 = (data, scale)`` replaces ``output``: the reduce also + MXFP8-quantizes the rotated rows for the FP8 wo_a (see + ``_rocm_sparse_attn_decode_ragged_triton``). It needs gfx950 and the fused + inverse RoPE, and always covers every row. + """ assert swa_k_cache.dtype == torch.uint8, ( "ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, " f"got {swa_k_cache.dtype}" @@ -3766,7 +4206,12 @@ def rocm_sparse_attn_decode( if topk_indices is not None: extra_indices = topk_indices.reshape(topk_indices.shape[0], -1) - direct_out = output if _ON_GFX950 and output.dtype == torch.bfloat16 else None + if output_mxfp8 is not None: + assert output is None, "output and output_mxfp8 are mutually exclusive" + direct_out = None + else: + assert output is not None + direct_out = output if _ON_GFX950 and output.dtype == torch.bfloat16 else None attn_out = _rocm_sparse_attn_decode_triton( q=q, main_cache=swa_k_cache, @@ -3786,6 +4231,13 @@ def rocm_sparse_attn_decode( out=direct_out, extra_cache_nan_free=extra_cache_nan_free, adaptive_splits=adaptive_splits, + inv_rope_positions=inv_rope_positions, + inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, + out_mxfp8=output_mxfp8, ) + if output_mxfp8 is not None: + return q.shape[0] + assert output is not None if direct_out is None: output.copy_(attn_out.to(output.dtype)) + return output.shape[0] if inv_rope_positions is not None else 0 diff --git a/vllm/v1/attention/ops/triton_attention_helpers.py b/vllm/v1/attention/ops/triton_attention_helpers.py index dff0b21c64..7804b01317 100644 --- a/vllm/v1/attention/ops/triton_attention_helpers.py +++ b/vllm/v1/attention/ops/triton_attention_helpers.py @@ -28,12 +28,16 @@ def cdiv_fn(x, y): def apply_softcap(S, x): """Softcap (aka tanh-style clamp) used to bound attention scores. - ``x * tanh(S / x)`` rewritten to avoid a direct ``tanh`` call. + ``x * tanh(S / x)`` computed without a direct ``tanh`` call. + + Use ``tanh(y) = 1 - 2 / (exp(2y) + 1)`` rather than + ``(exp(y) - exp(-y)) / (exp(y) + exp(-y))``. The symmetric-exponential + form overflows to ``inf`` for ``|y| > ~88`` and then evaluates to + ``inf / inf = NaN``, poisoning the attention row. The single-exponential + form saturates cleanly at +/-1 for either sign. """ - Sdiv = S / x - p1 = tl.exp(Sdiv) - p2 = tl.exp(-Sdiv) - return x * (p1 - p2) / (p1 + p2) + y = S / x + return x * (1.0 - 2.0 / (tl.exp(2.0 * y) + 1.0)) # =========================================================================== diff --git a/vllm/v1/attention/ops/triton_prefill_attention.py b/vllm/v1/attention/ops/triton_prefill_attention.py index 00d630edcd..1ed55c3e28 100644 --- a/vllm/v1/attention/ops/triton_prefill_attention.py +++ b/vllm/v1/attention/ops/triton_prefill_attention.py @@ -32,6 +32,16 @@ from vllm.triton_utils import tl, triton from vllm.utils.math_utils import RCP_LN2 +def _prefer_narrow_kv_tile() -> bool: + """RDNA3/RDNA4 prefer a narrower KV tile than the shared default.""" + if not current_platform.is_rocm(): + return False + from vllm.platforms.rocm import on_gfx1x + + # RDNA3/RDNA4 only: on_gfx1x() excludes gfx10xx and gfx1250. + return on_gfx1x() + + @triton.jit def _fwd_kernel( Q, @@ -249,6 +259,10 @@ def context_attention_fwd( grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) num_warps = 4 if Lk <= 64 else 8 + # BLOCK_M, num_warps and num_stages stay at the shared defaults; min() + # leaves dtypes whose default tile is already 32, such as float32, alone. + BLOCK_N = min(BLOCK, 32) if _prefer_narrow_kv_tile() else BLOCK + sliding_window_q = sliding_window_q if sliding_window_q is not None else 0 sliding_window_k = sliding_window_k if sliding_window_k is not None else 0 @@ -272,7 +286,7 @@ def context_attention_fwd( kv_group_num=kv_group_num, BLOCK_M=BLOCK, BLOCK_DMODEL=triton.next_power_of_2(Lk), - BLOCK_N=BLOCK, + BLOCK_N=BLOCK_N, IS_CAUSAL=is_causal, SLIDING_WINDOW_Q=sliding_window_q, SLIDING_WINDOW_K=sliding_window_k, diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index b90eb58e9b..dccc670ce5 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -241,9 +241,17 @@ def apply_sdpa( (batch_size x seq_len x num_heads x head_size) """ q, k, v = (einops.rearrange(x, "b s h d -> b h s d") for x in [q, k, v]) - output = F.scaled_dot_product_attention( - q, k, v, dropout_p=0.0, scale=scale, enable_gqa=enable_gqa - ) + if current_platform.is_zen_cpu() and hasattr(torch.ops.zentorch, "zentorch_sdpa"): + # Schema: (query, key, value, dropout_p=0., is_causal=False, *, + # attn_mask=None, scale=None) -> (attention, logsumexp). + # No enable_gqa: GQA is inferred from key.size(1) vs query.size(1). + output, _ = torch.ops.zentorch.zentorch_sdpa( + q, k, v, dropout_p=0.0, is_causal=False, scale=scale + ) + else: + output = F.scaled_dot_product_attention( + q, k, v, dropout_p=0.0, scale=scale, enable_gqa=enable_gqa + ) output = einops.rearrange(output, "b h s d -> b s h d ") return output diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 695916d819..a7e8cd0fd2 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -194,6 +194,9 @@ def get_attn_backend( backend=backend, attn_selector_config=attn_selector_config, num_heads=num_heads, + _run_kv_cache_dtype=( + cache_config.cache_dtype if cache_config is not None else None + ), ) @@ -202,7 +205,11 @@ def _cached_get_attn_backend( backend, attn_selector_config: AttentionSelectorConfig, num_heads: int | None = None, + *, + _run_kv_cache_dtype: CacheDType | None = None, ) -> type[AttentionBackend]: + # Some platform selectors inspect the run-wide cache dtype. Keep it in the + # cache key even though they read the value from the current config. from vllm.platforms import current_platform attention_cls = current_platform.get_attn_backend_cls( diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index b4c9dcaa29..9289e7c1f8 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -11,6 +11,7 @@ from vllm.distributed.kv_events import ( KVCacheEvent, ) from vllm.logger import init_logger +from vllm.multimodal.utils import get_mm_features_in_window from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import ( BlockHash, @@ -532,7 +533,14 @@ class BlockPool: else None ) block_end = num_tokens - curr_mm_idx = -1 if block_start > 0 else 0 + curr_mm_idx = 0 + mm_features = request.mm_features + if block_start > 0 and mm_features: + last_mm_pos = mm_features[-1].mm_position + if last_mm_pos.offset + last_mm_pos.length > block_start: + curr_mm_idx, _ = get_mm_features_in_window( + mm_features, block_start, block_end + ) extra_keys, _ = generate_block_hash_extra_keys( request, block_start, block_end, curr_mm_idx ) diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index f069ce7453..24cf609d22 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -59,6 +59,8 @@ class EncoderCacheManager: reference the cached entry. If the set is empty, the entry exists but is not referenced by any request and is eligible for reclamation. + num_embeds_by_hash: Mapping from mm_hash to the number of encoder + embeddings stored for that cache entry. freeable: List of tuples (mm_hash, num_encoder_embeds) representing entries whose no current running request is needed and that can be freed to make space when needed. @@ -80,6 +82,8 @@ class EncoderCacheManager: # mm_hash of mm_data => ids of requests that reference the mm_data self.cached: dict[str, set[str]] = {} + # mm_hash of mm_data => number of cached encoder embeddings + self.num_embeds_by_hash: dict[str, int] = {} # request_id => set of input_ids cached for that request self.request_cached_ids: dict[str, set[int]] = {} @@ -94,6 +98,7 @@ class EncoderCacheManager: Called when model weights are updated to invalidate stale embeddings. """ self.cached.clear() + self.num_embeds_by_hash.clear() self.request_cached_ids.clear() self.freeable.clear() self.freed.clear() @@ -130,6 +135,13 @@ class EncoderCacheManager: self.request_cached_ids.setdefault(request.request_id, set()).add(input_id) return True + def get_cached_num_encoder_embeds( + self, request: Request, input_id: int + ) -> int | None: + """Return the encoder-embedding count stored for a cached input.""" + mm_hash = request.mm_features[input_id].identifier + return self.num_embeds_by_hash.get(mm_hash) + def can_allocate( self, request: Request, @@ -188,6 +200,7 @@ class EncoderCacheManager: while num_embeds > self.num_free_slots: mm_hash, num_free_embeds = self.freeable.popitem(last=False) del self.cached[mm_hash] + del self.num_embeds_by_hash[mm_hash] self.freed.append(mm_hash) self.num_free_slots += num_free_embeds return True @@ -209,6 +222,12 @@ class EncoderCacheManager: self.cached[mm_hash] = set() num_encoder_embeds = request.get_num_encoder_embeds(input_id) + cached_num_encoder_embeds = self.num_embeds_by_hash.setdefault( + mm_hash, num_encoder_embeds + ) + # _try_schedule_encoder_inputs rejects local-hit and same-step + # identifier/count mismatches before allocation. + assert cached_num_encoder_embeds == num_encoder_embeds # NOTE: Encoder cache should always have enough space for encoder inputs # that are scheduled since eviction takes place at can_allocate(). @@ -258,7 +277,7 @@ class EncoderCacheManager: return self.cached[mm_hash].discard(req_id) if not self.cached[mm_hash]: - num_encoder_embeds = request.get_num_encoder_embeds(input_id) + num_encoder_embeds = self.num_embeds_by_hash[mm_hash] self.freeable[mm_hash] = num_encoder_embeds self.num_freeable_slots += num_encoder_embeds diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 8061999409..d69b2f5fb3 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import ( BlockHash, KVCacheBlock, - dcp_world_size_for_kv_cache_spec, ) from vllm.v1.core.single_type_kv_cache_manager import ( CrossAttentionManager, @@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import ( get_manager_for_kv_cache_spec, ) from vllm.v1.kv_cache_interface import ( + AttentionSpec, FullAttentionSpec, KVCacheConfig, KVCacheSpec, @@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC): role=kv_cache_group.role, enable_caching=enable_caching, kv_cache_group_id=i, - dcp_world_size=dcp_world_size_for_kv_cache_spec( - kv_cache_group.kv_cache_spec, dcp_world_size - ), + dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size, scheduler_block_size=self.scheduler_block_size, needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing, @@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): f"hash_block_size. block_sizes={cacheable_block_sizes}, " f"hash_block_size={hash_block_size}" ) - assert pcp_world_size == 1, "PCP not support hybrid attn now." - if dcp_world_size > 1: - # DCP shards full-attention KV across ranks and replicates Mamba - # state; other spec types (e.g. sliding window) have no DCP-aware - # handling yet, so reject them explicitly. - for g in kv_cache_config.kv_cache_groups: - assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), ( + for g in kv_cache_config.kv_cache_groups: + spec = g.kv_cache_spec + replicated = not spec.dcp_sharded + if pcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or ( + isinstance(spec, AttentionSpec) and replicated + ), "PCP only supports full attention and replicated draft groups." + if dcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or replicated, ( "DCP with hybrid KV cache layouts only supports " - "full-attention and Mamba groups, got: " - f"{type(g.kv_cache_spec).__name__}." + "full-attention, Mamba, and replicated draft groups, got: " + f"{type(spec).__name__}." ) # Fine-grained hash hits require Mamba "align" and compatible cache # managers in every group. TP needs hashing finer than the Mamba block; diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index f881fc9818..30bf584f8c 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -437,7 +437,7 @@ class KVCacheManager: ---------------------------------------------------------------------- ``` - Abbrivations: + Abbreviations: ``` comp = request.num_computed_tokens diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index af1112c54a..3f4f4e7bb1 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, ove from vllm import envs from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.multimodal.utils import get_mm_features_in_window from vllm.utils.hashing import xxhash, xxhash_cbor from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_utils import format_gib @@ -681,12 +682,7 @@ def hash_block_tokens( def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int: """Return the token span of a cache block under DCP.""" - layer_specs = iter_layer_specs(spec) - if len(layer_specs) > 0 and all( - isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs - ): - return spec.block_size * dcp_world_size - return spec.block_size + return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1) def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec: @@ -706,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache return replace(spec, block_size=block_size) -def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int: - """Return the DCP size that owns this group's block geometry. - - Full-attention KV (including MLA) is sharded across DCP ranks, so prefix - hashing and manager ``block_size`` use the process DCP size. Other specs - keep replicated per-rank state (Mamba, sliding window, chunked-local) and - must keep ``dcp_world_size=1`` even when the process runs with DCP > 1. - - Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` / - ``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated - draft group would need a different spec, not this helper. - """ - if dcp_world_size <= 1: - return 1 - inner = spec - if isinstance(spec, UniformTypeKVCacheSpecs): - inner = next(iter(spec.kv_cache_specs.values())) - if isinstance(inner, FullAttentionSpec): - return dcp_world_size - return 1 - - def resolve_kv_cache_block_sizes( kv_cache_config: KVCacheConfig, vllm_config: VllmConfig, @@ -751,6 +725,8 @@ def resolve_kv_cache_block_sizes( groups = kv_cache_config.kv_cache_groups if len(groups) <= 1: + if groups and not groups[0].kv_cache_spec.dcp_sharded: + dcp = 1 bs = cache_config.block_size * dcp return bs, bs @@ -845,12 +821,15 @@ def get_request_block_hasher( return [] curr_mm_idx = 0 - if start_token_idx > 0: - # Set curr_mm_idx = -1 to indicate the last mm input. - # Note that since we reach to this branch only when the block is - # completed with generated tokens, we only need to consider the - # last mm input. - curr_mm_idx = -1 + mm_features = request.mm_features + if start_token_idx > 0 and mm_features: + last_mm_pos = mm_features[-1].mm_position + if last_mm_pos.offset + last_mm_pos.length > start_token_idx: + curr_mm_idx, _ = get_mm_features_in_window( + mm_features, + start_token_idx, + start_token_idx + hash_block_size, + ) prev_block_hash_value = ( request.block_hashes[-1] if request.block_hashes else None @@ -1604,23 +1583,17 @@ def _get_kv_cache_bytes_per_block( for group in kv_cache_groups ) assert bytes_per_block > 0 - hot_page_sizes = [ + alignments = [ group.kv_cache_spec.page_size_bytes for group in kv_cache_groups if isinstance(group.kv_cache_spec, HiSparseHotSpec) ] - if hot_page_sizes: - bytes_per_block = round_up(bytes_per_block, math.lcm(*hot_page_sizes)) - stride_alignments = [ - spec.block_stride_alignment + alignments.extend( + _get_per_layer_spec(group, layer_name).block_stride_alignment or 1 for group in kv_cache_groups for layer_name in group.layer_names - if isinstance(spec := _get_per_layer_spec(group, layer_name), MLAAttentionSpec) - and spec.block_stride_alignment - ] - if stride_alignments: - bytes_per_block = round_up(bytes_per_block, math.lcm(*stride_alignments)) - return bytes_per_block + ) + return round_up(bytes_per_block, math.lcm(*alignments)) def validate_kv_cache_layout( @@ -1937,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]): kv_cache_spec: The kv cache spec of each attention layer in the model """ - if is_kv_cache_spec_uniform( - kv_cache_spec - ) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec): - return - - logger.warning( - "Hybrid KV cache manager is disabled for this hybrid model, " - "This means we do not enable any optimizations for saving KV cache " - "memory (e.g., dropping the KV cache outside the sliding window). " - "The compute of layers like sliding window is still saved." - ) - kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec)) + groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict) + for name, spec in kv_cache_spec.items(): + replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded + groups[replicated][name] = spec + for specs in groups.values(): + promoted_specs = _promote_local_kv_cache_specs(specs) + if promoted_specs == specs: + continue + logger.warning( + "Hybrid KV cache manager is disabled for this hybrid model, " + "This means we do not enable any optimizations for saving KV cache " + "memory (e.g., dropping the KV cache outside the sliding window). " + "The compute of layers like sliding window is still saved." + ) + kv_cache_spec.update(promoted_specs) def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int: @@ -2278,7 +2254,7 @@ def _ensure_min_page_size( scaled: list[KVCacheGroupSpec] = [] for g in groups: s = g.kv_cache_spec - kw: dict[str, int] = {"block_size": s.block_size * scale} + kw: dict[str, Any] = {"block_size": s.block_size * scale} if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None: kw["page_size_padded"] = s.page_size_padded * scale scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw))) diff --git a/vllm/v1/core/sched/async_scheduler.py b/vllm/v1/core/sched/async_scheduler.py index 56a532b809..290204d58c 100644 --- a/vllm/v1/core/sched/async_scheduler.py +++ b/vllm/v1/core/sched/async_scheduler.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import vllm.envs as envs from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.request import Request, RequestStatus @@ -15,6 +17,10 @@ class AsyncScheduler(Scheduler): # reusable read-only placeholder list for speculative decoding. self._spec_token_placeholders: list[int] = [-1] * self.num_spec_tokens self.pp_size = self.parallel_config.pipeline_parallel_size + # Only XPU can disable PP decode microbatching via VLLM_XPU_PP_MICROBATCH. + self.decode_stagger = self.pp_size + if current_platform.is_xpu() and not envs.VLLM_XPU_PP_MICROBATCH: + self.decode_stagger = 1 def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None: super()._update_after_schedule(scheduler_output) @@ -46,7 +52,9 @@ class AsyncScheduler(Scheduler): if self.use_v2_model_runner: # Set the next step index in which this request is eligible to be # scheduled for decode (for PP microbatching). - request.next_decode_eligible_step = self.current_step + self.pp_size + request.next_decode_eligible_step = ( + self.current_step + self.decode_stagger + ) def _update_request_with_output( self, request: Request, new_token_ids: list[int], is_stale: bool = False diff --git a/vllm/v1/core/sched/diffusion_scheduler.py b/vllm/v1/core/sched/diffusion_scheduler.py new file mode 100644 index 0000000000..96462aa6ab --- /dev/null +++ b/vllm/v1/core/sched/diffusion_scheduler.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Async scheduling for diffusion requests. + +These rules live here so Scheduler.schedule() and AsyncScheduler stay +unchanged. VllmConfig selects this class for a diffusion model under async +scheduling. A sync scheduler creates no output placeholders, which both rules +read. +""" + +from vllm.v1.core.sched.async_scheduler import AsyncScheduler +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.request import Request + + +def diffusion_canvas_width(request: Request, canvas_length: int) -> int: + """The canvas width a diffusion request asked for, else the served one.""" + params = request.sampling_params + extra = params.extra_args if params is not None else None + width = extra.get("diffusion_canvas_length") if extra else None + return int(width) if width else canvas_length + + +def _read_in_flight(request: Request, width: int) -> bool: + """True when a read-only request has all its denoise steps in flight. + + The request emits its canvas on the last step and ends, so a further + step is discarded. + """ + params = request.sampling_params + extra = params.extra_args if params is not None else None + if not extra or not extra.get("diffusion_read_only"): + return False + steps = extra.get("diffusion_max_steps") + if not steps: + return False + # Each in-flight denoise step holds one canvas of placeholders. + return request.num_output_placeholders >= steps * width + + +class DiffusionAsyncScheduler(AsyncScheduler): + def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: + for request in self.running: + width = diffusion_canvas_width(request, self.num_spec_tokens) + # The placeholders are as wide as the served canvas. + if len(request.spec_token_ids) > width: + request.spec_token_ids = request.spec_token_ids[:width] + if _read_in_flight(request, width): + # Scheduler.schedule()'s max_tokens guard cannot be reached + # from a subclass. schedule() advances current_step before it + # checks decode eligibility, so current_step + 2 skips this + # call alone. max keeps a longer pipeline-parallel wait. + request.next_decode_eligible_step = max( + request.next_decode_eligible_step, self.current_step + 2 + ) + return super().schedule(throttle_prefills) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 18bbc10283..ce832a18f6 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -134,6 +134,9 @@ class Scheduler(SchedulerInterface): if self.scheduler_config.max_num_scheduled_tokens is not None else self.scheduler_config.max_num_batched_tokens ) + self.adaptive_long_prefill_threshold = ( + self.scheduler_config.long_prefill_token_threshold_adaptive + ) self.max_model_len = vllm_config.model_config.max_model_len self.enable_kv_cache_events = ( self.kv_events_config is not None @@ -231,6 +234,7 @@ class Scheduler(SchedulerInterface): # Grammar compilation failures to finish as per-request errors in # update_from_output. self.grammar_compile_error_reqs: set[str] = set() + self.encoder_cache_mismatch_reqs: set[str] = set() # Encoder-related. # Calculate encoder cache size if applicable @@ -602,11 +606,20 @@ class Scheduler(SchedulerInterface): # `long_prefill_token_threshold` exists to stop a long prefill from # starving other requests of the token budget. When it is the only # request there is nobody to starve, so let it use the whole budget. + num_eligible_reqs = ( + len(self.running) + len(self.waiting) + len(self.skipped_waiting) + ) long_prefill_token_threshold = ( self.scheduler_config.long_prefill_token_threshold - if len(self.running) + len(self.waiting) + len(self.skipped_waiting) > 1 + if num_eligible_reqs > 1 else 0 ) + if long_prefill_token_threshold > 0 and self.adaptive_long_prefill_threshold: + # Floor the cap at a fair share of the input budget so it never + # cuts a request below max_num_batched_tokens / num requests. + long_prefill_token_threshold = max( + long_prefill_token_threshold, input_budget // num_eligible_reqs + ) # First, schedule the RUNNING requests. req_index = 0 @@ -1702,6 +1715,29 @@ class Scheduler(SchedulerInterface): num_output_tokens=num_output_tokens, ) + def _reject_on_encoder_cache_embed_mismatch( + self, + request: Request, + input_id: int, + identifier: str, + cached_num_encoder_embeds: int, + requested_num_encoder_embeds: int, + ) -> bool: + if cached_num_encoder_embeds == requested_num_encoder_embeds: + return False + logger.warning( + "Encoder cache entry for multimodal identifier %r holds %d " + "embeddings, but request %s input %d expects %d. A client-provided " + "multimodal UUID may have been reused for different content.", + identifier, + cached_num_encoder_embeds, + request.request_id, + input_id, + requested_num_encoder_embeds, + ) + self.encoder_cache_mismatch_reqs.add(request.request_id) + return True + def _try_schedule_encoder_inputs( self, request: Request, @@ -1740,7 +1776,7 @@ class Scheduler(SchedulerInterface): # NOTE: since scheduler operates on the request level (possibly with # multiple encoder inputs per request), we need to create temporary # trackers for accounting at the encoder input level. - mm_hashes_to_schedule = set() + mm_hashes_to_schedule: dict[str, int] = {} num_embeds_to_schedule = 0 encoder_window_end = ( @@ -1782,10 +1818,35 @@ class Scheduler(SchedulerInterface): # We are not using the encoder cache for encoder-decoder models, # yet. if item_identifier in mm_hashes_to_schedule: + scheduled_num_encoder_embeds = mm_hashes_to_schedule[ + item_identifier + ] + if self._reject_on_encoder_cache_embed_mismatch( + request, + i, + item_identifier, + scheduled_num_encoder_embeds, + num_encoder_embeds, + ): + return [], 0, encoder_compute_budget, [] # The same encoder input has already been scheduled in the # current step. continue + cached_num_encoder_embeds = ( + self.encoder_cache_manager.get_cached_num_encoder_embeds(request, i) + ) + if cached_num_encoder_embeds is not None and ( + self._reject_on_encoder_cache_embed_mismatch( + request, + i, + item_identifier, + cached_num_encoder_embeds, + num_encoder_embeds, + ) + ): + return [], 0, encoder_compute_budget, [] + if self.encoder_cache_manager.check_and_update_cache(request, i): # The encoder input is already computed and cached from a # previous step. @@ -1845,14 +1906,14 @@ class Scheduler(SchedulerInterface): if self.ec_connector is not None and self.ec_connector.has_cache_item( item_identifier ): - mm_hashes_to_schedule.add(item_identifier) + mm_hashes_to_schedule[item_identifier] = num_encoder_embeds external_load_encoder_input.append(i) num_embeds_to_schedule += num_encoder_embeds continue num_embeds_to_schedule += num_encoder_embeds encoder_compute_budget -= num_encoder_embeds - mm_hashes_to_schedule.add(item_identifier) + mm_hashes_to_schedule[item_identifier] = num_encoder_embeds encoder_inputs_to_schedule.append(i) return ( @@ -2173,6 +2234,8 @@ class Scheduler(SchedulerInterface): error_req_ids = set(self.grammar_compile_error_reqs) self.grammar_compile_error_reqs.clear() + error_req_ids.update(self.encoder_cache_mismatch_reqs) + self.encoder_cache_mismatch_reqs.clear() if failed_kv_load_req_ids and not self.recompute_kv_load_failures: error_req_ids.update(failed_kv_load_req_ids) if self.ec_connector is not None: @@ -3207,8 +3270,8 @@ class Scheduler(SchedulerInterface): # invalid block IDs cannot be mapped back to requests. raise RuntimeError( "A KV connector reported block-level load failures " - "(invalid_block_ids) on a layout with multiple KV cache " - "groups, where block IDs are only unique within a group. " + "(invalid_block_ids) which is not supported for models " + "with multiple KV cache groups. " "Connectors must report failed requests via " "KVConnectorTransferResults.failed_recving instead." ) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 70d56c6543..7352f71cb6 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC): # managers have been validated by the coordinator. self.cache_hit_alignment_tokens = scheduler_block_size # The block size for this manager; used for actual block allocation. - self.block_size = kv_cache_spec.block_size - self.dcp_world_size = dcp_world_size - self.pcp_world_size = pcp_world_size - if dcp_world_size > 1: - self.block_size *= dcp_world_size + self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.block_size = kv_cache_spec.block_size * self.dcp_world_size self.kv_cache_spec = kv_cache_spec self.block_pool = block_pool self.enable_caching = enable_caching @@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager): self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs ) -> None: super().__init__(kv_cache_spec, block_pool, **kwargs) - # Mamba layers use TP instead of DCP, so each rank holds the full - # recurrent state. Undo the DCP/PCP block_size scaling that the base - # class applies for attention groups whose KV cache is partitioned. - self.block_size = kv_cache_spec.block_size self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks self.has_prefill_checkpoint_blocks = ( @@ -2078,7 +2072,7 @@ class MambaManager(SingleTypeKVCacheManager): latest_prompt_hash_boundary = ( request.num_prompt_tokens // hash_block_size ) * hash_block_size - if self.use_eagle: + if self.drop_eagle_checkpoint_block: # Eagle groups match one hash unit past the candidate and drop it, # so register the tail one unit lower. latest_prompt_hash_boundary = max( diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 7d71761990..34f65f3ea2 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -28,7 +28,7 @@ from vllm.exceptions import ( VLLMValidationError, ) from vllm.inputs import EngineInput, PromptType -from vllm.logger import init_logger +from vllm.logger import configure_logging_if_needed, init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.outputs import STREAM_FINISHED, PoolingRequestOutput, RequestOutput @@ -122,6 +122,8 @@ class AsyncLLM(EngineClient): None """ + configure_logging_if_needed(vllm_config.logging_config) + # Ensure we can serialize custom transformer configs maybe_register_config_serialize_by_value() diff --git a/vllm/v1/engine/coordinator.py b/vllm/v1/engine/coordinator.py index efadbfea26..cb120ba5d9 100644 --- a/vllm/v1/engine/coordinator.py +++ b/vllm/v1/engine/coordinator.py @@ -9,8 +9,8 @@ import weakref import msgspec.msgpack import zmq -from vllm.config import ParallelConfig -from vllm.logger import init_logger +from vllm.config import LoggingConfig, ParallelConfig +from vllm.logger import configure_logging, init_logger from vllm.utils.network_utils import make_zmq_socket from vllm.utils.system_utils import get_mp_context, set_process_title from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType @@ -77,7 +77,10 @@ class DPCoordinator: zmq_addr_pipe.close() def __init__( - self, parallel_config: ParallelConfig, enable_wave_coordination: bool = True + self, + parallel_config: ParallelConfig, + enable_wave_coordination: bool = True, + logging_config: LoggingConfig | None = None, ): dp_size = parallel_config.data_parallel_size assert dp_size > 1, "Coordinator only used for data parallel" @@ -108,6 +111,7 @@ class DPCoordinator: "back_publish_address": back_publish_address, "zmq_addr_pipe": child_zmq_addr_pipe, "enable_wave_coordination": enable_wave_coordination, + "logging_config": logging_config, }, daemon=True, ) @@ -167,7 +171,11 @@ class DPCoordinatorProc: zmq_addr_pipe=None, min_stats_update_interval_ms: int = 100, enable_wave_coordination: bool = True, + logging_config: LoggingConfig | None = None, ): + if logging_config is not None: + configure_logging(logging_config) + coordinator = DPCoordinatorProc( engine_count=engine_count, min_stats_update_interval_ms=min_stats_update_interval_ms, diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 9b27f50edc..b36caf092c 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -28,7 +28,7 @@ from vllm.distributed import ( stateless_destroy_torch_distributed_process_group, ) from vllm.envs import enable_envs_cache -from vllm.logger import init_logger +from vllm.logger import configure_logging, init_logger from vllm.logging_utils.dump_input import dump_engine_exception from vllm.lora.request import LoRARequest from vllm.multimodal.cache import ( @@ -1350,6 +1350,10 @@ class EngineCoreProc(EngineCore): @staticmethod def run_engine_core(*args, dp_rank: int = 0, local_dp_rank: int = 0, **kwargs): """Launch EngineCore busy loop in background process.""" + vllm_config: VllmConfig = kwargs["vllm_config"] + if logging_config := getattr(vllm_config, "logging_config", None): + configure_logging(logging_config) + # Ensure we can serialize transformer config after spawning maybe_register_config_serialize_by_value() @@ -1357,7 +1361,6 @@ class EngineCoreProc(EngineCore): signal_callback: SignalCallback | None = None clean_shutdown = False try: - vllm_config: VllmConfig = kwargs["vllm_config"] parallel_config: ParallelConfig = vllm_config.parallel_config data_parallel = parallel_config.data_parallel_size > 1 or dp_rank > 0 if data_parallel: @@ -2473,6 +2476,9 @@ class EngineCoreActorMixin: dp_rank: int = 0, local_dp_rank: int = 0, ): + if logging_config := getattr(vllm_config, "logging_config", None): + configure_logging(logging_config) + # Initialize tracer for distributed tracing if configured. maybe_init_worker_tracer( instrumenting_module_name="vllm.engine_core", diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 6ad05e1f8c..5d59bdeaea 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -366,6 +366,11 @@ class InprocClient(EngineCoreClient): log_stats, executor_fail_callback=executor_fail_callback, ) + # Release the executor's workers when the client is collected, as + # MPClient does; they hold GPU memory for the life of the process + # otherwise. The callback must not reference self, or self would stay + # reachable and the finalizer would never run. + self._finalizer = weakref.finalize(self, self.engine_core.shutdown) def get_output(self) -> EngineCoreOutputs: outputs, model_executed = self.engine_core.step_fn() @@ -384,7 +389,8 @@ class InprocClient(EngineCoreClient): self.engine_core.abort_requests(request_ids) def shutdown(self, timeout: float | None = None) -> None: - self.engine_core.shutdown() + if self._finalizer.detach() is not None: + self.engine_core.shutdown() def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None: self.engine_core.profile(is_start, profile_prefix) diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 5fe3bed4d3..369c31ca27 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -30,6 +30,7 @@ from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask from vllm.tokenizers import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid from vllm.utils.async_utils import make_async +from vllm.utils.diffusion import validate_diffusion_sampling_params from vllm.utils.jsontree import json_iter_leaves from vllm.v1.engine import EngineCoreRequest from vllm.v1.kv_hints import KvHintsEnvelope @@ -53,6 +54,7 @@ class InputProcessor: self.speculative_config = vllm_config.speculative_config self.structured_outputs_config = vllm_config.structured_outputs_config self.observability_config = vllm_config.observability_config + self.diffusion_config = vllm_config.diffusion_config # Load the custom logits processor classes once; the returned callable # runs their validate_params hooks per request at admission. self.validate_logits_processors_params = ( @@ -95,6 +97,9 @@ class InputProcessor: params validator for the active model runner.""" custom_logitsprocs = self.model_config.logits_processors if self.vllm_config.use_v2_model_runner: + if self.model_config.runner_type == "pooling" and not custom_logitsprocs: + return lambda _: None + from vllm.v1.worker.gpu.sample.logits_processor import ( build_custom_logits_processors_params_validator, ) @@ -129,6 +134,20 @@ class InputProcessor: self.validate_logits_processors_params(params) + if self.model_config.is_diffusion: + # Without --diffusion-config the served canvas is unknown here; + # the ids and the read-only normalisation are still checked. + validate_diffusion_sampling_params( + params, + canvas_length=( + self.diffusion_config.canvas_length + if self.diffusion_config is not None + else None + ), + vocab_size=self.model_config.get_vocab_size(), + async_scheduling=self.vllm_config.scheduler_config.async_scheduling, + ) + if self.model_config.return_sampling_mask: if params.temperature <= 0: raise ValueError( diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index c925ba90e8..73f6c18b34 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -16,7 +16,7 @@ from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.distributed.parallel_state import get_dp_group from vllm.engine.arg_utils import EngineArgs from vllm.inputs import EngineInput, PromptType -from vllm.logger import init_logger +from vllm.logger import configure_logging_if_needed, init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.outputs import PoolingRequestOutput, RequestOutput @@ -60,6 +60,7 @@ class LLMEngine: mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, multiprocess_mode: bool = False, ) -> None: + configure_logging_if_needed(vllm_config.logging_config) self.vllm_config = vllm_config self.model_config = vllm_config.model_config self.observability_config = vllm_config.observability_config diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index c1002ba1dc..02b49819fd 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -1128,6 +1128,7 @@ def launch_core_engines( coordinator = DPCoordinator( parallel_config, enable_wave_coordination=vllm_config.model_config.is_moe, + logging_config=vllm_config.logging_config, ) addresses.coordinator_input, addresses.coordinator_output = ( diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index dbaefe2186..78290d25ba 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -42,7 +42,7 @@ from vllm.distributed.parallel_state import ( model_parallel_is_initialized, ) from vllm.envs import enable_envs_cache -from vllm.logger import init_logger +from vllm.logger import configure_logging, init_logger from vllm.platforms import current_platform from vllm.tracing import instrument, maybe_init_worker_tracer from vllm.utils import numa_utils @@ -369,7 +369,9 @@ class MultiprocExecutor(Executor): def take_draft_token_ids(self) -> DraftTokenIds | None: # OPTIMIZATION: Get output only from a single worker (output_rank) return self.collective_rpc( - "take_draft_token_ids", unique_reply_rank=self.output_rank + "take_draft_token_ids", + unique_reply_rank=self.output_rank, + timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, ) def collective_rpc( # type: ignore[override] @@ -853,6 +855,9 @@ class WorkerProc: def worker_main(*args, **kwargs): """Worker initialization and execution loops. This runs a background process""" + if logging_config := getattr(kwargs["vllm_config"], "logging_config", None): + configure_logging(logging_config) + # Signal handler used for graceful termination. # SystemExit exception is only raised once to allow this and worker # processes to terminate without error diff --git a/vllm/v1/executor/ray_executor_v2.py b/vllm/v1/executor/ray_executor_v2.py index b8a2eec73f..8d254020ff 100644 --- a/vllm/v1/executor/ray_executor_v2.py +++ b/vllm/v1/executor/ray_executor_v2.py @@ -16,7 +16,7 @@ from vllm.distributed.device_communicators.shm_broadcast import ( Handle, MessageQueue, ) -from vllm.logger import init_logger +from vllm.logger import configure_logging, init_logger from vllm.platforms import current_platform from vllm.utils.network_utils import ( get_distributed_init_method, @@ -113,6 +113,9 @@ class RayWorkerProc(WorkerProc): is_driver_worker: bool, is_driver_node: bool = False, ): + if logging_config := getattr(vllm_config, "logging_config", None): + configure_logging(logging_config) + # Defer WorkerProc.__init__ until GPU IDs are known. self._is_driver_node = is_driver_node self._parallel_config = vllm_config.parallel_config diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 5100379083..4614e1aa32 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -21,6 +21,7 @@ from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import current_stream +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor @@ -46,20 +47,8 @@ def is_hisparse_decode_batch(attn_metadata: Any | None) -> bool: ) -def _get_max_decode_query_len(vllm_config: VllmConfig) -> int: - speculative_config = getattr(vllm_config, "speculative_config", None) - if ( - speculative_config is not None - and speculative_config.num_speculative_tokens is not None - ): - return 1 + speculative_config.num_speculative_tokens * ( - 2 if speculative_config.parallel_drafting else 1 - ) - return 1 - - def _get_max_swap_rows(vllm_config: VllmConfig) -> int: - max_query_len = _get_max_decode_query_len(vllm_config) + max_query_len = max_decode_query_len(vllm_config) scheduler_config = vllm_config.scheduler_config return min( scheduler_config.max_num_batched_tokens, @@ -120,7 +109,7 @@ class ResolvedHiSparseConfig: if config is None: return None - max_query_len = _get_max_decode_query_len(vllm_config) + max_query_len = max_decode_query_len(vllm_config) min_device_buffer_size = max_query_len * model_top_k # Retain the worst-case union of every speculative step's top-k and # leave one additional top-k of LRU slack. diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index fc8b88921a..5791704b2b 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -6,7 +6,7 @@ from __future__ import annotations import copy from collections import Counter from collections.abc import Collection, Sequence -from dataclasses import dataclass, fields, replace +from dataclasses import dataclass, field, fields, replace from enum import Enum, IntEnum from fractions import Fraction from functools import cached_property @@ -158,6 +158,15 @@ class KVCacheSpec: # number of tokens in a block block_size: int + dcp_sharded: bool = field(default=False, kw_only=True) + """Whether DCP shards this cache's token positions across ranks.""" + + block_stride_alignment: int | None = field(default=None, kw_only=True) + """Required byte alignment between physical blocks, including packed layers.""" + + def __post_init__(self): + if self.block_stride_alignment is not None and self.block_stride_alignment <= 0: + raise ValueError("block_stride_alignment must be positive") @property def prefix_cacheable(self) -> bool: @@ -474,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec): @dataclass(frozen=True, kw_only=True) class AttentionSpec(KVCacheSpec): + dcp_sharded: bool = True num_kv_heads: int head_size: int dtype: torch.dtype @@ -492,6 +502,7 @@ class AttentionSpec(KVCacheSpec): token (Whisper block pooling: ``Fraction(1, block_pool_size)``).""" def __post_init__(self): + super().__post_init__() if self.head_size_v is None: object.__setattr__(self, "head_size_v", self.head_size) @@ -528,7 +539,9 @@ class AttentionSpec(KVCacheSpec): def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: parallel_config = vllm_config.parallel_config - kv_shard_count = parallel_config.decode_context_parallel_size + kv_shard_count = ( + parallel_config.decode_context_parallel_size if self.dcp_sharded else 1 + ) return cdiv(max_len, self.block_size * kv_shard_count) @@ -560,7 +573,7 @@ class FullAttentionSpec(AttentionSpec): def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: max_model_len = vllm_config.model_config.max_model_len dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size - if dcp_world_size > 1: + if self.dcp_sharded and dcp_world_size > 1: max_model_len = cdiv(max_model_len, dcp_world_size) return cdiv(max_model_len, self.block_size) * self.page_size_bytes @@ -598,11 +611,13 @@ class FullAttentionSpec(AttentionSpec): ) merged_spec = cls( block_size=specs[0].block_size, + block_stride_alignment=specs[0].block_stride_alignment, num_kv_heads=specs[0].num_kv_heads, head_size=specs[0].head_size, head_size_v=specs[0].head_size_v, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -648,12 +663,6 @@ class MLAAttentionSpec(FullAttentionSpec): is_index_group_leader: bool = False storage_block_size: int | None = None """Token width used to view storage when it differs from the kernel block.""" - block_stride_alignment: int | None = None - """Required alignment, in bytes, of the distance between consecutive - blocks of this cache. In block-major layouts that distance is the whole - block (all layers' pages), so the allocator rounds the block up to it. - DeepGEMM's paged sparse MQA-logits kernels address pages as - ``base + page * stride`` and need it 512B-aligned.""" # Group capability enabled when any member flattens a non-causal query block # into decode rows. Runtime metadata still selects causal vs. non-causal mode. non_causal_multi_token_decode: bool = False @@ -695,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec): head_size=specs[0].head_size, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -751,11 +761,13 @@ class RSWASpec(FullAttentionSpec): base = FullAttentionSpec.merge(specs) # type: ignore[arg-type] return cls( block_size=base.block_size, + block_stride_alignment=base.block_stride_alignment, num_kv_heads=base.num_kv_heads, head_size=base.head_size, head_size_v=base.head_size_v, dtype=base.dtype, kv_quant_mode=base.kv_quant_mode, + dcp_sharded=base.dcp_sharded, page_size_padded=base.page_size_padded, num_head_slots=base.num_head_slots, state_content_bytes=base.state_content_bytes, @@ -846,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec): return cdiv(num_tokens, self.block_size) + 1 def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - assert vllm_config.parallel_config.decode_context_parallel_size == 1, ( - "DCP not support sliding window." - ) + assert ( + not self.dcp_sharded + or vllm_config.parallel_config.decode_context_parallel_size == 1 + ), "DCP not support sliding window." max_blocks = self.max_admission_blocks_per_request( max_in_flight_tokens=vllm_config.max_in_flight_tokens, max_model_len=vllm_config.model_config.max_model_len, @@ -945,6 +958,8 @@ class SlidingWindowMLASpec(SlidingWindowSpec): sliding_window_set = set(spec.sliding_window for spec in specs) extra_retained_set = set(spec.extra_retained_tokens for spec in specs) bounded_replay_set = set(spec.bounded_replay for spec in specs) + block_stride_alignment_set = {spec.block_stride_alignment for spec in specs} + assert len({spec.dcp_sharded for spec in specs}) == 1 assert ( len(cache_dtype_str_set) == 1 and len(tokens_per_state_set) == 1 @@ -952,6 +967,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): and len(sliding_window_set) == 1 and len(extra_retained_set) == 1 and len(bounded_replay_set) == 1 + and len(block_stride_alignment_set) == 1 ), ( "All attention layers in the same KV cache group must use the same " "quantization method, tokens per state, model version, sliding " @@ -959,9 +975,11 @@ class SlidingWindowMLASpec(SlidingWindowSpec): ) return cls( block_size=specs[0].block_size, + block_stride_alignment=block_stride_alignment_set.pop(), num_kv_heads=specs[0].num_kv_heads, head_size=specs[0].head_size, dtype=specs[0].dtype, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1175,8 +1193,10 @@ class SinkFullAttentionSpec(FullAttentionSpec): head_size=specs[0].head_size, head_size_v=specs[0].head_size_v, sink_len=specs[0].sink_len, + block_stride_alignment=specs[0].block_stride_alignment, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1209,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): kv_cache_specs: dict[str, KVCacheSpec] + def __post_init__(self): + super().__post_init__() + if self.kv_cache_specs: + object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded) + @property def prefix_cacheable(self) -> bool: return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values()) @@ -1262,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): if len(block_sizes) > 1: # Different block sizes, not uniform. return False + if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1: + return False first_spec = next(iter(kv_cache_specs.values())) return first_spec.is_uniform_with_collection(kv_cache_specs) diff --git a/vllm/v1/sample/logits_processor/__init__.py b/vllm/v1/sample/logits_processor/__init__.py index ff9a6428c5..703a0188f3 100644 --- a/vllm/v1/sample/logits_processor/__init__.py +++ b/vllm/v1/sample/logits_processor/__init__.py @@ -125,7 +125,13 @@ def _load_logitsprocs_by_fqcns( continue logger.debug("- Loading logits processor %s", logitproc) - module_path, qualname = logitproc.split(":") + parts = logitproc.split(":") + if len(parts) != 2: + raise ValueError( + f"Invalid logits processor FQCN {logitproc!r}. " + "Expected format: ':'" + ) + module_path, qualname = parts try: # Load module diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 2f051b6768..69ec368309 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -79,9 +79,9 @@ def flashinfer_sampler_supported() -> bool: Returns False (with appropriate logging) when ``VLLM_USE_FLASHINFER_SAMPLER`` is 0, when the platform isn't CUDA, when the GPU's compute capability is - unsupported, or when FlashInfer cannot JIT-compile for the current - GPU/CUDA toolchain. Raises ``RuntimeError`` if the user explicitly opted in - via the env var but FlashInfer is unavailable. + unsupported, when the GPU has 16 or fewer SMs, or when FlashInfer cannot + JIT-compile for the current GPU/CUDA toolchain. Raises ``RuntimeError`` if + the user explicitly opted in via the env var but FlashInfer is unavailable. Assumes flashinfer is installed, as guaranteed by ``requirements/cuda.txt``; otherwise importing the FlashInfer backend below raises ``ImportError``. @@ -107,6 +107,16 @@ def flashinfer_sampler_supported() -> bool: unsupported_reason = ( f"unsupported compute capability {capability.as_version_str()}" ) + elif ( + num_sms := current_platform.num_compute_units( + torch.accelerator.current_device_index() + ) + ) <= 16: + # FlashInfer 0.7+ rejects multi-CTA top-k masking on <=16 SMs because + # its cross-CTA software barrier cannot guarantee forward progress. + unsupported_reason = ( + f"top-k masking requires more than 16 SMs; device has {num_sms}" + ) else: unsupported_reason = _flashinfer_jit_unsupported_reason(capability) diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 96bfb35a67..f70afaa343 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field, fields, replace from enum import Enum, auto from typing import TYPE_CHECKING, Any @@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import ( from vllm.v1.core.kv_cache_utils import ( BlockHashWithGroupId, ExternalBlockHash, - dcp_world_size_for_kv_cache_spec, generate_block_hash_extra_keys, get_block_hash, get_group_id, make_block_hash_with_group_id, maybe_convert_block_hash, resolve_block_hashes, + resolve_dcp_kv_block_size, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( @@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, SimpleCPUOffloadWorkerMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, +) if TYPE_CHECKING: from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -118,7 +123,8 @@ class BoundaryStoreStats: symptom is a lower cache hit rate with nothing in the logs, so each decline reason is counted and exposed through ``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by - ``reset()``; not otherwise cleared. + ``reset()``, which carries any not-yet-drained deltas; not otherwise + cleared. """ published: int = 0 @@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler: hash_block_size: int, lazy_offload: bool = False, disk_capacity_bytes: int = 0, + use_page_cache: bool = False, ): self.vllm_config = vllm_config self.kv_cache_config = kv_cache_config @@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler: self.cpu_kv_cache_config.prefix_cacheable_group_ids ) self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU + # Must track kv_event_medium. + self._info_labelvalues: tuple[str, ...] = ( + "disk" if disk_capacity_bytes > 0 else "cpu", + str(use_page_cache).lower(), + str(lazy_offload).lower(), + str(self.num_cpu_blocks), + ) # Find the full attention kv group for prefix cache matching. self.fa_gidx = -1 for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups): @@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler: self._load_event_counter: int = 0 self._store_event_counter: int = 0 self.boundary_store_stats = BoundaryStoreStats() + # Interval stats state drained by get_stats() + self._boundary_stats_snapshot = BoundaryStoreStats() + self._interval_load_blocks_completed = 0 # For TP/PP: track partial store completions across steps. # Events must be reported by all world_size workers before considered complete. @@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler: target = 0 for g in kv_cache_config.prefix_cacheable_groups: spec = g.kv_cache_spec - # Only full attention is sharded across DCP ranks; replicated specs - # (mamba, sliding window, chunked-local) keep their own block size. - block_size = spec.block_size * dcp_world_size_for_kv_cache_spec( - spec, cp_world_size - ) + block_size = resolve_dcp_kv_block_size(spec, cp_world_size) if isinstance(spec, MambaSpec): target += 2 elif isinstance(spec, SlidingWindowSpec): @@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler: def get_boundary_store_stats(self) -> BoundaryStoreStats: return replace(self.boundary_store_stats) + def get_stats(self) -> SimpleCPUOffloadStats: + """Drain per-step stats for the connector's stats hooks.""" + stats = SimpleCPUOffloadStats() + + current = self.boundary_store_stats + for outcome, field_name in OUTCOME_TO_FIELD.items(): + delta = getattr(current, field_name) - getattr( + self._boundary_stats_snapshot, field_name + ) + if delta > 0: + stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,)) + self._boundary_stats_snapshot = replace(current) + + if self._interval_load_blocks_completed: + stats.increase_counter( + MetricName.LOAD_BLOCKS, + self._interval_load_blocks_completed, + ) + self._interval_load_blocks_completed = 0 + + stats.set_gauge( + MetricName.USED_BLOCKS, + self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(), + ) + pending = sum( + len(t.cpu_block_ids) + for t in ( + *self._store_event_to_blocks.values(), + *self._pending_finished_stores, + *self._abandoned_store_event_to_blocks.values(), + ) + ) + stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending) + stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues) + return stats + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler: """ # --- Load completions --- for req_id in list(connector_output.finished_recving or []): - self._cleanup_load_request(req_id) + completed_blocks = self._cleanup_load_request(req_id) + if completed_blocks: + self._interval_load_blocks_completed += completed_blocks # --- Store completions --- meta = connector_output.kv_connector_worker_meta @@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler: if blocks_to_free: self.cpu_block_pool.free_blocks(blocks_to_free) - def _cleanup_load_request(self, req_id: str) -> None: + def _cleanup_load_request(self, req_id: str) -> int: """Release all load resources for a request. Shared between request_finished() and update_connector_output() paths. Removes the request from _reqs_to_load, cleans up event mappings, and frees CPU/GPU touch refs. + + Returns the number of blocks in the load if it had been issued to + the worker, else 0 (never-issued loads are not counted as completed + by the caller). """ state = self._reqs_to_load.pop(req_id, None) if state is None: state = self._abandoned_reqs_to_load.pop(req_id, None) if state is None: - return + return 0 # Remove from load event mapping (only this req, not whole event) if state.load_event is not None: reqs = self._load_event_to_reqs.get(state.load_event) @@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler: for bid in state.transfer_meta.gpu_block_ids ) + if state.load_event is None: + return 0 + return len(state.transfer_meta.gpu_block_ids) + def _cleanup_store_request(self, req_id: str) -> None: """Release store metadata for a request. @@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler: if event_idx in self._abandoned_store_event_to_blocks } self._cursor = None - self.boundary_store_stats = BoundaryStoreStats() + # Seed the fresh counters with outcomes since the last drain so a + # mid-interval reset() does not drop them from the next report. + undrained = BoundaryStoreStats( + **{ + f.name: max( + 0, + getattr(self.boundary_store_stats, f.name) + - getattr(self._boundary_stats_snapshot, f.name), + ) + for f in fields(BoundaryStoreStats) + } + ) + self.boundary_store_stats = undrained + self._boundary_stats_snapshot = BoundaryStoreStats() # NOTE: _load_event_counter / _store_event_counter are not # reset as they are monotonic and must stay ahead of the workers # high-water marks to avoid event index collisions diff --git a/vllm/v1/simple_kv_offload/metrics.py b/vllm/v1/simple_kv_offload/metrics.py new file mode 100644 index 0000000000..9017d40cfd --- /dev/null +++ b/vllm/v1/simple_kv_offload/metrics.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Prometheus metrics for SimpleCPUOffloadConnector. + +The stats payload uses the same self-describing wire format as the +offloading connector, so it survives IPC serialization without the +metric classes on the receiving side. +""" + +from dataclasses import dataclass +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) + + +class MetricName: + """Metric names for SimpleCPUOffloadConnector.""" + + SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total" + LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total" + USED_BLOCKS = "vllm:simple_kv_offload_used_blocks" + PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks" + INFO = "vllm:simple_kv_offload_info" + + +# outcome label value -> BoundaryStoreStats field name. +# ``published`` is the sum of all outcomes and is not labeled. +OUTCOME_TO_FIELD = { + "stored": "stored", + "dropped_cpu_full": "dropped_cpu_full", + "dropped_request_gone": "dropped_request_gone", + "dropped_null_block": "dropped_null_block", + "dropped_not_hashed": "dropped_not_hashed", + "skipped_already_cached": "skipped_already_cached", + "skipped_in_flight": "skipped_in_flight", +} + +INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"] + + +class _MetricType: + """Type tags embedded in the serialized stats payload.""" + + COUNTER = "counter" + GAUGE = "gauge" + + +class _StatsKey: + """Top-level keys in the serialized stats dict.""" + + # Maps metric name -> _MetricType value + TYPES = "types" + # Maps metric name -> {label values tuple -> observed value} + DATA = "data" + + +@dataclass +class SimpleCPUOffloadStats(KVConnectorStats): + """Stats payload for SimpleCPUOffloadConnector. + + The ``data`` dict is structured as:: + + { + _StatsKey.TYPES: {name: _MetricType.*, ...}, + _StatsKey.DATA: {name: {labelvalues: value, ...}, ...}, + } + + Counter values are summed when aggregating, gauge values keep the + latest snapshot, and unlabeled metrics use ``()`` as their + labelvalues tuple. + """ + + def __post_init__(self): + if _StatsKey.DATA not in self.data: + self.reset() + + def reset(self): + self.data: dict[str, Any] = { + _StatsKey.TYPES: {}, + _StatsKey.DATA: {}, + } + + @property + def _types(self) -> dict[str, str]: + return self.data[_StatsKey.TYPES] + + @property + def _values(self) -> dict[str, Any]: + return self.data[_StatsKey.DATA] + + def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": + if other.is_empty(): + return self + assert isinstance(other, SimpleCPUOffloadStats) + for key, other_label_values in other._values.items(): + type_str = other._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + self._types.setdefault(key, type_str) + current_label_values = self._values.setdefault(key, {}) + for labelvalues, value in other_label_values.items(): + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + current_label_values[labelvalues] = ( + current_label_values.get(labelvalues, 0) + value + ) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + current_label_values[labelvalues] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return self + + def reduce(self) -> dict[str, int | float]: + """Reduce the interval observations to a flat summary for logging.""" + return_dict: dict[str, int | float] = {} + for key, label_value_map in self._values.items(): + if key == MetricName.INFO: + # Constant config fingerprint; exclude from the log summary. + continue + type_str = self._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + for labelvalues, value in label_value_map.items(): + key_with_labels = f"{key}:{labelvalues}" if labelvalues else key + if type_str in (_MetricType.COUNTER, _MetricType.GAUGE): + assert isinstance(value, int | float) + return_dict[key_with_labels] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return return_dict + + def is_empty(self) -> bool: + return not self.data.get(_StatsKey.DATA) + + def increase_counter( + self, + counter_name: str, + counter_increase_value: int | float = 1, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Increase a counter on the stats payload.""" + self._types.setdefault(counter_name, _MetricType.COUNTER) + counter_values = self._values.setdefault(counter_name, {}) + counter_values[labelvalues] = ( + counter_values.get(labelvalues, 0) + counter_increase_value + ) + + def set_gauge( + self, + gauge_name: str, + gauge_value: int | float, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Set a gauge snapshot on the stats payload.""" + self._types.setdefault(gauge_name, _MetricType.GAUGE) + gauge_values = self._values.setdefault(gauge_name, {}) + gauge_values[labelvalues] = gauge_value + + +class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics): + """Prometheus registration and recording for SimpleCPUOffloadConnector.""" + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + # (engine_idx, metric_name, labelvalues) -> bound child metric + self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {} + + self._counter_save_outcomes = self._counter_cls( + name="vllm:simple_kv_offload_save_outcomes_total", + documentation=( + "Store admission decisions in SimpleCPUOffloadConnector, by " + "outcome. Outcomes classify eager-mode boundary hand-off " + "stores; lazy-mode stores are not classified." + ), + labelnames=self._labelnames + ["outcome"], + ) + self._counter_load_blocks = self._counter_cls( + name="vllm:simple_kv_offload_load_blocks_total", + documentation=( + "KV blocks restored to GPU from the offload pool, counted " + "when a load finishes. This is prefill work avoided " + "through offload cache hits." + ), + labelnames=self._labelnames, + ) + self._gauge_used_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_used_blocks", + documentation=( + "Offload-pool blocks currently pinned by in-flight " + "transfers or cache hits (capacity minus free). Evictable " + "cached blocks are not counted; capacity_blocks is on " + "vllm:simple_kv_offload_info." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_pending_store_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_pending_store_blocks", + documentation=( + "Offload blocks in pending stores: queued for dispatch, " + "issued to workers, or awaiting release after a cache " + "reset. A persistently growing value indicates a stuck " + "transfer." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_info = self._gauge_cls( + name="vllm:simple_kv_offload_info", + documentation=( + "SimpleCPUOffloadConnector deployment facts. Value is " + "always 1. backend is cpu or disk; page_cache and " + "lazy_offload are true/false; capacity_blocks is the " + "offload pool size." + ), + labelnames=self._labelnames + INFO_LABELS, + multiprocess_mode="mostrecent", + ) + + self._defs: dict[str, PromMetricT] = { + MetricName.SAVE_OUTCOMES: self._counter_save_outcomes, + MetricName.LOAD_BLOCKS: self._counter_load_blocks, + MetricName.USED_BLOCKS: self._gauge_used_blocks, + MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks, + MetricName.INFO: self._gauge_info, + } + self._num_label_values: dict[str, int] = { + MetricName.SAVE_OUTCOMES: 1, + MetricName.LOAD_BLOCKS: 0, + MetricName.USED_BLOCKS: 0, + MetricName.PENDING_STORE_BLOCKS: 0, + MetricName.INFO: len(INFO_LABELS), + } + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """Observe transfer statistics.""" + metric_types = transfer_stats_data.get(_StatsKey.TYPES, {}) + metric_data = transfer_stats_data.get(_StatsKey.DATA, {}) + for key, label_value_map in metric_data.items(): + prom_metric = self._defs.get(key) + if prom_metric is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + type_str = metric_types.get(key) + for labelvalues, value in label_value_map.items(): + if len(labelvalues) != self._num_label_values[key]: + raise AssertionError( + f"Metric {key} expects " + f"{self._num_label_values[key]} labels, " + f"got {len(labelvalues)}" + ) + child = self._metrics.get((engine_idx, key, labelvalues)) + if child is None: + engine_labelvalues = self.per_engine_labelvalues[engine_idx] + child = prom_metric.labels( + *(engine_labelvalues + list(labelvalues)) + ) + self._metrics[(engine_idx, key, labelvalues)] = child + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + child.inc(value) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + child.set(value) + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index 9fc16014c2..c120951eed 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -67,7 +67,6 @@ class SimpleCPUOffloadWorker: # Metadata for the current step self._connector_metadata: SimpleCPUOffloadMetadata | None = None - self._store_submitted = False # Compute-done event recorded before each store; reused across steps # (get_finished runs once per step, copy queue is FIFO). @@ -246,15 +245,12 @@ class SimpleCPUOffloadWorker: def bind_connector_metadata(self, metadata: SimpleCPUOffloadMetadata) -> None: self._connector_metadata = metadata - self._store_submitted = False if metadata.load_event >= 0: self._pending_load_event_indices.add(metadata.load_event) if metadata.store_event >= 0: self._pending_store_event_indices.add(metadata.store_event) def clear_connector_metadata(self) -> None: - # No-forward steps skip the normal wait_for_save hook. - self.wait_for_save() self._connector_metadata = None def start_load_kv(self) -> None: @@ -283,11 +279,7 @@ class SimpleCPUOffloadWorker: #45704 for the bug and #39306 for the srcAccessOrder rationale. """ metadata = self._connector_metadata - if ( - metadata is not None - and metadata.store_gpu_blocks - and not self._store_submitted - ): + if metadata is not None and metadata.store_gpu_blocks: backend = self._backend assert backend is not None if self._store_compute_done is None: @@ -301,7 +293,6 @@ class SimpleCPUOffloadWorker: events_list=self._store_events, wait_event=self._store_compute_done, ) - self._store_submitted = True def get_finished( self, finished_req_ids: set[str] diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py index 08542f03a1..17be2732e6 100644 --- a/vllm/v1/spec_decode/draft_model.py +++ b/vllm/v1/spec_decode/draft_model.py @@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer): def _create_draft_vllm_config(self) -> VllmConfig: base = super()._create_draft_vllm_config() spec = self.speculative_config + target_parallel = base.parallel_config return replace( base, quant_config=None, parallel_config=replace( spec.draft_parallel_config, - rank=self.vllm_config.parallel_config.rank, + rank=target_parallel.rank, + decode_context_parallel_size=( + target_parallel.decode_context_parallel_size + ), + cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size, + dcp_comm_backend=target_parallel.dcp_comm_backend, + dcp_q_replicate=target_parallel.dcp_q_replicate, ), model_config=spec.draft_model_config, ) diff --git a/vllm/v1/spec_decode/vocab_mapping.py b/vllm/v1/spec_decode/vocab_mapping.py index 9a6bbe7cfa..dad059fa77 100644 --- a/vllm/v1/spec_decode/vocab_mapping.py +++ b/vllm/v1/spec_decode/vocab_mapping.py @@ -137,16 +137,12 @@ class VocabMapping: def map_target_to_draft_ids(self, target_ids): draft_ids = self.target_to_draft_ids[target_ids] # new tensor; no clone needed - missing = draft_ids == -1 - if missing.any(): - draft_ids[missing] = self.draft_unk_token_id + draft_ids[draft_ids == -1] = self.draft_unk_token_id return draft_ids.to(target_ids.dtype) def map_draft_to_target_ids(self, draft_ids): target_ids = self.draft_to_target_ids[draft_ids] # new tensor; no clone needed - missing = target_ids == -1 - if missing.any(): - target_ids[missing] = self.target_unk_token_id + target_ids[target_ids == -1] = self.target_unk_token_id return target_ids.to(draft_ids.dtype) def constrain_draft_logits(self, logits): diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 7fd311c873..3b35d39d14 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -389,11 +389,16 @@ class StructuredOutputManager: ) state_advancements = 0 seen_padding = False - failed = False + # Row filled from the last valid grammar state before a draft + # was rejected; later rows reuse it rather than unconstraining. + failed_index: int | None = None + bitmask = self._grammar_bitmask for i, token in enumerate(req_tokens): - apply_bitmask = ( - not failed and not seen_padding and i >= constraint_start - ) + if failed_index is not None: + bitmask[cumulative_index].copy_(bitmask[failed_index]) + cumulative_index += 1 + continue + apply_bitmask = not seen_padding and i >= constraint_start self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),)) if token == -1: seen_padding = True @@ -403,7 +408,7 @@ class StructuredOutputManager: ): state_advancements += 1 else: - failed = True + failed_index = cumulative_index logger.error( "Unexpected: grammar terminated or rejected draft " "token %s for request %s during bitmask fill.", @@ -415,12 +420,13 @@ class StructuredOutputManager: # Diffusion LLMs don't sample a bonus token after the # scheduled positions, so skip its bitmask in that case. if not (self.vllm_config.model_config.is_diffusion and req_tokens): - bonus_apply = ( - not failed - and not seen_padding - and constraint_start <= len(req_tokens) - ) - self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),)) + if failed_index is not None: + bitmask[cumulative_index].copy_(bitmask[failed_index]) + else: + bonus_apply = not seen_padding and constraint_start <= len( + req_tokens + ) + self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),)) cumulative_index += 1 if state_advancements > 0: diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index bb49fb189b..dab4f2aa04 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -97,6 +97,7 @@ class OutlinesBackend(StructuredOutputBackend): ) return OutlinesGrammar( vocab_size=self.vocab_size, + eos_token_id=self.vocabulary.inner.get_eos_token_id(), guide=oc.Guide(index, max_rollback=max_rollback_tokens), ) @@ -115,60 +116,94 @@ class OutlinesBackend(StructuredOutputBackend): @dataclass class OutlinesGrammar(StructuredOutputGrammar): vocab_size: int + eos_token_id: int guide: oc.Guide = field(hash=False) num_processed_tokens: int = field( default_factory=lambda: 0, repr=False, hash=False, init=False ) - # outlines_core signals done on DFA accept; vLLM expects done after EOS. - # We delay the finished flag by one step so EOS can still be emitted. - _prev_finished: bool = field(default=False, init=False, repr=False, hash=False) + # outlines_core signals done on DFA accept and never advances on EOS + # (its mask allows only EOS in a final state, but advance(EOS) fails), + # so accepting EOS in a final state is tracked here instead. + _is_terminated: bool = field(default=False, init=False, repr=False, hash=False) def accept_tokens(self, request_id: str, tokens: list[int]) -> bool: """Accepts a list of tokens and advances the FSM. - Returns True if the FSM was advanced successfully. - Returns False if the FSM failed to advance. + Returns True if all grammar-constrained tokens were accepted. + Tokens after termination (EOS) are ignored. Returns False if the FSM + failed to advance. """ - if self.guide.accepts_tokens(tokens): - # Advance can fail when the next state reached after advancing with - # the current tokens is a dead state. This is because Guide.accepts_tokens() - # only checks whether the current tokens can be accepted, - # whereas guide.advance() additionally checks the next state - # after all tokens are accepted. - # We need to be aware that the FSM must be prepared without dead states. - for t in tokens: - self.guide.advance(t) - self.num_processed_tokens += 1 + if self._is_terminated: return True - return False + eos_index = next( + (i for i, t in enumerate(tokens) if t == self.eos_token_id), None + ) + if eos_index is not None: + tokens = tokens[:eos_index] + # Advance can fail when the next state reached after advancing with + # the current tokens is a dead state. This is because Guide.accepts_tokens() + # only checks whether the current tokens can be accepted, + # whereas guide.advance() additionally checks the next state + # after all tokens are accepted. + # We need to be aware that the FSM must be prepared without dead states. + if tokens and not self.guide.accepts_tokens(tokens): + return False + for t in tokens: + self.guide.advance(t) + if eos_index is not None: + if not self.guide.is_finished(): + if tokens: + self.guide.rollback_state(len(tokens)) + return False + self._is_terminated = True + self.num_processed_tokens += len(tokens) + self._is_terminated + return True def rollback(self, num_tokens: int) -> None: - self.guide.rollback_state(num_tokens) + if num_tokens <= 0: + return self.num_processed_tokens -= num_tokens + if self._is_terminated: + self._is_terminated = False + num_tokens -= 1 + if num_tokens: + self.guide.rollback_state(num_tokens) def validate_tokens(self, tokens: list[int]) -> list[int]: + if self._is_terminated: + return [] accepted: list[int] = [] for tok in tokens: + if tok == self.eos_token_id: + if self._is_finished_after(accepted): + accepted.append(tok) + break accepted.append(tok) if not self.guide.accepts_tokens(accepted): accepted.pop() break return accepted + def _is_finished_after(self, tokens: list[int]) -> bool: + if not tokens: + return self.guide.is_finished() + for t in tokens: + self.guide.advance(t) + finished = self.guide.is_finished() + self.guide.rollback_state(len(tokens)) + return finished + def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None: mask = bitmask[idx] self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size()) def is_terminated(self) -> bool: - curr = self.guide.is_finished() - prev = self._prev_finished - self._prev_finished = curr - return prev + return self._is_terminated def reset(self): self.num_processed_tokens = 0 - self._prev_finished = False + self._is_terminated = False self.guide.reset() @@ -206,6 +241,11 @@ def validate_structured_output_request_outlines(params: SamplingParams): choices = [regex_escape(str(choice)) for choice in so_params.choice] regex = "(" + "|".join(choices) + ")" validate_regex_is_buildable(regex) + elif so_params.json_object: + raise VLLMValidationError( + "Outlines structured outputs backend " + "does not support json_object specifications" + ) elif so_params.grammar: raise VLLMValidationError( "Outlines structured outputs backend " diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 3c48ff64a0..5d2c2a4985 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -21,7 +21,6 @@ from vllm.v1.structured_output.backend_types import ( from vllm.v1.structured_output.utils import ( choice_as_grammar, compile_regex_with_timeout, - convert_lark_to_ebnf, grammar_is_likely_lark, ) @@ -91,7 +90,10 @@ class XgrammarBackend(StructuredOutputBackend): '{"type": "object"}', any_whitespace=not self.disable_any_whitespace ) elif request_type == StructuredOutputOptions.GRAMMAR: - ctx = self.compiler.compile_grammar(grammar_spec) + if grammar_is_likely_lark(grammar_spec): + ctx = self.compiler.compile_lark(grammar_spec) + else: + ctx = self.compiler.compile_grammar(grammar_spec) elif request_type == StructuredOutputOptions.REGEX: ctx = compile_regex_with_timeout( self.compiler.compile_regex, @@ -404,19 +406,13 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: return if so_params.grammar: - if grammar_is_likely_lark(so_params.grammar): - # xgrammar supports EBNF grammars only - try: - so_params.grammar = convert_lark_to_ebnf(so_params.grammar) - except ValueError as e: - raise VLLMValidationError( - "Failed to convert the grammar from Lark to EBNF. " - ) from e - - # Test parsing EBNF grammar, possibly already converted from Lark + # Parse the grammar with the same syntax `compile_grammar` will use, + # but don't compile it. The grammar is passed on unchanged. try: - # parse the grammar, but we aren't compiling it. - xgr.Grammar.from_ebnf(so_params.grammar) + if grammar_is_likely_lark(so_params.grammar): + xgr.Grammar.from_lark(so_params.grammar) + else: + xgr.Grammar.from_ebnf(so_params.grammar) except Exception as e: raise VLLMValidationError("Invalid grammar specification.") from e return diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 4bb438d6f3..5a0b95c17b 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -435,136 +435,6 @@ def grammar_is_likely_lark(grammar_str: str) -> bool: return True -def convert_lark_to_ebnf(grammar_str: str) -> str: - """Convert a Lark grammar string to EBNF format. - - EBNF reference: - https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - Lark grammar reference: - https://lark-parser.readthedocs.io/en/latest/grammar.html - - Args: - grammar_str: Input grammar in Lark format - - Returns: - str: Converted grammar in EBNF format - - Examples: - >>> print(convert_lark_to_ebnf("rule: 'hello'")) - root ::= rule - rule ::= "hello" - - """ - if not isinstance(grammar_str, str): - raise ValueError(f"Grammar must be a string, got {type(grammar_str)}") - if not grammar_str.strip(): - raise ValueError("Grammar string cannot be empty") - - defined_rules = set() - referenced_rules = set() - output_lines = [] - - def clean_line(line: str) -> str: - """Remove comments and whitespace from line.""" - return re.sub(r"(#|//).*$", "", line).strip() - - def check_quotes(text: str, rule_name: str, line_num: int) -> None: - """Validate quote matching in text.""" - if text.count("'") % 2 != 0 or text.count('"') % 2 != 0: - raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}") - - def extract_references(text: str) -> set[str]: - """Extract rule references from text.""" - # Remove quoted strings and special characters - text = re.sub(r'"[^"]*"', "", text) - text = re.sub(r"[+*?()|\[\]{}]", " ", text) - return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text)) - - # First pass: Find root rule and validate rule definitions - lines = [clean_line(line) for line in grammar_str.split("\n")] - first_rule = None - - for line_num, line in enumerate(lines, 1): - if not line or line.startswith("|"): - continue - - if ":" in line: - try: - name = line.split(":", 1)[0].strip().strip("?") - defined_rules.add(name) - if first_rule is None: - first_rule = name - if name == "start": - first_rule = "start" - except IndexError as e: - raise ValueError( - f"Invalid rule format on line {line_num}. " - "Expected 'rule_name: definition'" - ) from e - - if not defined_rules: - raise ValueError("No valid rules found in grammar") - - # Add root rule - output_lines.append(f"root ::= {first_rule}") - - # Second pass: Process rule definitions and alternatives - current_rule = None - current_definition = [] - - for line_num, line in enumerate(lines, 1): - if not line: - continue - - try: - if ":" in line and not line.startswith("|"): - # Save previous rule if exists - if current_rule: - output_lines.append( - f"{current_rule} ::= {' | '.join(current_definition)}" - ) - - # Process new rule - name, definition = line.split(":", 1) - current_rule = name.strip().strip("?") - - check_quotes(definition, f"rule '{current_rule}'", line_num) - definition = re.sub(r"'([^']*)'", r'"\1"', definition) - referenced_rules.update(extract_references(definition)) - current_definition = [definition.strip()] - - elif line.startswith("|"): - if not current_rule: - raise ValueError( - f"Alternative '|' on line {line_num} " - "without a preceding rule definition" - ) - - alt_def = line[1:].strip() - check_quotes( - alt_def, f"alternative for rule '{current_rule}'", line_num - ) - alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def) - referenced_rules.update(extract_references(alt_def)) - current_definition.append(alt_def) - - except ValueError as e: - raise ValueError(f"Error on line {line_num}: {str(e)}") from e - - # Add final rule if exists - if current_rule: - output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}") - - # Validate all rules are defined - undefined_rules = referenced_rules - defined_rules - {"root"} - if undefined_rules: - raise ValueError( - f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}" - ) - - return "\n".join(output_lines) - - def choice_as_grammar(choice: list[str]) -> str: def escape_ebnf_string(s: str) -> str: """Escape EBNF literals, including raw LF, CR, and NUL terminators.""" diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index a89ccbb9ed..81f4eaf842 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -520,12 +520,17 @@ def run_api_server_worker_proc( listen_address, sock, args, client_config=None, **uvicorn_kwargs ) -> None: """Entrypoint for individual API server worker processes.""" + if logging_config := getattr(args, "logging_config", None): + from vllm.logger import configure_logging + + configure_logging(logging_config) + from vllm.entrypoints.launchers.api_server.entry import run_server_worker client_config = client_config or {} server_index = client_config.get("client_index", 0) - # Set process title and add process-specific prefix to stdout and stderr. + # Set process title and process-specific log metadata. set_process_title("APIServer", str(server_index)) decorate_logs() diff --git a/vllm/v1/worker/cp_utils.py b/vllm/v1/worker/cp_utils.py index 79651759aa..bcf56dc0bf 100644 --- a/vllm/v1/worker/cp_utils.py +++ b/vllm/v1/worker/cp_utils.py @@ -40,6 +40,8 @@ def check_attention_cp_compatibility( layer_impl = getattr(layer, "impl", None) if layer_impl is None: continue + if not check_pcp and layer_impl.dcp_world_size == 1: + continue if vllm_config.speculative_config is not None and interleave_size > 1: assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, ( "MTP with cp_kv_cache_interleave_size > 1 is not " diff --git a/vllm/v1/worker/cpu_model_runner.py b/vllm/v1/worker/cpu_model_runner.py index 04ca2b9194..32fa01bd27 100644 --- a/vllm/v1/worker/cpu_model_runner.py +++ b/vllm/v1/worker/cpu_model_runner.py @@ -1,13 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import sys from contextlib import AbstractContextManager, contextmanager from typing import Any import torch import torch.nn as nn -import vllm.utils.cpu_triton_utils as cpu_tl from vllm.config import ( CompilationMode, VllmConfig, @@ -37,7 +35,6 @@ class CPUModelRunner(GPUModelRunner): self.cascade_attn_enabled = False self._postprocess_tensors() - self._postprocess_triton() def _postprocess_tensors(self) -> None: # Note: replace device tensors with cpu tensors @@ -62,64 +59,6 @@ class CPUModelRunner(GPUModelRunner): if isinstance(v, CpuGpuBuffer): v.gpu = v.cpu - def _postprocess_triton(self) -> None: - from vllm.triton_utils import HAS_TRITON - - if HAS_TRITON: - logger.info( - "Triton-CPU backend is available; skipping C++ monkey-patches " - "for Triton kernels." - ) - return - - import vllm.v1.worker.block_table - - vllm.v1.worker.block_table._COMPUTE_SLOT_MAPPING_KERNEL.kernel = ( - cpu_tl.compute_slot_mapping_kernel - ) - - # Speculative decoding fallbacks - import vllm.v1.sample.rejection_sampler - import vllm.v1.spec_decode.utils as spec_decode_utils - - spec_decode_utils._eagle_prepare_inputs_padded.kernel = ( - cpu_tl.eagle_prepare_inputs_padded_kernel - ) - spec_decode_utils._eagle_prepare_next_token_padded.kernel = ( - cpu_tl.eagle_prepare_next_token_padded_kernel - ) - spec_decode_utils._copy_and_expand_eagle_inputs.kernel = ( - cpu_tl.copy_and_expand_eagle_inputs_kernel - ) - spec_decode_utils.copy_and_expand_dflash_inputs_kernel = ( - cpu_tl.copy_and_expand_dflash_inputs_kernel - ) - dflash_module = sys.modules.get("vllm.v1.spec_decode.dflash") - if dflash_module is not None: - dflash_kernel_name = "copy_and_expand_dflash_inputs_kernel" - setattr( - dflash_module, - dflash_kernel_name, - cpu_tl.copy_and_expand_dflash_inputs_kernel, - ) - spec_decode_utils.eagle_step_slot_mapping_metadata_kernel = ( - cpu_tl.eagle_step_slot_mapping_metadata_kernel - ) - vllm.v1.sample.rejection_sampler.rejection_greedy_sample_kernel = ( - cpu_tl.rejection_greedy_sample_kernel - ) - vllm.v1.sample.rejection_sampler.rejection_random_sample_kernel = ( - cpu_tl.rejection_random_sample_kernel - ) - vllm.v1.sample.rejection_sampler.expand_kernel = cpu_tl.expand_kernel - vllm.v1.sample.rejection_sampler.sample_recovered_tokens_kernel = ( - cpu_tl.sample_recovered_tokens_kernel - ) - - import vllm.v1.worker.mamba_utils - - vllm.v1.worker.mamba_utils.batch_memcpy_kernel = cpu_tl.batch_memcpy_kernel - @instrument(span_name="Loading (CPU)") def load_model(self, load_dummy_weights: bool = False) -> None: if load_dummy_weights: diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index e8b4da8f36..423bcebb2d 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -328,6 +328,40 @@ def get_query_lens_mismatch_unsupported_backend( return None +def get_varlen_cudagraph_unsupported_backend( + attn_groups: list[list[AttentionGroup]], + vllm_config: VllmConfig, + max_query_len: int, + checked_layer_names: set[str] | None = None, +) -> tuple[str, int | None] | None: + """Name the first backend whose FULL cudagraphs cannot replay decode batches + of 1 to max_query_len tokens per request, with its bound, if any. + + ALWAYS builders replay any batch; the others need a bound of at least + max_query_len. See AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len(). + """ + for groups in attn_groups: + for group in groups: + if checked_layer_names is not None and checked_layer_names.isdisjoint( + group.layer_names + ): + continue + builder = group.get_metadata_builder(0) + spec = group.kv_cache_spec + support = builder.get_cudagraph_support(vllm_config, spec) + if support == AttentionCGSupport.ALWAYS: + continue + bound = ( + # NEVER means never, whatever an inherited override says. + None + if support == AttentionCGSupport.NEVER + else builder.get_varlen_cudagraph_max_query_len(vllm_config, spec) + ) + if bound is None or bound < max_query_len: + return group.backend.__name__, bound + return None + + def init_kv_cache( forward_context: dict[str, Any], kv_cache_config: KVCacheConfig, diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 6563e30c34..8cf3ba98dd 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -27,6 +27,7 @@ class BlockTables: cp_rank: int = 0, cp_interleave: int = 1, slot_mapping_enabled: list[bool] | None = None, + dcp_sharded: list[bool] | None = None, ): self.block_sizes = block_sizes self.kernel_block_sizes = kernel_block_sizes @@ -44,6 +45,10 @@ class BlockTables: slot_mapping_enabled = [True] * self.num_kv_cache_groups assert len(slot_mapping_enabled) == self.num_kv_cache_groups self._slot_mapping_enabled = slot_mapping_enabled + if dcp_sharded is None: + dcp_sharded = [True] * self.num_kv_cache_groups + assert len(dcp_sharded) == self.num_kv_cache_groups + self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device) self.blocks_per_kv_block = [ bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes) @@ -59,8 +64,7 @@ class BlockTables: self.block_tables.append(block_table) self.num_blocks = UvaBackedTensor( - (self.num_kv_cache_groups, self.max_num_reqs), - dtype=torch.int32, + (self.num_kv_cache_groups, self.max_num_reqs), dtype=torch.int32 ) self.fused_writer: FusedStagedWriter | None = None if self.num_kv_cache_groups > 1: @@ -211,6 +215,7 @@ class BlockTables: self.block_sizes_tensor, self.kernel_block_sizes_tensor, self.slot_mapping_enabled, + self.dcp_sharded, slot_mappings, slot_mappings.stride(0), self.cp_rank, @@ -284,6 +289,7 @@ def _compute_slot_mappings_kernel( block_sizes, # [num_kv_cache_groups] kernel_block_sizes, # [num_kv_cache_groups] slot_mapping_enabled, # [num_kv_cache_groups] + dcp_sharded, # [num_kv_cache_groups] slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens] slot_mappings_stride, cp_rank, @@ -313,6 +319,8 @@ def _compute_slot_mappings_kernel( kv_block_size = tl.load(block_sizes + group_id) kernel_block_size = tl.load(kernel_block_sizes + group_id) mapping_enabled = tl.load(slot_mapping_enabled + group_id) + if CP_SIZE != 1: + sharded = tl.load(dcp_sharded + group_id) req_state_idx = tl.load(idx_mapping + batch_idx) # idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns @@ -338,6 +346,8 @@ def _compute_slot_mappings_kernel( remainder = virtual_block_offsets % CP_INTERLEAVE local_offsets = rounds * CP_INTERLEAVE + remainder local_positions = virtual_block_indices * kv_block_size + local_offsets + local_positions = tl.where(sharded, local_positions, positions) + is_local = ~sharded | is_local block_indices = tl.where( mapping_enabled, local_positions // kernel_block_size, 0 diff --git a/vllm/v1/worker/gpu/cp_utils.py b/vllm/v1/worker/gpu/cp_utils.py index 470271a95b..1c906f4214 100644 --- a/vllm/v1/worker/gpu/cp_utils.py +++ b/vllm/v1/worker/gpu/cp_utils.py @@ -5,7 +5,7 @@ import torch from vllm.triton_utils import tl, triton -def maybe_prepare_dcp_local_seq_lens( +def prepare_dcp_local_seq_lens( dcp_local_seq_lens: torch.Tensor, seq_lens: torch.Tensor, num_reqs: int, @@ -16,8 +16,7 @@ def maybe_prepare_dcp_local_seq_lens( num_reqs_padded: int | None = None, ) -> torch.Tensor | None: """Populate caller-owned storage and return its padded view, or None without DCP.""" - if dcp_size == 1: - return None + assert dcp_size > 1 max_num_reqs = dcp_local_seq_lens.shape[0] BLOCK_SIZE = 128 diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index ddab7799d4..fa1dc4a04b 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -40,7 +40,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches @@ -69,7 +69,8 @@ class BatchExecutionDescriptor: num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs) uniform_token_count: int | None = None # Upper bound on per-request query length. Varlen decode graphs leave - # uniform_token_count unset, so this is what keeps a prefill batch out of one. + # uniform_token_count unset, so this is what keeps a prefill batch out of one: + # the runner passes None for any batch with a prefill. max_query_len: int | None = None num_active_loras: int = 0 # Number of microbatches the batch is split into (DBO). 1 means no splitting. @@ -643,8 +644,7 @@ class ModelCudaGraphManager(CudaGraphManager): self.intermediate_tensors[k][:num_tokens] = v def create_forward_fn( - desc: BatchExecutionDescriptor, - warmup: bool, + desc: BatchExecutionDescriptor, warmup: bool ) -> Callable[[CUDAGraphMode], None]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) @@ -705,7 +705,7 @@ class ModelCudaGraphManager(CudaGraphManager): attn_groups, kv_cache_config, full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, - max_query_len=desc.max_query_len, + max_query_len=desc.max_query_len or desc.uniform_token_count, pcp_manager=pcp_manager, ) @@ -776,6 +776,10 @@ def prepare_inputs_to_capture( max_query_len: int | None = None, pcp_manager: "PCPManager | None" = None, ) -> AttentionState: + if full_cudagraph and max_query_len is None: + # Mixed graphs can replay a single prefill spanning the entire batch, + # even when the dummy batch distributes one token to each request. + max_query_len = num_tokens input_batch = InputBatch.make_dummy( num_reqs, num_tokens, input_buffers, max_query_len=max_query_len ) @@ -789,15 +793,16 @@ def prepare_inputs_to_capture( slot_mappings, kv_cache_config ) - input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens( - input_buffers.dcp_local_seq_lens, - input_batch.seq_lens, - input_batch.num_reqs, - block_tables.cp_size, - block_tables.cp_rank, - block_tables.cp_interleave, - num_reqs_padded=input_batch.num_reqs_after_padding, - ) + if block_tables.cp_size > 1: + input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens( + input_buffers.dcp_local_seq_lens, + input_batch.seq_lens, + input_batch.num_reqs, + block_tables.cp_size, + block_tables.cp_rank, + block_tables.cp_interleave, + num_reqs_padded=input_batch.num_reqs_after_padding, + ) # NOTE(woosuk): Attention metadata is required not just by standard attention # kernels, but also by specialized attention-like operations (e.g., Inkling's sconv, diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index 8eabc3adbf..15db7c197c 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -243,7 +243,8 @@ def dispatch_cg_and_sync_dp( dp_size: Data-parallel world size. 1 skips all cross-rank work. dp_rank: This rank's index in the DP group. max_query_len: Upper bound on per-request query length, for selecting - varlen decode graphs. None means the graph must not constrain it. + varlen decode graphs. None, as for a batch with a prefill, keeps + the batch out of graphs that constrain it. need_eager: Force `CUDAGraphMode.NONE` instead of dispatching. num_active_loras: Active LoRA count for this rank. Does not need cross-rank agreement; it never changes a bucket's token count. diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py index c81fd7c11e..72f09f74e7 100644 --- a/vllm/v1/worker/gpu/ec_connector.py +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -29,10 +29,7 @@ class ECConnector: ) -> Generator[ECConnectorOutput | None, None, None]: yield None - def no_forward( - self, - scheduler_output: "SchedulerOutput", - ) -> ModelRunnerOutput: + def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput: return EMPTY_MODEL_RUNNER_OUTPUT @@ -84,10 +81,7 @@ class ActiveECConnector(ECConnector): output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta() ec_connector.clear_connector_metadata() - def no_forward( - self, - scheduler_output: "SchedulerOutput", - ) -> ModelRunnerOutput: + def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput: # EC send/recv even if no work to do. with self.maybe_get_output(scheduler_output) as ec_connector_output: pass @@ -99,8 +93,7 @@ NO_OP_EC_CONNECTOR = ECConnector() def get_ec_connector( - vllm_config: VllmConfig, - encoder_cache: "EncoderCache | None", + vllm_config: VllmConfig, encoder_cache: "EncoderCache | None" ) -> ECConnector: if ( not has_ec_transfer() diff --git a/vllm/v1/worker/gpu/eplb_utils.py b/vllm/v1/worker/gpu/eplb_utils.py index 24c0f0c4e1..d1bbd3338e 100644 --- a/vllm/v1/worker/gpu/eplb_utils.py +++ b/vllm/v1/worker/gpu/eplb_utils.py @@ -77,19 +77,13 @@ class EPLBController: assert speculative_config is not None assert speculative_config.draft_model_config is not None assert self.state is not None - self.state.add_model( - draft_moe_model, - speculative_config.draft_model_config, - ) + self.state.add_model(draft_moe_model, speculative_config.draft_model_config) speculator.set_eplb_state(self.state) self._has_registered_models = True return True def maybe_register_model( - self, - model: nn.Module, - model_config: Any, - load_dummy_weights: bool, + self, model: nn.Module, model_config: Any, load_dummy_weights: bool ) -> bool: if not self.parallel_config.enable_eplb or load_dummy_weights: return False @@ -110,11 +104,7 @@ class EPLBController: if eplb_models_added and self.state is not None and self.state.is_async: self.state.start_async_loop() - def step( - self, - is_dummy: bool = False, - is_profile: bool = False, - ) -> None: + def step(self, is_dummy: bool = False, is_profile: bool = False) -> None: if ( not self.parallel_config.enable_eplb or self.suppressed @@ -123,11 +113,8 @@ class EPLBController: ): return - self.state.step( - is_dummy, - is_profile, - log_stats=self.parallel_config.eplb_config.log_balancedness, - ) + log_stats = self.parallel_config.eplb_config.log_balancedness + self.state.step(is_dummy, is_profile, log_stats=log_stats) def prepare_forward( self, diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 32c083e459..75bbef5d8d 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -17,12 +17,7 @@ if TYPE_CHECKING: class InputBuffers: - def __init__( - self, - max_num_reqs: int, - max_num_tokens: int, - device: torch.device, - ): + def __init__(self, max_num_reqs: int, max_num_tokens: int, device: torch.device): self.max_num_reqs = max_num_reqs self.max_num_tokens = max_num_tokens self.device = device @@ -132,7 +127,7 @@ class InputBatch: req_ids = [f"req_{i}_{random_uuid()}" for i in range(num_reqs)] idx_mapping_np = np.arange(num_reqs, dtype=np.intp) - idx_mapping = torch.arange(num_reqs, dtype=torch.int64, device=device) + idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device) expanded_idx_mapping = idx_mapping expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device) @@ -474,11 +469,7 @@ def combine_sampled_and_draft_tokens( num_reqs = idx_mapping.shape[0] num_speculative_steps = draft_tokens.shape[-1] - logits_indices = torch.empty( - num_logits, - dtype=torch.int64, - device=input_ids.device, - ) + logits_indices = torch.empty(num_logits, dtype=torch.int64, device=input_ids.device) _combine_sampled_and_draft_tokens_kernel[(num_reqs,)]( input_ids, idx_mapping, @@ -711,9 +702,7 @@ def expand_idx_mapping( ) -> tuple[torch.Tensor, torch.Tensor]: num_reqs = idx_mapping.shape[0] expanded_idx_mapping = idx_mapping.new_empty(total_num_logits) - expanded_local_pos = torch.empty( - total_num_logits, dtype=torch.int32, device=idx_mapping.device - ) + expanded_local_pos = idx_mapping.new_empty(total_num_logits) _expand_idx_mapping_kernel[(num_reqs,)]( idx_mapping, expanded_idx_mapping, diff --git a/vllm/v1/worker/gpu/kv_connector.py b/vllm/v1/worker/gpu/kv_connector.py index 88b7b0cf9d..1ba75beae2 100644 --- a/vllm/v1/worker/gpu/kv_connector.py +++ b/vllm/v1/worker/gpu/kv_connector.py @@ -35,9 +35,7 @@ class KVConnector: def finish_forward(self) -> None: pass - def post_forward( - self, finished_req_ids: set[str], wait_for_save: bool = True - ) -> KVConnectorOutput | None: + def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None: return None def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput: @@ -97,9 +95,7 @@ class ActiveKVConnector(KVConnector): def reset_capture_state(self) -> None: self.kv_connector.reset_capture_state() - def post_forward( - self, finished_req_ids: set[str], wait_for_save: bool = True - ) -> KVConnectorOutput | None: + def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None: if self._disabled: return None @@ -107,8 +103,7 @@ class ActiveKVConnector(KVConnector): self._start_load_kv() output = KVConnectorOutput() - if wait_for_save: - self.kv_connector.wait_for_save() + self.kv_connector.wait_for_save() transfer_results = self.kv_connector.get_transfer_results(finished_req_ids) output.finished_sending = transfer_results.finished_sending or None output.finished_recving = transfer_results.finished_recving or None @@ -129,7 +124,7 @@ class ActiveKVConnector(KVConnector): self.pre_forward(scheduler_output) self.finish_forward() finished_req_ids = scheduler_output.finished_req_ids - kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False) + kv_connector_output = self.post_forward(finished_req_ids) return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) def set_disabled(self, disabled: bool) -> None: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a5e5ac2497..9f25425ab6 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -21,6 +21,7 @@ import gc import time from contextlib import AbstractContextManager from copy import deepcopy +from dataclasses import replace from typing import Any, NamedTuple import numpy as np @@ -36,10 +37,7 @@ from vllm.distributed.aux_output_connector.worker import ( AuxOutputWorkerConnector, get_aux_output_connector, ) -from vllm.distributed.parallel_state import ( - get_dcp_group, - get_pp_group, -) +from vllm.distributed.parallel_state import get_dcp_group, get_pp_group from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager @@ -58,6 +56,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.encoder_budget import ( MultiModalBudget, ) +from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture @@ -97,7 +96,7 @@ from vllm.v1.worker.gpu.attn_utils import ( ) from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.buffer_utils import set_default_max_concurrency -from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, ModelCudaGraphManager, @@ -442,6 +441,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.vllm_config, self.req_states, self.is_pooling_model, config_processors ) if self.is_last_pp_rank and not self.is_pooling_model: + # V2 bypasses TopKTopPSampler, which registers native warmups. + # ROCm only: on CUDA these warmups add ~2 min to every engine start. + if current_platform.is_rocm(): + from vllm.v1.sample.ops.topk_topp_sampler import ( + register_top_k_top_p_warmups, + ) + + with self.jit_warmup_registry.activate(): + register_top_k_top_p_warmups() + sampler_kwargs: dict[str, Any] = { "vllm_config": self.vllm_config, "max_num_reqs": self.max_num_reqs, @@ -485,8 +494,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): ), ) self.prompt_logprobs_worker = PromptLogprobsWorker( - self.max_num_reqs, - logprobs_mode=self.model_config.logprobs_mode, + self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode ) self.structured_outputs_worker = StructuredOutputsWorker( max_num_logits=self.max_num_reqs * self.decode_query_len, @@ -499,9 +507,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.is_pooling_model and self.is_last_pp_rank: self.pooling_runner = PoolingRunner(self.model, self.vllm_config) eplb_models_added |= self.eplb.maybe_register_model( - self.model, - self.model_config, - load_dummy_weights, + self.model, self.model_config, load_dummy_weights ) self.eplb.maybe_start_async_loop(eplb_models_added) @@ -556,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): return encoder_runner.get_encoder_timing_stats() def get_kv_cache_spec(self): - return get_kv_cache_spec(self.vllm_config) + kv_cache_spec = get_kv_cache_spec(self.vllm_config) + # A draft running at DCP=1 under a DCP target keeps a replicated cache. + if ( + isinstance(self.speculator, DraftModelSpeculator) + and self.speculator.dcp_size == 1 + and self.dcp_size > 1 + ): + for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys(): + kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False) + return kv_cache_spec def initialize_kv_cache( self, @@ -582,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): block_sizes = [] max_num_blocks_per_group = [] slot_mapping_enabled = [] + dcp_sharded = [] for kv_cache_group in kv_cache_config.kv_cache_groups: spec = kv_cache_group.kv_cache_spec block_sizes.append(spec.block_size) @@ -589,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec ) slot_mapping_enabled.append(layer_spec.uses_slot_mapping) + dcp_sharded.append(spec.dcp_sharded) # Let each cache type account for CP. Attention KV is DCP-sharded, # while Mamba/GDN recurrent state is replicated across DCP ranks. max_num_blocks = spec.max_num_blocks_per_req( @@ -635,7 +652,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.speculator, "enable_adaptive_verification", False ), attn_groups=self.attn_groups, - attn_cg_support=attn_cg_support, req_states=self.req_states, query_start_loc=self.input_buffers.query_start_loc, num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step, @@ -653,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): device=self.device, kernel_block_sizes=self.kernel_block_sizes, slot_mapping_enabled=slot_mapping_enabled, + dcp_sharded=dcp_sharded, cp_size=self.dcp_size, cp_rank=self.dcp_rank, cp_interleave=self.cp_interleave, @@ -792,8 +809,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): assert num_tokens % self.decode_query_len == 0 elif self.speculator is not None: num_reqs = min( - num_reqs, - self.max_num_tokens // self.speculator.num_query_per_req, + num_reqs, self.max_num_tokens // self.speculator.num_query_per_req ) # Distribute the remainder evenly so no dummy request exceeds # ceil(num_tokens / num_reqs) <= max_model_len tokens. @@ -860,14 +876,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.step_timing.drafter_start() mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None if self.speculator.supports_mm_inputs: - mm_inputs = ( - [], - torch.zeros( - input_batch.num_tokens, - dtype=torch.bool, - device="cpu", - ), + all_false = torch.zeros( + input_batch.num_tokens, dtype=torch.bool, device="cpu" ) + mm_inputs = [], all_false # Let the target override the hidden state fed to the drafter # (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The @@ -936,9 +948,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): mm_config = self.model_config.multimodal_config if mm_config is not None and not mm_config.skip_mm_profiling: mm_budget = MultiModalBudget( - self.vllm_config, - self.mm_registry, - enable_cache=False, + self.vllm_config, self.mm_registry, enable_cache=False ) dummy_mm_inputs = mm_budget.get_dummy_encoder_profile_inputs() self.model_state.encoder_runner.profile_encoder_cache( @@ -1258,14 +1268,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs, num_toks, max_query_len, batch_state.has_prefill ) - def _prepare_padding_mask( - self, num_tokens: int, num_tokens_after_padding: int - ) -> torch.Tensor: - is_padding = self.input_buffers.is_padding[:num_tokens_after_padding] - is_padding[:num_tokens].fill_(False) - is_padding[num_tokens:].fill_(True) - return is_padding - def prepare_inputs( self, scheduler_output: SchedulerOutput, @@ -1278,14 +1280,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): assert num_tokens > 0 is_padding = self.input_buffers.is_padding[:num_tokens_after_padding] if envs.VLLM_MOE_SKIP_PADDING: - is_padding = self._prepare_padding_mask( - num_tokens, num_tokens_after_padding - ) + is_padding[:num_tokens].fill_(False) + is_padding[num_tokens:].fill_(True) req_ids = batch_req_state.req_ids num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens idx_mapping_np = batch_req_state.idx_mapping_np - idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device) + idx_mapping = async_tensor_h2d( + idx_mapping_np, device=self.device, dtype=torch.int32 + ) num_reqs = len(req_ids) # Get the number of draft tokens for each request. @@ -1462,11 +1465,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): else None ), ) - input_batch = pcp.maybe_partition_pcp_batch( - self.pcp_manager, - input_batch, - batch_desc, - ) + if self.pcp_manager is not None: + input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc) return input_batch def prepare_attn( @@ -1477,8 +1477,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Block tables: num_kv_cache_groups x [num_reqs_padded, max_num_blocks]. block_tables = self.block_tables.gather_block_tables( - input_batch.idx_mapping, - num_reqs_padded=input_batch.num_reqs_after_padding, + input_batch.idx_mapping, num_reqs_padded=input_batch.num_reqs_after_padding ) # Slot mappings: [num_kv_cache_groups, num_tokens_padded]. # Kernel pads beyond num_tokens with PAD_SLOT_ID. @@ -1497,15 +1496,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): block_tables = block_table_provider.get_dummy_block_tables(input_batch.num_reqs) if valid_state_slots: state_slots = torch.arange( - 1, - input_batch.num_reqs + 1, - dtype=torch.int32, - device=self.device, + 1, input_batch.num_reqs + 1, dtype=torch.int32, device=self.device ) for block_table in block_tables: block_table[:, 0].copy_(state_slots) - slot_mappings = pcp.maybe_get_pcp_dummy_slot_mappings( - self.pcp_manager, self.block_tables, input_batch.num_tokens + slot_mappings = block_table_provider.get_dummy_slot_mappings( + input_batch.num_tokens ) return block_tables, slot_mappings @@ -1598,11 +1594,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): query_start_loc: torch.Tensor | None = None, ) -> None: # Update the number of computed tokens. + output_bin_counts = None if self.is_last_pp_rank: assert self.sampler is not None output_bin_counts = self.sampler.penalties_state.output_bin_counts - else: - output_bin_counts = None post_update( idx_mapping, self.req_states.num_computed_tokens.gpu, @@ -1663,12 +1658,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get batch descriptor and sync across DP ranks. num_reqs = len(scheduler_output.num_scheduled_tokens) num_toks = scheduler_output.total_num_scheduled_tokens - max_query_len = max(scheduler_output.num_scheduled_tokens.values()) + max_query_len: int | None = max(scheduler_output.num_scheduled_tokens.values()) batch_req_state, uniform_tok_count = self.gather_batch_req_state( scheduler_output, dummy_run ) if batch_req_state is not None: num_toks = batch_req_state.num_tokens + if batch_req_state.has_prefill: + # Varlen decode graphs replay decodes only, and their bound + # alone would admit a short prefill. + max_query_len = None if self.pcp_manager is not None: num_toks = self.pcp_manager.get_num_tokens_for_dispatch( batch_req_state.num_scheduled_tokens, @@ -1760,32 +1759,27 @@ class GPUModelRunner(LoRAModelRunnerMixin): "Attention metadata must be prepared for dummy runs when using " "FULL cudagraph mode." ) - block_tables = None - slot_mappings = None + block_tables, slot_mappings = None, None else: block_tables, slot_mappings = self.prepare_dummy_attn( input_batch, valid_dummy_state_slots ) - if not skip_attn_for_dummy_run and context_len: - assert block_tables is not None - set_dummy_context( - input_batch, - self.block_tables, - context_len, - self.kv_cache_config.num_blocks, - self.max_model_len, - input_block_tables=block_tables, - ) + if context_len: + set_dummy_context( + input_batch, + self.block_tables, + context_len, + self.kv_cache_config.num_blocks, + self.max_model_len, + input_block_tables=block_tables, + ) - attn_metadata = None - slot_mappings_by_layer = None - if not (dummy_run and skip_attn_for_dummy_run): - input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens( - ( - self.pcp_manager.input_buffers - if self.pcp_manager is not None - else self.input_buffers - ).dcp_local_seq_lens, + if self.dcp_size > 1 and not (dummy_run and skip_attn_for_dummy_run): + input_buffers = self.input_buffers + if self.pcp_manager is not None: + input_buffers = self.pcp_manager.input_buffers + input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens( + input_buffers.dcp_local_seq_lens, input_batch.seq_lens, input_batch.num_reqs, self.dcp_size, @@ -1793,6 +1787,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.cp_interleave, num_reqs_padded=input_batch.num_reqs_after_padding, ) + + attn_metadata = None + slot_mappings_by_layer = None ubatch_state: UBatchState | None = None if batch_desc.num_ubatches > 1: assert self.ubatch_runner is not None @@ -2037,16 +2034,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output) # Last rank: sample tokens + assert hidden_states is not None draft_hidden_states = hidden_states - assert draft_hidden_states is not None - hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling( - self.pcp_manager, hidden_states, input_batch - ) - if self.pcp_manager is not None and aux_hidden_states is not None: - aux_hidden_states = [ - self.pcp_manager.restore_hidden_states(states) - for states in aux_hidden_states - ] + if self.pcp_manager is not None: + hidden_states, aux_hidden_states, input_batch = ( + self.pcp_manager.restore_for_sampling(hidden_states, aux_hidden_states) + ) sampler_output, num_sampled, num_rejected = self.sample( hidden_states, input_batch, grammar_output @@ -2248,6 +2241,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): torch.accelerator.synchronize() if self.aux_output_connector is not None: self.aux_output_connector.close() + set_offloader(None) self.cudagraph_manager = None self.fast_prefill = None self.pooling_runner = None @@ -2295,13 +2289,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.eplb.suppressed = suppressed def setup_eplb_from_mapping( - self, - expanded_physical_to_logical: torch.Tensor, + self, expanded_physical_to_logical: torch.Tensor ) -> None: self.eplb.setup_from_mapping( - self.model, - self.model_config, - expanded_physical_to_logical, + self.model, self.model_config, expanded_physical_to_logical ) ########### EPLB methods end ########### diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index a61384ed7f..bebb9cd41b 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -27,9 +27,7 @@ class EncoderDecoderAttnMetadata(ModelSpecificAttnMetadata): encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] def get_extra_common_attn_kwargs( - self, - kv_cache_group_id: int, - num_reqs: int, + self, kv_cache_group_id: int, num_reqs: int ) -> dict[str, Any]: encoder_seq_lens = self.encoder_seq_lens.get(kv_cache_group_id) if encoder_seq_lens is None: diff --git a/vllm/v1/worker/gpu/model_states/encoder_only.py b/vllm/v1/worker/gpu/model_states/encoder_only.py index 19c97e2658..1b23723fdc 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_only.py +++ b/vllm/v1/worker/gpu/model_states/encoder_only.py @@ -188,10 +188,7 @@ class EncoderOnlyModelState(DefaultModelState): return attn_metadata def _build_encoder_attn_metadata( - self, - input_batch: InputBatch, - cudagraph_mode: CUDAGraphMode, - for_capture: bool, + self, input_batch: InputBatch, cudagraph_mode: CUDAGraphMode, for_capture: bool ) -> dict[str, Any]: if cudagraph_mode == CUDAGraphMode.FULL: num_reqs = input_batch.num_reqs_after_padding diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index a5c8d6f4cf..ee3e519a6e 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -28,16 +28,12 @@ class ModelSpecificAttnMetadata: """Base class for model-specific attention metadata.""" def get_extra_common_attn_kwargs( - self, - kv_cache_group_id: int, - num_reqs: int, + self, kv_cache_group_id: int, num_reqs: int ) -> dict[str, Any]: return {} def get_extra_attn_kwargs( - self, - attn_metadata_builder: Any, - num_reqs: int, + self, attn_metadata_builder: Any, num_reqs: int ) -> dict[str, Any]: return {} @@ -59,7 +55,6 @@ class ModelState(ABC): self.model = model self.device = device - self.max_model_len = self.model_config.max_model_len self.max_num_reqs = self.scheduler_config.max_num_seqs self.max_num_tokens = self.scheduler_config.max_num_batched_tokens self.inputs_embeds_size = self.model_config.get_inputs_embeds_size() @@ -99,6 +94,11 @@ class ModelState(ABC): ), ) + @property + def max_model_len(self) -> int: + # Auto-fit can reduce the limit after model state initialization. + return self.model_config.max_model_len + def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]: from vllm.model_executor.models.interfaces import ( supports_realtime, diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index bfad8a1b73..2b73b55bac 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -43,16 +43,12 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): num_decode_draft_tokens_cpu: torch.Tensor | None = None def get_extra_common_attn_kwargs( - self, - kv_cache_group_id: int, - num_reqs: int, + self, kv_cache_group_id: int, num_reqs: int ) -> dict[str, Any]: return {"is_prefilling": self.is_prefilling[:num_reqs]} def get_extra_attn_kwargs( - self, - attn_metadata_builder: Any, - num_reqs: int, + self, attn_metadata_builder: Any, num_reqs: int ) -> dict[str, Any]: if not isinstance( attn_metadata_builder, @@ -250,7 +246,11 @@ class MambaHybridModelState(DefaultModelState): num_reqs = input_batch.num_reqs num_tokens = input_batch.num_tokens query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) - max_query_len = input_batch.num_scheduled_tokens.max().item() + # Prefer the promised bound: a capture dummy's measured max is its even + # split, not the length the graph must replay. + max_query_len = input_batch.max_query_len + if max_query_len is None: + max_query_len = input_batch.num_scheduled_tokens.max().item() seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound if for_capture: # Capture with worst-case max_seq_len so the graph is valid at any replay. @@ -278,10 +278,18 @@ class MambaHybridModelState(DefaultModelState): num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32) num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req if num_draft_tokens_per_req is not None: - # A row is a spec-decode row only when its whole prompt is already - # computed, i.e. exactly one non-draft (decode) token is scheduled. - is_decode = ( - input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1 + # Test request state, not num_scheduled_tokens == draft_count+1: + # adaptive rewrites num_scheduled_tokens to an even split, so that + # equality rarely holds and would demote every verify row to decode. + # A one-token prompt tail over prior state that the scheduler padded + # with placeholder drafts is also a spec-decode row: the prefill + # kernels can't roll the placeholders back. + num_computed = input_batch.num_computed_prefill_tokens_np + is_prompt_tail = (num_computed > 0) & ( + input_batch.prefill_len_np - num_computed == 1 + ) + is_decode = (~input_batch.is_prefilling_np | is_prompt_tail) & ( + input_batch.num_scheduled_tokens > 0 ) spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where( @@ -333,9 +341,7 @@ class MambaHybridModelState(DefaultModelState): ) if self.recoverssm is not None: self.recoverssm.record_step( - attn_metadata, - attn_groups, - for_capture=for_capture, + attn_metadata, attn_groups, for_capture=for_capture ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/prompt_embeds.py b/vllm/v1/worker/gpu/model_states/prompt_embeds.py index 251558f9c1..b9c82bb849 100644 --- a/vllm/v1/worker/gpu/model_states/prompt_embeds.py +++ b/vllm/v1/worker/gpu/model_states/prompt_embeds.py @@ -37,8 +37,8 @@ class PromptEmbedsState: # Indexed by req_state index. Stale entries after removal are # harmless: add_request rewrites all fields for every index it claims. - self.embeds_ptrs = UvaBackedTensor(max_num_reqs, dtype=torch.int64) - self.mask_ptrs = UvaBackedTensor(max_num_reqs, dtype=torch.int64) + self.embeds_ptrs = UvaBackedTensor(max_num_reqs, dtype=torch.uint64) + self.mask_ptrs = UvaBackedTensor(max_num_reqs, dtype=torch.uint64) self.embeds_lens = UvaBackedTensor(max_num_reqs, dtype=torch.int32) def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: @@ -100,8 +100,8 @@ class PromptEmbedsState: def _apply_prompt_embeds_kernel( inputs_embeds_ptr, inputs_embeds_stride, - embeds_ptrs_ptr, # int64 [max_num_reqs], device pointers (0-len = unused) - mask_ptrs_ptr, # int64 [max_num_reqs], 0 = no is-token-ids mask + embeds_ptrs_ptr, # uint64 [max_num_reqs], device pointers (0-len = unused) + mask_ptrs_ptr, # uint64 [max_num_reqs], 0 = no is-token-ids mask embeds_lens_ptr, # int32 [max_num_reqs] idx_mapping_ptr, query_start_loc_ptr, diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index 376e0f2510..de49fcffb9 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -145,7 +145,10 @@ class PCPManager: if speculative_config is not None: if speculative_config.use_dspark(): dcp_size = parallel_config.decode_context_parallel_size - if dcp_size not in (1, pcp_size): + if ( + dcp_size not in (1, pcp_size) + and speculative_config.draft_model_config.use_mla + ): raise NotImplementedError( "MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got " f"DCP={dcp_size}, PCP={pcp_size}." @@ -192,8 +195,7 @@ class PCPManager: @staticmethod def _reorder_segments( - segments: list[RankSegment], - is_prefilling: np.ndarray, + segments: list[RankSegment], is_prefilling: np.ndarray ) -> list[RankSegment]: """Order this rank's rows decodes-first, then prefills, canonically.""" @@ -218,9 +220,7 @@ class PCPManager: return segments def replicated_requests( - self, - num_scheduled_tokens: np.ndarray, - is_prefilling: np.ndarray, + self, num_scheduled_tokens: np.ndarray, is_prefilling: np.ndarray ) -> np.ndarray: """Per global request, whether every PCP rank gets the whole query.""" num_chunks = 2 * self.pcp_world_size @@ -407,10 +407,7 @@ class PCPManager: return self._input_buffers def partition_batch( - self, - input_batch: InputBatch, - padded_num_tokens: int | None = None, - padded_num_reqs: int | None = None, + self, input_batch: InputBatch, batch_desc: "BatchExecutionDescriptor" ) -> InputBatch: assert self._input_buffers is not None input_buffers = self._input_buffers @@ -422,6 +419,13 @@ class PCPManager: num_computed_tokens = global_batch.num_computed_tokens_np is_prefilling = global_batch.is_prefilling_np + padded_num_tokens = None + padded_num_reqs = None + if batch_desc.cg_mode != CUDAGraphMode.NONE: + padded_num_tokens = batch_desc.num_tokens + if batch_desc.cg_mode == CUDAGraphMode.FULL: + padded_num_reqs = batch_desc.num_reqs + segments_by_rank, per_rank_num_tokens = self._build_batch_layout( num_scheduled_tokens, num_computed_tokens, @@ -442,9 +446,7 @@ class PCPManager: num_local_reqs = len(local_segments) num_reqs_after_padding = self._resolve_num_reqs_after_padding( - global_batch, - padded_num_reqs, - num_local_reqs, + global_batch, padded_num_reqs, num_local_reqs ) if num_reqs_after_padding > input_buffers.max_num_reqs: raise RuntimeError( @@ -714,8 +716,7 @@ class PCPManager: return self._gathered_kv_slot_mappings[:, : num_tokens * self.pcp_world_size] def _convert_to_gathered_slot_mappings( - self, - global_batch_slot_mappings: torch.Tensor, + self, global_batch_slot_mappings: torch.Tensor ) -> torch.Tensor: assert self._padded_gather_idx is not None assert self._gathered_kv_write_mask is not None @@ -754,10 +755,9 @@ class PCPManager: return self.draft_prefill_batch or input_buffers def prepare_draft_prefill( - self, - input_batch: InputBatch, - input_ids: torch.Tensor, + self, input_batch: InputBatch, input_ids: torch.Tensor ) -> None: + self.draft_prefill_batch = None if input_batch is not self._global_batch or self._local_batch is None: return local_batch = self._local_batch @@ -789,54 +789,15 @@ class PCPManager: return last_hidden_states, hidden_states def restore_for_sampling( - self, - hidden_states: torch.Tensor, - ) -> tuple[torch.Tensor, InputBatch]: + self, hidden_states: torch.Tensor, aux_hidden_states: list[torch.Tensor] | None + ) -> tuple[torch.Tensor, list[torch.Tensor] | None, InputBatch]: assert self._global_batch is not None - return self.restore_hidden_states(hidden_states), self._global_batch - - -def maybe_partition_pcp_batch( - manager: PCPManager | None, - input_batch: InputBatch, - batch_desc: "BatchExecutionDescriptor", -) -> InputBatch: - if manager is None: - return input_batch - - padded_num_tokens = None - padded_num_reqs = None - if batch_desc.cg_mode != CUDAGraphMode.NONE: - padded_num_tokens = batch_desc.num_tokens - if batch_desc.cg_mode == CUDAGraphMode.FULL: - padded_num_reqs = batch_desc.num_reqs - - return manager.partition_batch( - input_batch, - padded_num_tokens=padded_num_tokens, - padded_num_reqs=padded_num_reqs, - ) - - -def maybe_get_pcp_dummy_slot_mappings( - manager: PCPManager | None, - block_tables: BlockTables, - num_tokens: int, -) -> torch.Tensor: - if manager is None: - return block_tables.get_dummy_slot_mappings(num_tokens) - return manager.get_dummy_slot_mappings(num_tokens) - - -def maybe_restore_pcp_for_sampling( - manager: PCPManager | None, - hidden_states: torch.Tensor | None, - input_batch: InputBatch, -) -> tuple[torch.Tensor, InputBatch]: - assert hidden_states is not None - if manager is None: - return hidden_states, input_batch - return manager.restore_for_sampling(hidden_states) + hidden_states = self.restore_hidden_states(hidden_states) + if aux_hidden_states is not None: + aux_hidden_states = [ + self.restore_hidden_states(states) for states in aux_hidden_states + ] + return hidden_states, aux_hidden_states, self._global_batch def maybe_build_pcp_manager( diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index c2aaa1fba2..0a246c98ce 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -8,6 +8,7 @@ from dataclasses import dataclass import numpy as np import torch +import vllm.envs as envs from vllm.distributed.parallel_state import get_pp_group from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -71,8 +72,12 @@ class PPHandler: # pushed by step T's `receive` is consumed pp_size steps later. Pre-seeded # with pp_size None placeholders so the first pp_size consumes are no-ops. # None means no postprocess is pending for that step (broadcast skipped). + # Only XPU can disable microbatching via VLLM_XPU_PP_MICROBATCH. + ring_depth = get_pp_group().world_size + if current_platform.is_xpu() and not envs.VLLM_XPU_PP_MICROBATCH: + ring_depth = 1 self.queue: deque[PendingRecv | None] = ( - deque() if self.is_last_rank else deque([None] * get_pp_group().world_size) + deque() if self.is_last_rank else deque([None] * ring_depth) ) # Per req-index generation counter, incremented every time a request diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index 44cbadb9e0..e54a458709 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -62,21 +62,6 @@ def apply_temperature( ) -@triton.jit -def tl_rand64(seed, offset, includes_zero: tl.constexpr): - lo, hi, _, _ = tl.randint4x(seed, offset) - lo = lo.to(tl.uint32, bitcast=True).to(tl.uint64) - hi = hi.to(tl.uint32, bitcast=True).to(tl.uint64) - r = (hi << 32) | lo - - # 1 / 2**64 - scale = 5.421010862427522170037e-20 - u = r.to(tl.float64) * scale - if not includes_zero: - u = tl.maximum(u, 2.2250738585072014e-308) # float64 tiny - return u - - @triton.jit def tl_rand32(seed, offset, includes_zero: tl.constexpr): u = tl.rand(seed, offset) diff --git a/vllm/v1/worker/gpu/sample/logits_processor/loader.py b/vllm/v1/worker/gpu/sample/logits_processor/loader.py index e3b94d5394..1382abd537 100644 --- a/vllm/v1/worker/gpu/sample/logits_processor/loader.py +++ b/vllm/v1/worker/gpu/sample/logits_processor/loader.py @@ -196,10 +196,10 @@ def build_custom_logits_processors_params_validator( RuntimeError: if an FQCN fails to import. """ - if not custom_logitsprocs: - return lambda _: None + classes = _cached_load_v2_logitsprocs(tuple(custom_logitsprocs or ())) - classes = _cached_load_v2_logitsprocs(tuple(custom_logitsprocs)) + if not classes: + return lambda _: None def validate_params(sampling_params: "SamplingParams") -> None: for cls in classes: diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index b527c9f21b..ede733a755 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -15,6 +15,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device): return ExtractHiddenStatesSpeculator(vllm_config, device) elif speculative_config.method == "dflash": + if "LiLiCorrDraftModel" in speculative_config.draft_model_config.architectures: + from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import ( + LiLiCorrSpeculator, + ) + + return LiLiCorrSpeculator(vllm_config, device) if "DFlash2DraftModel" in speculative_config.draft_model_config.architectures: from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import ( DFlash2Speculator, diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 7e829cf309..5f91af59e5 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -19,8 +19,8 @@ from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream from vllm.v1.worker.gpu.attn_utils import ( - get_attn_cg_support, get_query_lens_mismatch_unsupported_backend, + get_varlen_cudagraph_unsupported_backend, ) logger = init_logger(__name__) @@ -28,7 +28,6 @@ _PROFILE_REPLAYS = 5 if TYPE_CHECKING: from vllm.config import VllmConfig - from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup @@ -255,9 +254,7 @@ class AdaptiveVerificationManager: logger.debug("DSpark cost tables: %s", self.cost_tables) def record_confidences( - self, - confidence_probs: torch.Tensor, - input_batch: "InputBatch", + self, confidence_probs: torch.Tensor, input_batch: "InputBatch" ) -> None: """Publish this step's raw confidences for the ranking kernel and start copying them to the CPU, where a later step's budget reads them.""" @@ -305,9 +302,9 @@ class AdaptiveVerificationManager: ) num_non_draft_tokens = scheduled_tokens - scheduled_drafts slots = np.fromiter( - (self.req_states.req_id_to_index[req_id] for req_id in req_ids), + map(self.req_states.req_id_to_index.__getitem__, req_ids), dtype=np.int32, - count=len(req_ids), + count=num_reqs, ) stale_confidences = self._stale_confidences[self._stale_idx].np[slots] survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) @@ -454,7 +451,6 @@ def maybe_create_adaptive_verification_manager( *, enable_adaptive_verification: bool, attn_groups: list[list["AttentionGroup"]], - attn_cg_support: "AttentionCGSupportInfo", req_states: "RequestState", query_start_loc: torch.Tensor, num_bonus_tokens: int, @@ -469,8 +465,7 @@ def maybe_create_adaptive_verification_manager( # The selector rejects unsupported backends, but models that # hard-wire theirs (e.g. DeepSeek-V4) never go through it. backend = get_query_lens_mismatch_unsupported_backend( - attn_groups, - checked_layer_names=target_layer_names, + attn_groups, checked_layer_names=target_layer_names ) if backend is not None: raise ValueError( @@ -480,25 +475,30 @@ def maybe_create_adaptive_verification_manager( "use a backend that does." ) - target_attn_cg_support = attn_cg_support - if target_layer_names is not None: - target_attn_cg_support = get_attn_cg_support( + # The runner's decode_query_len, the width varlen decode graphs capture. + max_query_len = req_states.num_speculative_steps + num_bonus_tokens + unsupported: tuple[str | None, int | None] | None = ( + get_varlen_cudagraph_unsupported_backend( attn_groups, vllm_config, + max_query_len, checked_layer_names=target_layer_names, ) - if additional_attn_cg_support is not None: - target_attn_cg_support = target_attn_cg_support.narrow( - *additional_attn_cg_support - ) - if target_attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: + ) + if unsupported is None and additional_attn_cg_support is not None: + # Groups built outside init_attn_backend report only their support + # level, and without a bound only ALWAYS replays varlen batches. + additional_support, additional_backend = additional_attn_cg_support + if additional_support != AttentionCGSupport.ALWAYS: + unsupported = additional_backend, None + if unsupported is not None: + backend, bound = unsupported + allowed = "none" if bound is None else f"at most {bound}" raise ValueError( - "Adaptive verification captures varlen decode cudagraphs, so every" - " target attention builder must report AttentionCGSupport.ALWAYS, but " - f"{target_attn_cg_support.min_cg_attn_backend} reports " - f"{target_attn_cg_support.min_cg_support}. Pass " - "enable_adaptive_verification=false in the speculative config, or " - "use a backend that does." + "Adaptive verification replays decode cudagraphs whose per-request " + f"query lengths vary up to {max_query_len}, but {backend} allows " + f"{allowed}. Pass enable_adaptive_verification=false in the " + "speculative config, or use a backend that does." ) return AdaptiveVerificationManager( diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py index ef3b6e2ed5..fe3bdef269 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py @@ -57,6 +57,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager): attn_groups, kv_cache_config, full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, + max_query_len=desc.max_query_len or desc.uniform_token_count, ) return lambda cg_mode: forward_fn( diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index c787efa07f..ca0662076b 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -53,10 +53,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): return self.inputs_embeds = torch.zeros( - self.max_num_tokens, - self.hidden_size, - dtype=self.dtype, - device=self.device, + self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=self.device ) # Lifecycle hooks for model-specific optimizations. Subclasses override @@ -140,10 +137,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): # Initialize cudagraph manager for draft decodes (draft positions > 0). self.decode_cudagraph_manager = SpeculatorCudaGraphManager( - self.vllm_config, - self.device, - cudagraph_mode, - decode_query_len=1, + self.vllm_config, self.device, cudagraph_mode, decode_query_len=1 ) def capture(self) -> None: @@ -272,8 +266,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): if self.pcp_manager is not None: self.pcp_manager.prepare_draft_prefill( - input_batch, - self.input_buffers.input_ids[:num_tokens_padded], + input_batch, self.input_buffers.input_ids[:num_tokens_padded] ) prefill = self.pcp_manager.draft_prefill_batch if prefill is not None: diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py index 725714fcdb..e21ca9f870 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -11,7 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( AttentionState, BatchExecutionDescriptor, @@ -42,15 +42,16 @@ def _prepare_dflash_inputs_to_capture( attn_metadata = None if not skip_attn: query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) - input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens( - input_buffers.dcp_local_seq_lens, - input_batch.seq_lens, - input_batch.num_reqs, - block_tables.cp_size, - block_tables.cp_rank, - block_tables.cp_interleave, - num_reqs_padded=input_batch.num_reqs_after_padding, - ) + if block_tables.cp_size > 1: + input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens( + input_buffers.dcp_local_seq_lens, + input_batch.seq_lens, + input_batch.num_reqs, + block_tables.cp_size, + block_tables.cp_rank, + block_tables.cp_interleave, + num_reqs_padded=input_batch.num_reqs_after_padding, + ) attn_metadata = build_attn_metadata( attn_groups=attn_groups, num_reqs=num_reqs, @@ -83,8 +84,12 @@ class DFlashCudaGraphManager(CudaGraphManager): kv_cache_config: KVCacheConfig, max_model_len: int, causal: bool | Mapping[int, bool], + precompute_context_kv: Callable[[int], None], progress_bar_desc: str = "Capturing CUDA graphs", ) -> None: + """``precompute_context_kv(num_reqs)`` is captured ahead of the query + forward.""" + def create_forward_fn( desc: BatchExecutionDescriptor, warmup: bool, @@ -109,13 +114,17 @@ class DFlashCudaGraphManager(CudaGraphManager): ) attn_metadata, slot_mappings = attn_state - return lambda cg_mode: forward_fn( - num_reqs, - num_tokens, - attn_metadata, - slot_mappings, - num_tokens_across_dp, - cg_mode, - ) + def forward(cg_mode: CUDAGraphMode) -> None: + precompute_context_kv(num_reqs) + forward_fn( + num_reqs, + num_tokens, + attn_metadata, + slot_mappings, + num_tokens_across_dp, + cg_mode, + ) + + return forward super().capture(create_forward_fn, progress_bar_desc) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index a5710829e0..57110e0b05 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator): def __init__(self, vllm_config: VllmConfig, device: torch.device): parallel_config = vllm_config.parallel_config - if parallel_config.prefill_context_parallel_size > 1: - vllm_config = copy.copy(vllm_config) - vllm_config.parallel_config = replace( - parallel_config, - prefill_context_parallel_size=1, - ) + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + vllm_config = copy.copy(vllm_config) + vllm_config.parallel_config = replace( + parallel_config, + prefill_context_parallel_size=1, + decode_context_parallel_size=( + parallel_config.decode_context_parallel_size + if speculative_config.draft_model_config.use_mla + else 1 + ), + ) super().__init__(vllm_config, device) self.hidden_states = torch.zeros( @@ -79,9 +85,7 @@ class DFlashSpeculator(DraftModelSpeculator): ) self.sample_from_anchor = False - # Context positions for the K/V precompute. Populated by - # prepare_dflash_inputs, and processed by the model's - # precompute_and_store_context_kv method. NOT captured by CUDA graphs. + # Context positions for the K/V precompute, populated by prepare_dflash_inputs. self.context_positions = torch.zeros( self.max_num_tokens, dtype=torch.int64, device=device ) @@ -151,6 +155,8 @@ class DFlashSpeculator(DraftModelSpeculator): self.sample_indices.zero_() self.sample_pos.zero_() self.sample_idx_mapping.fill_(-1) + # Capture must not write context K/V. + self._context_slot_mappings.fill_(PAD_SLOT_ID) assert self.query_cudagraph_manager is not None self.query_cudagraph_manager.capture( self._generate_draft, @@ -160,6 +166,9 @@ class DFlashSpeculator(DraftModelSpeculator): self.kv_cache_config, self.max_model_len, causal=self._group_causal, + precompute_context_kv=lambda num_reqs: self._precompute_context_kv( + 0, self._num_graph_context_tokens(num_reqs) + ), progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs", ) @@ -297,6 +306,33 @@ class DFlashSpeculator(DraftModelSpeculator): num_reqs, self.num_speculative_steps ) + def _num_graph_context_tokens(self, num_reqs: int) -> int: + # Context rows a captured draft step stores: one full verify per request. + return min(num_reqs * (self.num_speculative_steps + 1), self.max_num_tokens) + + def _precompute_context_kv( + self, start: int, end: int, dummy_run: bool = False + ) -> None: + if dummy_run: + context_slots: torch.Tensor | list[torch.Tensor | None] | None = None + elif self._layer_group_idx is not None: + context_slots = [ + self._context_slot_mappings[gidx][start:end] + for gidx in self._layer_group_idx + ] + else: + context_slots = self._context_slot_mappings[0][start:end] + self.model.precompute_and_store_context_kv( + self.hidden_states[start:end], + self.context_positions[start:end], + context_slots, + ) + + def prepare_context_anchor( + self, input_batch: InputBatch, num_rejected: torch.Tensor + ) -> None: + """Publish context features required by a draft's candidate head.""" + @torch.inference_mode() def propose( self, @@ -344,6 +380,7 @@ class DFlashSpeculator(DraftModelSpeculator): else: hidden_states = last_hidden_states self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens]) + self.prepare_context_anchor(input_batch, num_rejected) if dummy_run and skip_attn_for_dummy_run: # Memory profiling path: block_tables / kv_cache_config are not initialized. @@ -396,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator): self.block_tables.input_block_tables[gid], self.block_tables.kernel_block_sizes[gid], self.block_tables.cp_rank, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_interleave, self.parallel_drafting_token_id, self.num_query_per_req, @@ -407,25 +444,6 @@ class DFlashSpeculator(DraftModelSpeculator): self.sample_from_anchor, ) - # Pre-insert context K/V into the cache. Runs eagerly outside the captured graph - # because the context shape varies per step. During dummy runs the block tables - # are placeholders, so we skip the cache write to avoid clobbering real entries. - # Each layer uses the context slots of its own kv-cache group. - if dummy_run: - context_slots: torch.Tensor | list[torch.Tensor | None] | None = None - elif self._layer_group_idx is not None: - context_slots = [ - self._context_slot_mappings[gidx][:num_target_tokens] - for gidx in self._layer_group_idx - ] - else: - context_slots = self._context_slot_mappings[0][:num_target_tokens] - self.model.precompute_and_store_context_kv( - self.hidden_states[:num_target_tokens], - self.context_positions[:num_target_tokens], - context_slots, - ) - batch_sync, num_batch_tokens = ( self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req) if dp_sync is not None @@ -447,6 +465,24 @@ class DFlashSpeculator(DraftModelSpeculator): batch_sync.num_tokens_across_dp if batch_sync is not None else None ) + if batch_desc.cg_mode == CUDAGraphMode.FULL: + # The graph stores the first num_context context rows. + assert batch_desc.num_reqs is not None + num_context = self._num_graph_context_tokens(batch_desc.num_reqs) + if dummy_run: + # Dummy block tables are placeholders: write no context K/V. + self._context_slot_mappings[:, :num_context].fill_(PAD_SLOT_ID) + elif num_target_tokens <= num_context: + # Rows past the batch keep stale positions but write no K/V. + self._context_slot_mappings[:, num_target_tokens:num_context].fill_( + PAD_SLOT_ID + ) + else: + # Prefill context beyond the graph's rows is stored before replay. + self._precompute_context_kv(num_context, num_target_tokens) + else: + self._precompute_context_kv(0, num_target_tokens, dummy_run) + # Rebuild the draft attention metadata even when replaying the FULL # graph so that any attention metadata builder state is updated. draft_attn_metadata = self._build_uniform_attn_metadata( diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py index 2e58263623..d32587d266 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py @@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo target_lm_head = get_target_lm_head(target_model, target_language_model) draft_lm_head = getattr(dflash_model, "lm_head", None) - if target_lm_head is not None and _should_share( - dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head + if ( + target_lm_head is not None + and not getattr(dflash_model.config, "has_own_lm_head", False) + and _should_share( + dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head + ) ): if draft_lm_head is not None: del dflash_model.lm_head diff --git a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py index 7db5284497..bb14621129 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py @@ -108,26 +108,93 @@ def _cache_draft_logits_kernel( tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask) +class CandidateSampler: + """The shared DFlash2/LiLiCorr candidate walk and realized proposal cache.""" + + def __init__( + self, max_num_reqs: int, num_steps: int, top_k: int, device: torch.device + ): + self.num_steps = num_steps + self.top_k = top_k + self.scores = torch.empty( + max_num_reqs, num_steps, top_k, dtype=torch.float32, device=device + ) + self.cached_candidate_ids = torch.zeros( + self.scores.shape, dtype=torch.int64, device=device + ) + + def sample( + self, + candidate_ids: torch.Tensor, + scores: torch.Tensor, + num_reqs: int, + sample_pos: torch.Tensor, + idx_mapping: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + draft_tokens: torch.Tensor, + draft_logits: torch.Tensor | None, + use_fp64: bool, + ) -> None: + candidate_ids = candidate_ids.contiguous() + sample_pos = sample_pos.contiguous() + idx_mapping = idx_mapping.contiguous() + block_k = triton.next_power_of_2(self.top_k) + _selector_walk_kernel[(num_reqs,)]( + scores.contiguous(), + candidate_ids.contiguous(), + sample_pos, + idx_mapping, + temperature, + seeds, + draft_tokens, + self.scores, + num_steps=self.num_steps, + top_k=self.top_k, + BLOCK_K=block_k, + SAMPLE_PROBABILISTIC=draft_logits is not None, + USE_FP64=use_fp64, + num_warps=1, + ) + + if draft_logits is not None: + self._cache_draft_logits( + candidate_ids, num_reqs * self.num_steps, idx_mapping, draft_logits + ) + + def _cache_draft_logits( + self, + candidate_ids: torch.Tensor, + num_sample: int, + idx_mapping: torch.Tensor, + draft_logits: torch.Tensor, + ) -> None: + block_k = triton.next_power_of_2(self.top_k) + _cache_draft_logits_kernel[(num_sample,)]( + draft_logits, + self.cached_candidate_ids, + candidate_ids, + self.scores, + idx_mapping, + draft_logits.stride(0), + draft_logits.stride(1), + num_steps=self.num_steps, + top_k=self.top_k, + BLOCK_K=block_k, + num_warps=1, + ) + + class DFlash2Speculator(DFlashSpeculator): _speculator_name = "DFlash2" + _candidate_top_k_key = "selector_top_k" def __init__(self, vllm_config: VllmConfig, device: torch.device): super().__init__(vllm_config, device) draft_config = self.draft_model_config.hf_config.dflash_config - self.selector_top_k = int(draft_config["selector_top_k"]) - self._anchor_indices = ( - torch.arange(self.max_num_reqs, dtype=torch.int64, device=device) - * self.num_query_per_req - ) - self._selector_scores = torch.empty( - self.max_num_reqs, - self.num_speculative_steps, - self.selector_top_k, - dtype=torch.float32, - device=device, - ) - self._cached_candidate_ids = torch.zeros( - self._selector_scores.shape, dtype=torch.int64, device=device + self.top_k = int(draft_config[self._candidate_top_k_key]) + self.candidate_sampler = CandidateSampler( + self.max_num_reqs, self.num_speculative_steps, self.top_k, device ) def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]: @@ -136,48 +203,6 @@ class DFlash2Speculator(DFlashSpeculator): # candidates. return torch.float32, -float("inf") - def _sample_path( - self, - candidate_ids: torch.Tensor, - scores: torch.Tensor, - num_reqs: int, - ) -> None: - block_k = triton.next_power_of_2(self.selector_top_k) - _selector_walk_kernel[(num_reqs,)]( - scores.contiguous(), - candidate_ids.contiguous(), - self.sample_pos, - self.sample_idx_mapping, - self.temperature, - self.seeds, - self.draft_tokens, - self._selector_scores, - num_steps=self.num_speculative_steps, - top_k=self.selector_top_k, - BLOCK_K=block_k, - SAMPLE_PROBABILISTIC=self.draft_logits is not None, - USE_FP64=self.use_fp64_gumbel, - num_warps=1, - ) - - def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None: - draft_logits = self.draft_logits - assert draft_logits is not None - block_k = triton.next_power_of_2(self.selector_top_k) - _cache_draft_logits_kernel[(num_sample,)]( - draft_logits, - self._cached_candidate_ids, - candidate_ids, - self._selector_scores, - self.sample_idx_mapping, - draft_logits.stride(0), - draft_logits.stride(1), - num_steps=self.num_speculative_steps, - top_k=self.selector_top_k, - BLOCK_K=block_k, - num_warps=1, - ) - def _generate_draft( self, num_reqs: int, @@ -202,22 +227,39 @@ class DFlash2Speculator(DFlashSpeculator): hidden_states.flatten(0, 1) ) candidate_ids = candidate_ids.view( - num_reqs, self.num_speculative_steps, self.selector_top_k + num_reqs, self.num_speculative_steps, self.top_k ) unary_logits = unary_logits.view_as(candidate_ids) - anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]] - scores = self.model.model.candidate_selector( + scores = self._score_candidates(candidate_ids, unary_logits, hidden_states) + self.candidate_sampler.sample( candidate_ids, - unary_logits, - hidden_states, - anchor_token_ids, + scores, + num_reqs, + self.sample_pos, + self.sample_idx_mapping, + self.temperature, + self.seeds, + self.draft_tokens, + self.draft_logits, + self.use_fp64_gumbel, ) - self._sample_path(candidate_ids, scores, num_reqs) if self.enable_adaptive_verification: self._maybe_predict_acceptance( - self._selector_scores[:num_reqs].flatten(0, 1), + self.candidate_sampler.scores[:num_reqs].flatten(0, 1), self.sample_idx_mapping[:num_sample], self.sample_col[:num_sample], ) - if self.draft_logits is not None: - self._cache_draft_logits(candidate_ids, num_sample) + + def _score_candidates( + self, + candidate_ids: torch.Tensor, + unary_logits: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + num_reqs = candidate_ids.shape[0] + anchor_token_ids = self.input_buffers.input_ids[ + : num_reqs * self.num_query_per_req : self.num_query_per_req + ] + return self.model.model.candidate_selector( + candidate_ids, unary_logits, hidden_states, anchor_token_ids + ) diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 411aa7f041..b86f58a450 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -149,10 +149,7 @@ class DSparkSpeculator(DFlashSpeculator): ) if self.draft_watermarker is not None: sampled = self.draft_watermarker.sample( - logits, - sampled, - idx_map, - self.temperature, + logits, sampled, idx_map, self.temperature ) return sampled diff --git a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py index 9a489bcc48..6f76e5055a 100644 --- a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py @@ -24,6 +24,27 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( logger = init_logger(__name__) +def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None: + """Copy target KV scales while preserving their tensor representation. + + Default attention scales are scalar buffers, while some quantization + methods replace them with length-one or per-head parameters. Preserve the + draft layer's registration type so the shared KV cache is interpreted with + the target's values and shapes without aliasing target parameters. + """ + for scale_name in ("_k_scale", "_v_scale"): + draft_scale = getattr(attn, scale_name) + target_scale = getattr(target_attn, scale_name) + scale = target_scale.detach().clone() + if isinstance(draft_scale, nn.Parameter): + scale = nn.Parameter(scale, requires_grad=False) + setattr(attn, scale_name, scale) + for scale_name in ("_k_scale_float", "_v_scale_float"): + setattr(attn, scale_name, getattr(target_attn, scale_name)) + for scale_name in ("_k_scale_cpu", "_v_scale_cpu"): + getattr(attn, scale_name).copy_(getattr(target_attn, scale_name)) + + class Gemma4Speculator(AutoRegressiveSpeculator): @property def advance_draft_positions(self) -> bool: @@ -77,7 +98,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator): model: nn.Module, target_attn_layer_names: set[str], ) -> None: - """Wire draft layers to share KV with the target model. + """Wire draft layers to share KV and KV scales with the target model. Each draft decoder layer is mapped to the last non-KV-shared target layer of the same attention type (sliding or full). @@ -92,7 +113,10 @@ class Gemma4Speculator(AutoRegressiveSpeculator): return target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0) - num_non_shared = len(target_layer_types) - target_num_kv_shared + target_num_layers = getattr( + target_text_config, "num_hidden_layers", len(target_layer_types) + ) + num_non_shared = target_num_layers - target_num_kv_shared type_to_target_indices: dict[str, list[int]] = defaultdict(list) for idx, lt in enumerate(target_layer_types[:num_non_shared]): type_to_target_indices[lt].append(idx) @@ -128,6 +152,17 @@ class Gemma4Speculator(AutoRegressiveSpeculator): target_idx = candidates[-1] target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn" attn.kv_sharing_target_layer_name = target_layer_name + + # KV-cache sharing aliases the cache tensor during allocation, but + # the quantization scales live on the Attention modules themselves. + # The BF16 draft model has no quantization config, so its K/V scales + # otherwise remain at the default 1.0 while it reads the target's + # calibrated FP8 cache. + target_attn = self.vllm_config.compilation_config.static_forward_context[ + target_layer_name + ] + _copy_target_kv_scales(attn, target_attn) + logger.info( "Gemma4 MTP: draft layer %d (%s) -> %s", draft_idx, diff --git a/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py b/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py b/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py new file mode 100644 index 0000000000..213dd962c6 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator +from vllm.v1.worker.gpu.spec_decode.eagle.utils import get_target_lm_head + + +class LiLiCorrSpeculator(DFlash2Speculator): + _speculator_name = "LiLiCorr" + _candidate_top_k_key = "lilicorr_candidate_topk" + + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + config = self.draft_model_config.hf_config + self.anchor_hidden = torch.zeros( + self.max_num_reqs, config.hidden_size, dtype=self.dtype, device=device + ) + self.anchor_valid = torch.zeros( + self.max_num_reqs, dtype=torch.bool, device=device + ) + + def load_draft_model( + self, target_model: nn.Module, target_attn_layer_names: set[str] + ) -> nn.Module: + model = super().load_draft_model(target_model, target_attn_layer_names) + language_model = ( + target_model.get_language_model() + if hasattr(target_model, "get_language_model") + else target_model + ) + inner = getattr(language_model, "model", language_model) + embedding = getattr(inner, "embed_tokens", None) or getattr( + inner, "embedding", None + ) + lm_head = ( + model.lm_head + if model.has_own_lm_head + else get_target_lm_head(target_model, language_model) + ) + if not isinstance(embedding, VocabParallelEmbedding) or not isinstance( + lm_head, VocabParallelEmbedding + ): + raise ValueError( + "LiLiCorr requires the target input embedding and its selected " + "LM head on the draft rank." + ) + if embedding.embedding_dim != model.config.hidden_size: + raise ValueError( + "LiLiCorr target embedding width must match its trained " + "token projection input." + ) + if ( + embedding.shard_indices.num_added_elements + or lm_head.shard_indices.num_added_elements + ): + raise ValueError("LiLiCorr does not support added vocabulary entries.") + # Candidate embeddings always come from the target; the checkpoint may + # supply its own candidate LM head. + self.target_embeddings = embedding + model.lm_head = lm_head + return model + + def prepare_context_anchor( + self, input_batch: InputBatch, num_rejected: torch.Tensor + ) -> None: + num_reqs = input_batch.num_reqs + starts = input_batch.query_start_loc[:num_reqs] + ends = input_batch.query_start_loc[1 : num_reqs + 1] - num_rejected[:num_reqs] + valid = ends > starts + indices = (ends - 1).clamp_min(0).long() + self.anchor_hidden.zero_() + self.anchor_valid.zero_() + self.anchor_hidden[:num_reqs].copy_( + self.model.model.hidden_norm(self.hidden_states[indices]) * valid[:, None] + ) + self.anchor_valid[:num_reqs].copy_(valid) + + def _score_candidates( + self, + candidate_ids: torch.Tensor, + unary_logits: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + num_reqs = candidate_ids.shape[0] + return self.model.model.lilicorr( + self.target_embeddings(candidate_ids), + unary_logits, + hidden_states, + self.anchor_hidden[:num_reqs], + self.anchor_valid[:num_reqs], + ) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 427c93b73c..743d66010d 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -134,8 +134,7 @@ class RejectionSampler: if expanded_logits: if self.enable_adaptive_verification: # Adaptive verification keeps the true per-request boundaries - # on device only; cu_num_logits_np holds the pre-compacted - # layout. + # on device only; cu_num_logits_np holds the pre-compacted layout. cu_num_generated_tokens = cu_num_logits.clone() else: cu_num_generated_tokens = cu_num_logits_np.tolist() diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index eef1704ac1..6f88febb66 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -26,7 +26,7 @@ from vllm.v1.worker.gpu.attn_utils import ( init_attn_backend, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import DPSyncState from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers @@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator): # DP configuration self.dp_size = vllm_config.parallel_config.data_parallel_size self.dp_rank = vllm_config.parallel_config.data_parallel_rank + self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size self.eplb_state: EplbState | None = None @@ -202,9 +203,7 @@ class DraftModelSpeculator(BaseSpeculator): @abstractmethod def load_draft_model( - self, - target_model: nn.Module, - target_attn_layer_names: set[str], + self, target_model: nn.Module, target_attn_layer_names: set[str] ) -> nn.Module: pass @@ -334,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator): out=draft_seq_lens_cpu_upper_bound[:num_reqs], ) draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len) - if dcp_local_seq_lens is None and self.block_tables.cp_size > 1: + if dcp_local_seq_lens is None and self.dcp_size > 1: # Draft steps advance and rewind their own global sequence lengths, # so the target model's DCP-local lengths may already be stale. - dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens( + dcp_local_seq_lens = prepare_dcp_local_seq_lens( self.input_buffers.dcp_local_seq_lens, self.input_buffers.seq_lens, num_reqs, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_rank, self.block_tables.cp_interleave, ) @@ -366,7 +365,7 @@ class DraftModelSpeculator(BaseSpeculator): kv_cache_config=self.kv_cache_config, causal=causal, seq_lens_cpu_upper_bound=draft_seq_lens_cpu_upper_bound, - is_prefilling=self.draft_is_prefilling[:num_reqs], + is_prefilling=self.draft_is_prefilling[:num_reqs_padded], ) return attn_metadata @@ -422,10 +421,7 @@ class DraftModelSpeculator(BaseSpeculator): ) if self.draft_watermarker is not None: sampled = self.draft_watermarker.sample( - logits, - sampled, - idx_mapping, - temperature, + logits, sampled, idx_mapping, temperature ) elif self.use_local_argmax_reduction: return self.model.get_top_tokens(hidden_states) @@ -436,10 +432,7 @@ class DraftModelSpeculator(BaseSpeculator): return sampled def _maybe_predict_acceptance( - self, - logits: torch.Tensor, - idx_mapping: torch.Tensor, - draft_step: torch.Tensor, + self, logits: torch.Tensor, idx_mapping: torch.Tensor, draft_step: torch.Tensor ) -> None: if self.acceptance_estimator is not None: self.acceptance_estimator.predict( diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index 025c8300fc..db27bf013f 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -78,10 +78,7 @@ class RequestState: ) self.next_prefill_tokens = torch.zeros( - num_prefill_lookahead, - self.max_num_reqs, - dtype=torch.int32, - device=device, + num_prefill_lookahead, self.max_num_reqs, dtype=torch.int32, device=device ) @property diff --git a/vllm/v1/worker/gpu/ubatch_utils.py b/vllm/v1/worker/gpu/ubatch_utils.py index e3cade2d0b..d6f24fcc69 100644 --- a/vllm/v1/worker/gpu/ubatch_utils.py +++ b/vllm/v1/worker/gpu/ubatch_utils.py @@ -24,7 +24,7 @@ from vllm.sequence import IntermediateTensors from vllm.utils.torch_utils import current_stream from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer -from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.ubatch_utils import ( @@ -150,7 +150,7 @@ def _slice_input_batch( dcp_local_seq_lens = None if dcp_size > 1: assert dcp_local_seq_lens_buf is not None - dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens( + dcp_local_seq_lens = prepare_dcp_local_seq_lens( dcp_local_seq_lens_buf, seq_lens, num_reqs, diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 79db440cff..4707e4f2ca 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -541,6 +541,7 @@ class InputBatch: self.req_output_token_ids[req_index] = None self.spec_token_ids[req_index].clear() self.block_table.clear_row(req_index) + self.req_prompt_embeds.pop(req_index, None) # LoRA lora_id = self.request_lora_mapping[req_index] @@ -687,12 +688,8 @@ class InputBatch: swap_dict_values(self.bad_words_token_ids, i1, i2) if self.allowed_token_ids_mask_cpu_tensor is not None: - ( - self.allowed_token_ids_mask_cpu_tensor[i1], - self.allowed_token_ids_mask_cpu_tensor[i2], - ) = ( - self.allowed_token_ids_mask_cpu_tensor[i2], - self.allowed_token_ids_mask_cpu_tensor[i1], + self.allowed_token_ids_mask_cpu_tensor[[i1, i2]] = ( + self.allowed_token_ids_mask_cpu_tensor[[i2, i1]] ) def _get_active_token_count(self, req_index: int) -> int: @@ -820,6 +817,7 @@ class InputBatch: self.allowed_token_ids_mask_cpu_tensor[empty_index] = ( self.allowed_token_ids_mask_cpu_tensor[last_req_index] ) + self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False) bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None) if bad_words_token_ids is not None: diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index f2cab7898b..c6b380d63e 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging import threading from collections.abc import Callable from dataclasses import dataclass @@ -8,7 +9,6 @@ from typing import Any import torch -import vllm.envs as envs from vllm.compilation.cuda_graph import CUDAGraphWrapper from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id @@ -89,7 +89,7 @@ class UBatchWrapper: self.sm_control = create_sm_control_context(vllm_config.parallel_config) self.device = device - self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG" + self.is_debugging_mode = logger.isEnabledFor(logging.DEBUG) self._runnable_str = str(runnable) if self.is_debugging_mode else None @property diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index df1d9de6ae..778d71bdbc 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -13,7 +13,6 @@ from types import NoneType from typing import TYPE_CHECKING, Any, cast import numpy as np -import regex as re import torch import torch.nn as nn @@ -367,27 +366,32 @@ class Worker(WorkerBase): @contextmanager def _scoped_allocator_max_split(self, max_split_size_mb: int): - """Temporarily set max_split_size_mb to reduce allocator fragmentation at the - cost of more cudaMalloc calls (negligible in practice). Restores the original - value on exit.""" - if not current_platform.is_cuda(): + """Limit CUDA/ROCm allocator splitting, restoring settings on exit.""" + if ( + not current_platform.is_cuda_alike() + or torch.cuda.memory.get_allocator_backend() != "native" + ): yield return - conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") - match = re.search(r"max_split_size_mb:(\d+)", conf) - original_value = match.group(1) if match else None + get_settings = getattr(torch._C, "_accelerator_getAllocatorSettings", None) + if get_settings is not None: + original_settings = get_settings() + else: + original_settings = torch.cuda.memory._snapshot()["allocator_settings"][ + "PYTORCH_CUDA_ALLOC_CONF" + ] - torch._C._accelerator_setAllocatorSettings( - f"max_split_size_mb:{max_split_size_mb}" - ) + # The setter resets other allocator options unless they are included. + prefix = original_settings.strip().rstrip(",") + settings = f"max_split_size_mb:{max_split_size_mb}" + if prefix: + settings = f"{prefix},{settings}" try: + torch._C._accelerator_setAllocatorSettings(settings) yield finally: - # PyTorch defaults to SIZE_MAX (no limit). - _SIZE_MAX_MB = (2**64 - 1) // (1024 * 1024) - restore = original_value if original_value else str(_SIZE_MAX_MB) - torch._C._accelerator_setAllocatorSettings(f"max_split_size_mb:{restore}") + torch._C._accelerator_setAllocatorSettings(original_settings) @instrument(span_name="Init device") def init_device(self): @@ -406,13 +410,14 @@ class Worker(WorkerBase): if dp_local_rank is None: dp_local_rank = self.parallel_config.data_parallel_index - tp_pp_world_size = ( + tp_pcp_pp_world_size = ( self.parallel_config.pipeline_parallel_size + * self.parallel_config.prefill_context_parallel_size * self.parallel_config.tensor_parallel_size ) - # DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK - self.local_rank += dp_local_rank * tp_pp_world_size + # DP_LOCAL_RANK * TP_PCP_PP_WORLD_SIZE + TP_LOCAL_RANK + self.local_rank += dp_local_rank * tp_pcp_pp_world_size # Publish the logical-to-physical mapping for topology queries # such as NIC affinity and P2P checks. @@ -550,6 +555,11 @@ class Worker(WorkerBase): self.model_runner.get_model(), ) + # Preserve parallel weight loading, then use serving's thread count + # for profiling and compilation so Dynamo's global-state guards remain + # valid when requests arrive. + set_torch_threads_for_runtime() + def update_config(self, overrides: dict[str, Any]) -> None: self.model_runner.update_config(overrides) @@ -599,10 +609,20 @@ class Worker(WorkerBase): # Execute a forward pass with dummy inputs to profile the memory usage # of the model. - with memory_profiling( - self.init_snapshot, - weights_memory=int(self.model_runner.model_memory_usage), - ) as profile_result: + with ( + memory_profiling( + self.init_snapshot, + weights_memory=int(self.model_runner.model_memory_usage), + ) as profile_result, + # Workspaces (e.g. the MoE workspace) grow in steps during this pass, + # freeing each smaller buffer. Without a split limit, a later small + # allocation can be carved out of a freed multi-GiB block and pin the + # whole segment past the empty_cache() in memory_profiling, so it is + # counted as consumed and taken from the KV cache. Blocks above the + # limit are never split, so they stay releasable. Exits before + # memory_profiling measures, restoring the original limit. + self._scoped_allocator_max_split(max_split_size_mb=20), + ): self.model_runner.profile_run() # Profile CUDA graph memory if graphs will be captured. @@ -610,12 +630,10 @@ class Worker(WorkerBase): # torch.accelerator.get_memory_info (reliable on ROCm, as used by # the AMD-CI mem tests), and graph_pool_handle resolves to the same # torch.cuda handle the live capture path already uses on ROCm. - # XPU stays excluded (see #39977). cudagraph_memory_estimate = 0 if ( - current_platform.is_cuda_alike() - and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE - ): + current_platform.is_cuda_alike() or current_platform.is_xpu() + ) and self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: cudagraph_memory_estimate = self.model_runner.profile_cudagraph_memory() # Respect the opt-in flag as originally designed. @@ -783,11 +801,20 @@ class Worker(WorkerBase): # related to kv cache connector (e.g. kv cache sharing layers). ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config) + # If the connector provides a custom memory pool (e.g. Mooncake + # NVLink/BAREX), use it for KV cache allocation; otherwise fall + # back to the standard CuMem pool. + mem_pool_context = ( + get_kv_transfer_group().get_mem_pool_context() + if has_kv_transfer_group() + else None + ) + if mem_pool_context is None: + mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache") + self.model_runner.initialize_kv_cache( kv_cache_config, - kv_cache_allocation_context=self._maybe_get_memory_pool_context( - tag="kv_cache" - ), + kv_cache_allocation_context=mem_pool_context, ) # Build KV-zero metadata outside the CuMem pool so the bookkeeping @@ -838,9 +865,12 @@ class Worker(WorkerBase): # cuda graph capture. kernel_warmup(self) - if self.use_v2_model_runner: - # A workspace resize after capture frees what the graphs point at. - warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens) + if self.use_v2_model_runner: # noqa: SIM102 + if self.vllm_config.kernel_config.enable_jit_warmup: + # A workspace resize after capture frees what the graphs point at. + warmup_kernels( + self.model_runner, self.execute_model, self.sample_tokens + ) cuda_graph_memory_bytes = 0 if not self.model_config.enforce_eager: @@ -959,12 +989,7 @@ class Worker(WorkerBase): # All warmup is done — start monitoring for unexpected JIT # compilations that would cause latency spikes during inference. - from vllm.utils.jit_monitor import activate as activate_jit_monitor - - activate_jit_monitor( - mode=self.observability_config.jit_monitor_mode, - verbose=self.observability_config.jit_monitor_verbose, - ) + self._maybe_activate_jit_monitor() # Freeze the worker heap so the GC won't scan static objects # (model weights, KV caches, CUDA graphs) during inference. @@ -975,15 +1000,25 @@ class Worker(WorkerBase): # gate so subsequent `execute_model` / `sample_tokens` calls enforce it. enable_gpu_sync_check() - # Startup is done; steady-state serving gets no benefit from torch - # intra-op parallelism. - set_torch_threads_for_runtime() - return CompilationTimes( language_model=self.compilation_config.compilation_time, encoder=self.compilation_config.encoder_compilation_time, ) + def _maybe_activate_jit_monitor(self) -> None: + # When JIT warmup is disabled (e.g. enforce_eager), runtime JIT + # compilation is expected, so monitoring would only produce noise + # (or spurious errors in "error" mode). + if not self.vllm_config.kernel_config.enable_jit_warmup: + return + + from vllm.utils.jit_monitor import activate as activate_jit_monitor + + activate_jit_monitor( + mode=self.observability_config.jit_monitor_mode, + verbose=self.observability_config.jit_monitor_verbose, + ) + def _get_cudagraph_capture_context(self) -> AbstractContextManager[None]: """Let the configured profiler observe CUDA graph capture.""" if not self.use_v2_model_runner: diff --git a/vllm/v1/worker/kv_connector_model_runner_mixin.py b/vllm/v1/worker/kv_connector_model_runner_mixin.py index 26ffc159f3..40382b5c74 100644 --- a/vllm/v1/worker/kv_connector_model_runner_mixin.py +++ b/vllm/v1/worker/kv_connector_model_runner_mixin.py @@ -29,7 +29,7 @@ class KVConnectorModelRunnerMixin: with ( set_forward_context(None, vllm_config), KVConnectorModelRunnerMixin._get_kv_connector_output( - scheduler_output, wait_for_save=False + scheduler_output ) as kv_connector_output, ): pass @@ -66,7 +66,6 @@ class KVConnectorModelRunnerMixin: @contextmanager def _get_kv_connector_output( scheduler_output: "SchedulerOutput", - wait_for_save: bool = True, defer_finalize: bool = False, ) -> Generator[KVConnectorOutput, None, None]: output = KVConnectorOutput() @@ -89,7 +88,7 @@ class KVConnectorModelRunnerMixin: finally: if start_after_forward: kv_connector.start_load_kv(get_forward_context()) - if wait_for_save and not defer_finalize: + if not defer_finalize: kv_connector.wait_for_save() transfer_results = kv_connector.get_transfer_results( diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 6c2a05a626..ff6d1a5a6f 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -412,8 +412,8 @@ def allocate_kv_cache( warmup_rocm_skinny_gemm_workspaces(device) # Pad to the page granularity MoRIIO needs to register the shared # backing as a single RDMA memory region. Other platforms keep the - # exact-size allocation: NIXL and SimpleCPUOffload rely on - # storage.nbytes() matching the logical KV size (see #53974). + # exact-size allocation (see #53974), so anything reading + # storage.nbytes() has to tolerate the tail on ROCm alone. page_size = 4096 buf_size = ((raw_size + page_size - 1) // page_size) * page_size else: diff --git a/vllm/v1/worker/worker_base.py b/vllm/v1/worker/worker_base.py index b0839bc4ca..017b1230cd 100644 --- a/vllm/v1/worker/worker_base.py +++ b/vllm/v1/worker/worker_base.py @@ -268,6 +268,12 @@ class WorkerWrapperBase: load_general_plugins() + # Let the platform replace core Triton kernels (e.g. CPU fallback + # implementations) before any worker or model code can launch them. + from vllm.platforms import current_platform + + current_platform.register_triton_kernel_overrides() + parallel_config = vllm_config.parallel_config if isinstance(parallel_config.worker_cls, str): worker_class: type[WorkerBase] = resolve_obj_by_qualname( diff --git a/vllm/vllm_flash_attn/flash_attn_interface.py b/vllm/vllm_flash_attn/flash_attn_interface.py index c10b381e40..3c4c0d8948 100644 --- a/vllm/vllm_flash_attn/flash_attn_interface.py +++ b/vllm/vllm_flash_attn/flash_attn_interface.py @@ -434,6 +434,14 @@ def flash_attn_varlen_func( from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd + # SM90 FA4 fp8-KV path: fp8 e4m3 paged K/V dequantized and the K/V descale folded + # in-kernel; accepts bf16/fp16 Q and writes O in its native dtype (no Q cast, no + # output copy). Only the (batch, num_kv_heads) f32 K/V descales are forwarded. + fa4_fp8_kv_dequant = ( + k.dtype == torch.float8_e4m3fn + and torch.cuda.get_device_capability()[0] == 9 + ) + out, softmax_lse, _, _ = _flash_attn_fwd( q, k, @@ -458,10 +466,11 @@ def flash_attn_varlen_func( block_sparse_tensors=block_sparse_tensors, aux_tensors=aux_tensors, aux_tensor_leading_dims=aux_tensor_leading_dims, - q_descale=q_descale, + q_descale=None if fa4_fp8_kv_dequant else q_descale, k_descale=k_descale, v_descale=v_descale, output_scale=output_scale, + fp8_kv_dequant=fa4_fp8_kv_dequant, ) else: raise ValueError(f"Unsupported FA version: {fa_version}")